Fix stats tests after merge of #2582

The problem was, msgq started to open more file descriptors than before.
Also, the changes meant the tests didn't shut down their internal copy
of msgq properly, leaking more file descriptors.

* Do the shutdown correctly.
* Wait for msgq to terminate, so there are no race conditions between
  the old (shutting down one) and a new one for another test.
* Close all the data sockets in msgq, to prevent further leaks. The
  other ends probably still leak, but at least the threads on the other
  ends terminate on EOF, so we don't leak the threads too.
* Wait for the other threads too, so we don't spawn too many threads.
* Unrelated cleanup: remove unneeded output-suppression workaround from
  the tests, it is no longer needed.
* Unrelated cleanup: Initialize testing logger for the tests. But it
  still logs for no apparent reason, needs to be examined.

src/bin/msgq/msgq.py.in

@@ -322,7 +322,6 @@ class MsgQ:
             logger.error(MSGQ_READ_UNKNOWN_FD, fd)
             return
         sock = self.sockets[fd]
-#        sys.stderr.write("[b10-msgq] Got read on fd %d\n" %fd)
         self.process_packet(fd, sock)
 
     def kill_socket(self, fd, sock):
@@ -650,6 +649,18 @@ class MsgQ:
         logger.debug(TRACE_START, MSGQ_SHUTDOWN)
         self.listen_socket.close()
         self.cleanup_signalsock()
+        # Close all the sockets too. In real life, there should be none now,
+        # as Msgq should be the last one. But some tests don't adhere to this
+        # and create a new Msgq for each test, which led to huge socket leaks.
+        # Some other threads put some other things in instead of sockets, so
+        # we catch whatever exceptions there we can. This should be safe,
+        # because in real operation, we will terminate now anyway, implicitly
+        # closing anything anyway.
+        for sock in self.sockets.values():
+            try:
+                sock.close()
+            except Exception:
+                pass
         if os.path.exists(self.socket_file):
             os.remove(self.socket_file)
 

src/bin/stats/tests/b10-stats-httpd_test.py

@@ -42,6 +42,7 @@ except ImportError:
     lxml_etree = None
 
 import isc
+import isc.log
 import stats_httpd
 import stats
 from test_utils import BaseModules, ThreadingServerManager, MyStats,\
@@ -1066,4 +1067,5 @@ class TestStatsHttpd(unittest.TestCase):
             imp.reload(stats_httpd)
 
 if __name__ == "__main__":
+    isc.log.resetUnitTestRootLogger()
     unittest.main()

src/bin/stats/tests/b10-stats_test.py

@@ -29,6 +29,7 @@ import time
 import imp
 
 import stats
+import isc.log
 import isc.cc.session
 from test_utils import BaseModules, ThreadingServerManager, MyStats, SignalHandler, send_command, send_shutdown
 from isc.testutils.ccsession_mock import MockModuleCCSession
@@ -1254,8 +1255,6 @@ class TestOSEnv(unittest.TestCase):
         os.environ["B10_FROM_SOURCE"] = path
         imp.reload(stats)
 
-def test_main():
-    unittest.main()
-
 if __name__ == "__main__":
-    test_main()
+    isc.log.resetUnitTestRootLogger()
+    unittest.main()

src/bin/stats/tests/test_utils.py

@@ -103,20 +103,9 @@ class ThreadingServerManager:
         else:
             self.server._thread.join(0) # timeout is 0
 
-def do_nothing(*args, **kwargs): pass
-
-class dummy_sys:
-    """Dummy for sys"""
-    class dummy_io:
-        write = do_nothing
-    stdout = stderr = dummy_io()
-
 class MockMsgq:
     def __init__(self):
         self._started = threading.Event()
-        # suppress output to stdout and stderr
-        msgq.sys = dummy_sys()
-        msgq.print = do_nothing
         self.msgq = msgq.MsgQ(verbose=False)
         result = self.msgq.setup()
         if result:
@@ -124,10 +113,15 @@ class MockMsgq:
 
     def run(self):
         self._started.set()
-        self.msgq.run()
+        try:
+            self.msgq.run()
+        finally:
+            # Make sure all the sockets, etc, are removed once it stops.
+            self.msgq.shutdown()
 
     def shutdown(self):
-        self.msgq.shutdown()
+        # Ask it to terminate nicely
+        self.msgq.stop()
 
 class MockCfgmgr:
     def __init__(self):
@@ -554,15 +548,20 @@ class BaseModules:
 
 
     def shutdown(self):
+        # MockMsgq. We need to wait (blocking) for it, otherwise it'll wipe out
+        # a socket for another test during its shutdown.
+        self.msgq.shutdown(True)
+
+        # We also wait for the others, but these are just so we don't create
+        # too many threads in parallel.
+
         # MockAuth
-        self.auth2.shutdown()
-        self.auth.shutdown()
+        self.auth2.shutdown(True)
+        self.auth.shutdown(True)
         # MockBoss
-        self.boss.shutdown()
+        self.boss.shutdown(True)
         # MockCfgmgr
-        self.cfgmgr.shutdown()
-        # MockMsgq
-        self.msgq.shutdown()
+        self.cfgmgr.shutdown(True)
         # remove the unused socket file
         socket_file = self.msgq.server.msgq.socket_file
         try: