[master] Merge branch 'trac2781'

src/bin/stats/stats.py.in

@@ -1,6 +1,6 @@
 #!@PYTHON@
 
-# Copyright (C) 2010, 2011, 2012  Internet Systems Consortium.
+# Copyright (C) 2010-2013  Internet Systems Consortium.
 #
 # Permission to use, copy, modify, and distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
@@ -124,8 +124,8 @@ def _accum(a, b):
                              if len(a) <= i ]
     # If both of args are integer or float type, two
     # values are added.
-    elif (type(a) is int and type(b) is int) \
-            or (type(a) is float or type(b) is float):
+    elif (type(a) is int or type(a) is float) \
+            and (type(b) is int or type(b) is float):
         return a + b
 
     # If both of args are string type,
@@ -186,6 +186,11 @@ class StatsError(Exception):
     """Exception class for Stats class"""
     pass
 
+class InitSessionTimeout(isc.cc.session.SessionTimeout):
+    """SessionTimeout Exception in the session between the stats module and the
+    init module"""
+    pass
+
 class Stats:
     """
     Main class of stats module
@@ -243,14 +248,12 @@ class Stats:
 
         """
         self.update_modules()
-        if self.update_statistics_data(
+        self.update_statistics_data(
             self.module_name,
             self.cc_session.lname,
             {'lname': self.cc_session.lname,
              'boot_time': get_datetime(_BASETIME),
-             'last_update_time': get_datetime()}):
-            logger.warn(STATS_RECEIVED_INVALID_STATISTICS_DATA,
-                        self.module_name)
+             'last_update_time': get_datetime()})
         # define the variable of the last time of polling
         self._lasttime_poll = 0.0
 
@@ -258,12 +261,8 @@ class Stats:
         """return the current value of 'poll-interval'"""
         return self.config['poll-interval']
 
-    def do_polling(self):
-        """Polls modules for statistics data. Return nothing. First
-           search multiple instances of same module. Second requests
-           each module to invoke 'getstats'. Finally updates internal
-           statistics data every time it gets from each instance."""
-
+    def _get_multi_module_list(self):
+        """Returns a module list which is running as multiple modules."""
         # It counts the number of instances of same module by
         # examining the third value from the array result of
         # 'show_processes' of Init
@@ -277,6 +276,8 @@ class Stats:
             # TODO: Is it OK to just pass? As part of refactoring, preserving
             # the original behaviour.
             value = None
+        except isc.cc.session.SessionTimeout as e:
+            raise InitSessionTimeout(e)
         modules = []
         if type(value) is list:
             # NOTE: For example, the "show_processes" command
@@ -306,6 +307,12 @@ class Stats:
             # release.
             modules = [ v[2] if type(v) is list and len(v) > 2 \
                             else None for v in value ]
+        return modules
+
+    def _query_statistics(self, modules):
+        """Queries each module statistics and returns sequences to use
+        for receiving that answers based on the argument 'modules' which
+        is a list of modules running as multiple modules"""
         # start requesting each module to collect statistics data
         sequences = []
         for (module_name, data) in self.get_statistics_data().items():
@@ -327,11 +334,17 @@ class Stats:
             if cnt > 1:
                 sequences = sequences + [ (module_name, seq) \
                                               for i in range(cnt-1) ]
+        return sequences
+
+    def _collect_statistics(self, sequences):
+        """Based on sequences which is a list of values returned from
+        group_sendmsg(), collects statistics data from each module. If
+        a SessionTimeout exception is raised when collecting from the
+        module, skips it and goes to collect from the next module."""
         # start receiving statistics data
         _statistics_data = []
-        while len(sequences) > 0:
+        for (module_name, seq) in sequences:
             try:
-                (module_name, seq) = sequences.pop(0)
                 answer, env = self.cc_session.group_recvmsg(False, seq)
                 if answer:
                     rcode, args = isc.config.ccsession.parse_answer(answer)
@@ -340,25 +353,37 @@ class Stats:
                             (module_name, env['from'], args))
             # skip this module if SessionTimeout raised
             except isc.cc.session.SessionTimeout:
-                pass
+                logger.warn(STATS_SKIP_COLLECTING, module_name)
+        return _statistics_data
 
+    def _refresh_statistics(self, statistics_data):
+        """Refreshes statistics_data into internal statistics data to
+        display, which is a list of a tuple of module_name, lname, and
+        args"""
         # update statistics data
+        _statistics_data = statistics_data[:]
         self.update_modules()
         while len(_statistics_data) > 0:
             (_module_name, _lname, _args) = _statistics_data.pop(0)
-            if self.update_statistics_data(_module_name, _lname, _args):
-                logger.warn(
-                STATS_RECEIVED_INVALID_STATISTICS_DATA,
-                _module_name)
-            else:
-                if self.update_statistics_data(
+            if not self.update_statistics_data(_module_name, _lname, _args):
+                self.update_statistics_data(
                     self.module_name,
                     self.cc_session.lname,
-                    {'last_update_time': get_datetime()}):
-                    logger.warn(
-                        STATS_RECEIVED_INVALID_STATISTICS_DATA,
-                        self.module_name)
-        # if successfully done, set the last time of polling
+                    {'last_update_time': get_datetime()})
+
+    def do_polling(self):
+        """Polls modules for statistics data. Return nothing. First
+           search multiple instances of same module. Second requests
+           each module to invoke 'getstats'. Finally updates internal
+           statistics data every time it gets from each instance."""
+        try:
+            modules = self._get_multi_module_list()
+            sequences = self._query_statistics(modules)
+            _statistics_data = self._collect_statistics(sequences)
+            self._refresh_statistics(_statistics_data)
+        except InitSessionTimeout:
+            logger.warn(STATS_SKIP_POLLING)
+        # if successfully done or skipped, set the last time of polling
         self._lasttime_poll = get_timestamp()
 
     def _check_command(self, nonblock=False):
@@ -601,7 +626,10 @@ class Stats:
                         self.statistics_data[m],
                         _accum_bymodule(self.statistics_data_bymid[m]))
 
-        if errors: return errors
+        if errors:
+            logger.warn(
+                STATS_RECEIVED_INVALID_STATISTICS_DATA, owner)
+            return errors
 
     def command_status(self):
         """

src/bin/stats/stats_messages.mes

@@ -64,6 +64,17 @@ will respond with an error and the command will be ignored.
 This debug message is printed when a request is sent to the module
 to send its data to the stats module.
 
+% STATS_SKIP_COLLECTING skipped collecting statistics from %1
+The stats module temporarily encountered an internal messaging bus error while
+collecting statistics from the module, then it skipped collecting statistics
+from it and will try to collect from the next module. The lack of statistics
+will be recovered at the next polling round.
+
+% STATS_SKIP_POLLING skipped polling statistics to modules
+The stats module temporarily encountered an internal messaging bus error while
+collecting initial information to collect statistics from the init module, then
+it skipped polling statistics and will try to do next time.
+
 % STATS_STARTING starting
 The stats module will be now starting.
 

src/bin/stats/tests/stats_test.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2010, 2011, 2012  Internet Systems Consortium.
+# Copyright (C) 2010-2013  Internet Systems Consortium.
 #
 # Permission to use, copy, modify, and distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
@@ -31,6 +31,7 @@ import sys
 import stats
 import isc.log
 from test_utils import MyStats
+from isc.config.ccsession import create_answer
 
 class TestUtilties(unittest.TestCase):
     items = [
@@ -132,6 +133,8 @@ class TestUtilties(unittest.TestCase):
         self.assertEqual(stats._accum("a", None), "a")
         self.assertEqual(stats._accum(1, 2), 3)
         self.assertEqual(stats._accum(0.5, 0.3), 0.8)
+        self.assertEqual(stats._accum(1, 0.3), 1.3)
+        self.assertEqual(stats._accum(0.5, 2), 2.5)
         self.assertEqual(stats._accum('aa','bb'), 'bb')
         self.assertEqual(stats._accum('1970-01-01T09:00:00Z','2012-08-09T09:33:31Z'),
                          '2012-08-09T09:33:31Z')
@@ -296,6 +299,43 @@ class TestStats(unittest.TestCase):
         return isc.config.ccsession.parse_answer(
             stats.command_handler(command_name, params))
 
+    def test_check_command(self):
+        """Test _check_command sets proper timeout values and sets proper values
+        returned from group_recvmsg"""
+        stat = MyStats()
+        set_timeouts = []
+        orig_timeout = 1
+        stat.cc_session.get_timeout = lambda: orig_timeout
+        stat.cc_session.set_timeout = lambda x: set_timeouts.append(x)
+        msg = create_answer(0, 'msg')
+        env = {'from': 'frominit'}
+        stat._answers = [(msg, env)]
+        stat._check_command()
+        self.assertListEqual([500, orig_timeout], set_timeouts)
+        self.assertEqual(msg, stat.mccs._msg)
+        self.assertEqual(env, stat.mccs._env)
+
+    def test_check_command_sessiontimeout(self):
+        """Test _check_command doesn't perform but sets proper timeout values in
+        case that a SesstionTimeout exception is caught while doing
+        group_recvmsg()"""
+        stat = MyStats()
+        set_timeouts = []
+        orig_timeout = 1
+        stat.cc_session.get_timeout = lambda: orig_timeout
+        stat.cc_session.set_timeout = lambda x: set_timeouts.append(x)
+        msg = create_answer(0, 'msg')
+        env = {'from': 'frominit'}
+        stat._answers = [(msg, env)]
+        # SessionTimeout is raised while doing group_recvmsg()
+        ex = isc.cc.session.SessionTimeout
+        def __raise(*x): raise ex(*x)
+        stat.cc_session.group_recvmsg = lambda x: __raise()
+        stat._check_command()
+        self.assertListEqual([500, orig_timeout], set_timeouts)
+        self.assertEqual(None, stat.mccs._msg)
+        self.assertEqual(None, stat.mccs._env)
+
     def test_start(self):
         # Define a separate exception class so we can be sure that's actually
         # the one raised in __check_start() below
@@ -315,6 +355,25 @@ class TestStats(unittest.TestCase):
         self.assertRaises(CheckException, self.stats.start)
         self.assertEqual(self.__send_command(self.stats, "status"),
                          (0, "Stats is up. (PID " + str(os.getpid()) + ")"))
+        self.assertTrue(self.stats.mccs.stopped)
+
+    def test_start_set_next_polltime(self):
+        """Test start() properly sets the time next_polltime to do_poll() next
+        time"""
+        orig_get_timestamp = stats.get_timestamp
+        stats.get_timestamp = lambda : self.const_timestamp
+        stat = MyStats()
+        # manupilate next_polltime to go it through the inner if-condition
+        stat.next_polltime = self.const_timestamp - stat.get_interval() - 1
+        # stop an infinity loop at once
+        def __stop_running(): stat.running = False
+        stat.do_polling = __stop_running
+        # do nothing in _check_command()
+        stat._check_command = lambda: None
+        stat.start()
+        # check stat.next_polltime reassigned
+        self.assertEqual(self.const_timestamp, stat.next_polltime)
+        stats.get_timestamp = orig_get_timestamp
 
     def test_shutdown(self):
         def __check_shutdown(tested_stats):
@@ -697,7 +756,6 @@ class TestStats(unittest.TestCase):
 
         # We use the knowledge of what kind of messages are sent via
         # do_polling, and return the following faked answer directly.
-        create_answer = isc.config.ccsession.create_answer # shortcut
         self.stats._answers = [
             # Answer for "show_processes"
             (create_answer(0, [[1034, 'b10-auth-1', 'Auth'],
@@ -817,7 +875,6 @@ class TestStats(unittest.TestCase):
 
         # see the comment for test_update_statistics_data_withmid.  We abuse
         # do_polling here, too.  With #2781 we should make it more direct.
-        create_answer = isc.config.ccsession.create_answer # shortcut
         stat._answers = [\
             # Answer for "show_processes"
             (create_answer(0, []),  None),
@@ -868,7 +925,6 @@ class TestStats(unittest.TestCase):
         self.stats.update_modules = lambda: None
 
         # Test data borrowed from test_update_statistics_data_withmid
-        create_answer = isc.config.ccsession.create_answer # shortcut
         self.stats._answers = [
             (create_answer(0, [[1034, 'b10-auth-1', 'Auth'],
                                [1035, 'b10-auth-2', 'Auth']]),  None),
@@ -1281,6 +1337,163 @@ class TestStats(unittest.TestCase):
                          isc.config.create_answer(
                 1, "module name is not specified"))
 
+    def test_get_multi_module_list(self):
+        """Test _get_multi_module_list() returns a module list which is running
+        as multiple modules."""
+        stat = MyStats()
+        # no answer
+        self.assertListEqual([], stat._get_multi_module_list())
+        # proc list returned
+        proc_list = [
+            [29317, 'b10-xfrout', 'Xfrout'],
+            [29318, 'b10-xfrin', 'Xfrin'],
+            [20061, 'b10-auth','Auth'],
+            [20103, 'b10-auth-2', 'Auth']]
+        mod_list = [ a[2] for a in proc_list ]
+        stat._answers = [
+            # Answer for "show_processes"
+            (create_answer(0, proc_list), {'from': 'init'})
+            ]
+        self.assertListEqual(mod_list, stat._get_multi_module_list())
+        # invalid proc list
+        stat._answers = [
+            # Answer for "show_processes"
+            (create_answer(0, [[999, 'invalid', 'Invalid'], 'invalid']),
+             {'from': 'init'})
+            ]
+        self.assertListEqual(['Invalid', None], stat._get_multi_module_list())
+
+    def test_get_multi_module_list_rpcrecipientmissing(self):
+        """Test _get_multi_module_list() raises an RPCRecipientMissing exception
+        if rcp_call() raise the exception"""
+        # RPCRecipientMissing case
+        stat = MyStats()
+        ex = isc.config.RPCRecipientMissing
+        def __raise(*x): raise ex(*x)
+        stat.mccs.rpc_call = lambda x,y: __raise('Error')
+        self.assertRaises(ex, stat._get_multi_module_list)
+
+    def test_get_multi_module_list_rpcerror(self):
+        """Test _get_multi_module_list() returns an empty list if rcp_call()
+        raise an RPCError exception"""
+        # RPCError case
+        stat = MyStats()
+        ex = isc.config.RPCError
+        def __raise(*x): raise ex(*x)
+        stat.mccs.rpc_call = lambda x,y: __raise(99, 'Error')
+        self.assertListEqual([], stat._get_multi_module_list())
+
+    def test_get_multi_module_list_initsessiontimeout(self):
+        """Test _get_multi_module_list() raises an InitSeeionTimeout exception
+        if a CC session times out in rcp_call()"""
+        # InitSeeionTimeout case
+        stat = MyStats()
+        ex = isc.cc.session.SessionTimeout
+        def __raise(*x): raise ex(*x)
+        stat.mccs.rpc_call = lambda x,y: __raise()
+        self.assertRaises(stats.InitSessionTimeout, stat._get_multi_module_list)
+
+    def test_query_statistics(self):
+        """Test _query_statistics returns a list of pairs of module and
+        sequences from group_sendmsg()"""
+        stat = MyStats()
+        # imitate stat.get_statistics_data().items. The order of the array
+        # returned by items() should be preserved.
+        class DummyDict:
+            def items(self):
+                return [('Init', 'dummy'), ('Stats', 'dummy'), ('Auth', 'dummy')]
+        stat.get_statistics_data = lambda: DummyDict()
+        mod = ('Init', 'Auth', 'Auth')
+        seq = [('Init', stat._seq + 1),
+               ('Auth', stat._seq + 2),
+               ('Auth', stat._seq + 2) ]
+        self.assertListEqual(seq, stat._query_statistics(mod))
+
+    def test_collect_statistics(self):
+        """Test _collect_statistics() collects statistics data from each module
+        based on the sequences which is a list of values returned from
+        group_sendmsg()"""
+        stat = MyStats()
+        seq = [ ('Init', stat._seq + 1),
+                ('Auth', stat._seq + 2),
+                ('Auth', stat._seq + 2) ]
+        ret = [('Init', 'frominit', {'boot_time': '2013-01-01T00:00:00Z'}),
+               ('Auth', 'fromauth1', {'queries.tcp': 100}),
+               ('Auth', 'fromauth2', {'queries.udp': 200})]
+        stat._answers = [
+            (create_answer(0, r[2]), {'from': r[1]}) for r in ret
+            ]
+        self.assertListEqual(ret, stat._collect_statistics(seq))
+
+    def test_collect_statistics_nodata(self):
+        """Test _collect_statistics() returns empty statistics data if
+        a module returns an empty list"""
+        stat = MyStats()
+        seq = []
+        stat._answers = []
+        ret = []
+        self.assertListEqual(ret, stat._collect_statistics(seq))
+
+    def test_collect_statistics_nonzero_rcode(self):
+        """Test _collect_statistics() returns empty statistics data if
+        a module returns non-zero rcode"""
+        stat = MyStats()
+        seq = [('Init', stat._seq + 1)]
+        stat._answers = [
+            (create_answer(1, 'error'), {'from': 'frominit'})
+            ]
+        ret = []
+        self.assertListEqual(ret, stat._collect_statistics(seq))
+
+    def test_collect_statistics_sessiontimeout(self):
+        """Test _collect_statistics() collects statistics data from each module
+        based on the sequences which is a list of values returned from
+        group_sendmsg(). In this test case, SessionTimeout exceptions are raised
+        while collecting from Auth. This tests _collect_statistics skips
+        collecting from Auth."""
+        # SessionTimeout case
+        stat = MyStats()
+        ex = isc.cc.session.SessionTimeout
+        def __raise(*x): raise ex(*x)
+        # SessionTimeout is raised when asking to Auth
+        stat.cc_session.group_recvmsg = lambda x,seq: \
+            __raise() if seq == stat._seq + 2 else stat._answers.pop(0)
+        seq = [ ('Init', stat._seq + 1),
+                ('Auth', stat._seq + 2),
+                ('Auth', stat._seq + 2) ]
+        ret = [('Init', 'frominit', {'boot_time': '2013-01-01T00:00:00Z'})]
+        stat._answers = [
+            (create_answer(0, r[2]), {'from': r[1]}) for r in ret
+            ]
+        self.assertListEqual(ret, stat._collect_statistics(seq))
+
+    def test_refresh_statistics(self):
+        """Test _refresh_statistics() refreshes statistics data from given data
+        """
+        stat = MyStats()
+        self.assertEqual(self.const_default_datetime,
+                         stat.statistics_data['Init']['boot_time'])
+        self.assertEqual(0,
+                         stat.statistics_data['Auth']['queries.tcp'])
+        self.assertEqual(0,
+                         stat.statistics_data['Auth']['queries.udp'])
+        # change stats.get_datetime() for testing 'last_update_time'
+        orig_get_datetime = stats.get_datetime
+        stats.get_datetime = lambda : self.const_datetime
+        arg = [('Init', 'frominit', {'boot_time': '2013-01-01T00:00:00Z'}),
+               ('Auth', 'fromauth1', {'queries.tcp': 100}),
+               ('Auth', 'fromauth2', {'queries.udp': 200})]
+        stat._refresh_statistics(arg)
+        self.assertEqual('2013-01-01T00:00:00Z',
+                         stat.statistics_data['Init']['boot_time'])
+        self.assertEqual(100,
+                         stat.statistics_data['Auth']['queries.tcp'])
+        self.assertEqual(200,
+                         stat.statistics_data['Auth']['queries.udp'])
+        self.assertEqual(self.const_datetime,
+                         stat.statistics_data['Stats']['last_update_time'])
+        stats.get_datetime = orig_get_datetime
+
     def test_polling_init(self):
         """check statistics data of 'Init'."""
 
@@ -1295,7 +1508,6 @@ class TestStats(unittest.TestCase):
         del stat.statistics_data['Auth']
 
         stat.update_modules = lambda: None
-        create_answer = isc.config.ccsession.create_answer # shortcut
 
         stat._answers = [
             # Answer for "show_processes"
@@ -1314,7 +1526,6 @@ class TestStats(unittest.TestCase):
         """check statistics data of multiple instances of same module."""
         stat = MyStats()
         stat.update_modules = lambda: None
-        create_answer = isc.config.ccsession.create_answer # shortcut
 
         # Test data borrowed from test_update_statistics_data_withmid
         stat._answers = [
@@ -1380,35 +1591,52 @@ class TestStats(unittest.TestCase):
         self.assertEqual(stat.statistics_data['Stats']['lname'],
                          stat.mccs._session.lname)
 
-    def test_polling2(self):
-        """Test do_polling() doesn't incorporate broken statistics data.
-
-        Actually, this is not a test for do_polling() itself.  It's bad, but
-        fixing that is a subject of different ticket.
-
+    def test_refresh_statistics_broken_statistics_data(self):
+        """Test _refresh_statistics() doesn't incorporate broken statistics data
         """
         stat = MyStats()
         # check default statistics data of 'Init'
         self.assertEqual(
-             stat.statistics_data['Init'],
-             {'boot_time': self.const_default_datetime})
+            {'boot_time': self.const_default_datetime},
+            stat.statistics_data['Init'])
+        last_update_time = stat.statistics_data['Stats']['last_update_time']
+        # _refresh_statistics() should ignore the invalid statistics_data(type
+        # of boot_time is invalid); default data shouldn't be replaced.
+        arg = [('Init', 'lname', {'boot_time': 1})]
+        stat._refresh_statistics(arg)
+        self.assertEqual(
+            {'boot_time': self.const_default_datetime},
+            stat.statistics_data['Init'])
+        # 'last_update_time' doesn't change
+        self.assertEqual(
+            last_update_time,
+            stat.statistics_data['Stats']['last_update_time'])
 
-        # set invalid statistics
-        create_answer = isc.config.ccsession.create_answer # shortcut
-        stat._answers = [
-            # Answer for "show_processes"
-            (create_answer(0, []),  None),
-            # Answers for "getstats" for Init (type of boot_time is invalid)
-            (create_answer(0, {'boot_time': 1}), {'from': 'init'}),
-            ]
-        stat.update_modules = lambda: None
+    def test_polling_update_lasttime_poll(self):
+        """Test _lasttime_poll is updated after do_polling()
+        """
+        orig_get_timestamp = stats.get_timestamp
+        stats.get_timestamp = lambda : self.const_timestamp
+        stat = MyStats()
+        self.assertEqual(0.0, stat._lasttime_poll)
+        stat.do_polling()
+        self.assertEqual(self.const_timestamp, stat._lasttime_poll)
+        stats.get_timestamp = orig_get_timestamp
 
-        # do_polling() should ignore the invalid answer;
-        # default data shouldn't be replaced.
+    def test_polling_initsessiontimeout(self):
+        """Test _lasttime_poll is updated after do_polling() in case that it catches
+        InitSesionTimeout at _get_multi_module_list()
+        """
+        orig_get_timestamp = stats.get_timestamp
+        stats.get_timestamp = lambda : self.const_timestamp
+        ex = stats.InitSessionTimeout
+        def __raise(*x): raise ex(*x)
+        stat = MyStats()
+        self.assertEqual(0.0, stat._lasttime_poll)
+        stat._get_multi_module_list = lambda: __raise()
         stat.do_polling()
-        self.assertEqual(
-             stat.statistics_data['Init'],
-             {'boot_time': self.const_default_datetime})
+        self.assertEqual(self.const_timestamp, stat._lasttime_poll)
+        stats.get_timestamp = orig_get_timestamp
 
 class Z_TestOSEnv(unittest.TestCase):
     # Running this test would break logging setting.  To prevent it from

src/bin/stats/tests/test_utils.py

@@ -311,6 +311,8 @@ class MyModuleCCSession(isc.config.ConfigData):
         self.stopped = False
         self.closed = False
         self.lname = 'mock_mod_ccs'
+        self._msg = None
+        self._env = None
 
     def start(self):
         pass
@@ -321,6 +323,10 @@ class MyModuleCCSession(isc.config.ConfigData):
     def close(self):
         self.closed = True
 
+    def check_command_without_recvmsg(self, msg, env):
+        self._msg = msg
+        self._env = env
+
 class MyStats(stats.Stats):
     """A faked Stats class for unit tests.
 
@@ -338,7 +344,7 @@ class MyStats(stats.Stats):
         # may want to inspect or tweak them.
 
         # initial seq num for faked group_sendmsg, arbitrary choice.
-        self.__seq = 4200
+        self._seq = 4200
         # if set, use them as faked response to group_recvmsg (see below).
         # it's a list of tuples, each of which is of (answer, envelope).
         self._answers = []
@@ -408,10 +414,10 @@ class MyStats(stats.Stats):
         generated sequence number.
 
         """
-        self.__seq += 1
-        return self.__seq
+        self._seq += 1
+        return self._seq
 
-    def __group_recvmsg(self, nonblocking, seq):
+    def __group_recvmsg(self, nonblocking = True, seq = None):
         """Faked ModuleCCSession.group_recvmsg for tests.
 
         Skipping actual network communication, and returning an internally