Merge branch 'trac1751'

Conflicts:
	ChangeLog

ChangeLog

@@ -1,3 +1,9 @@
+xxx.	[bug]		naokikambe
+	The statistic counters are now properly accumulated across multiple
+	instances of b10-auth (if there are multiple instances), instead of
+	providing result for random instance.
+	(Trac #1751, git TBD)
+
 403.	[build]*	jelte
 	The configure option for botan (--with-botan=PATH) is replaced by
 	--with-botan-config=PATH, which takes a full path to a botan-config

src/bin/auth/statistics.cc

@@ -112,9 +112,13 @@ AuthCountersImpl::submitStatistics() const {
         return (false);
     }
     std::stringstream statistics_string;
+    // add pid in order for stats to identify which auth sends
+    // statistics in the situation that multiple auth instances are
+    // working
     statistics_string << "{\"command\": [\"set\","
                       <<   "{ \"owner\": \"Auth\","
-                      <<   "  \"data\":"
+                      <<   "  \"pid\":" << getpid()
+                      <<   ", \"data\":"
                       <<     "{ \"queries.udp\": "
                       <<     server_counter_.get(AuthCounters::SERVER_UDP_QUERY)
                       <<     ", \"queries.tcp\": "

src/bin/auth/tests/statistics_unittest.cc

@@ -281,6 +281,8 @@ TEST_F(AuthCountersTest, submitStatisticsWithoutValidator) {
                          ->get(0)->stringValue());
     EXPECT_EQ("Auth", statistics_session_.sent_msg->get("command")
                          ->get(1)->get("owner")->stringValue());
+    EXPECT_EQ(statistics_session_.sent_msg->get("command")
+              ->get(1)->get("pid")->intValue(), getpid());
     ConstElementPtr statistics_data = statistics_session_.sent_msg
                                           ->get("command")->get(1)
                                           ->get("data");

src/bin/stats/b10-stats.8

@@ -59,23 +59,19 @@ command does not have any configurable settings\&.
 The configuration commands are:
 .PP
 
-
-\fBremove\fR
-removes the named statistics name and data\&.
-.PP
-
-
-\fBreset\fR
-will reset all statistics data to default values except for constant names\&. This may re\-add previously removed statistics names\&.
-.PP
-
 \fBset\fR
+will set new statistics data specified in arguments\&. Statistics data to be set and the module name which owns statistics data are required in argument\&. Pid of the module in argument is optional\&.
 .PP
 
 \fBshow\fR
 will send the statistics data in JSON format\&. By default, it outputs all the statistics data it has collected\&. An optional item name may be specified to receive individual output\&.
 .PP
 
+\fBshowschema\fR
+will send the schema of the statistics data in JSON format\&. The output is equivalent to the statistics part of
+stats\&.spec\&.
+.PP
+
 \fBshutdown\fR
 will shutdown the
 \fBb10\-stats\fR

src/bin/stats/b10-stats.xml

@@ -101,20 +101,10 @@
     </para>
 
     <para>
-<!-- TODO: remove is removed in trac930 -->
-      <command>remove</command> removes the named statistics name and data.
-    </para>
-
-    <para>
-<!-- TODO: reset is removed in trac930 -->
-      <command>reset</command> will reset all statistics data to
-      default values except for constant names.
-      This may re-add previously removed statistics names.
-    </para>
-
-    <para>
-      <command>set</command>
-<!-- TODO: document this -->
+      <command>set</command> will set new statistics data specified in
+      arguments. Statistics data to be set and the module name which owns
+      statistics data are required in argument. Pid of the module in argument
+      is optional.
     </para>
 
     <para>
@@ -124,7 +114,11 @@
       An optional item name may be specified to receive individual output.
     </para>
 
-<!-- TODO: document showschema -->
+    <para>
+      <command>showschema</command> will send the schema of the statistics data
+      in JSON format. The output is equivalent to the statistics part
+      of <filename>stats.spec</filename>.
+    </para>
 
     <para>
       <command>shutdown</command> will shutdown the

src/bin/stats/stats.py.in

@@ -129,6 +129,8 @@ class Stats:
         self.module_name = self.mccs.get_module_spec().get_module_name()
         self.modules = {}
         self.statistics_data = {}
+        # statistics data by each pid
+        self.statistics_data_bypid = {}
         # get commands spec
         self.commands_spec = self.mccs.get_module_spec().get_commands_spec()
         # add event handler related command_handler of ModuleCCSession
@@ -265,36 +267,129 @@ class Stats:
                          + "owner: " + str(owner) + ", "
                          + "name: " + str(name))
 
-    def update_statistics_data(self, owner=None, **data):
+    def update_statistics_data(self, owner=None, pid=-1, **data):
         """
         change statistics date of specified module into specified
         data. It updates information of each module first, and it
         updates statistics data. If specified data is invalid for
         statistics spec of specified owner, it returns a list of error
-        messeges. If there is no error or if neither owner nor data is
-        specified in args, it returns None.
+        messages. If there is no error or if neither owner nor data is
+        specified in args, it returns None. pid is the process id of
+        the sender module in order for stats to identify which
+        instance sends statistics data in the situation that multiple
+        instances are working.
         """
+        # Note:
+        # The fix of #1751 is for multiple instances working. It is
+        # assumed here that they send different statistics data with
+        # each PID. Stats should save their statistics data by
+        # PID. The statistics data, which is the existing variable, is
+        # preserved by accumlating from statistics data by PID. This
+        # is an ad-hoc fix because administrators can not see
+        # statistics by each instance via bindctl or HTTP/XML. These
+        # interfaces aren't changed in this fix.
+
+        def _accum_bymodule(statistics_data_bypid):
+            # This is an internal function for the superordinate
+            # function. It accumulates statistics data of each PID by
+            # module. It returns the accumulation result.
+            def _accum(a, b):
+                # If the first arg is dict or list type, two values
+                # would be merged and accumlated.
+                if type(a) is dict:
+                    return dict([ (k, _accum(v, b[k])) \
+                                      if k in b else (k, v) \
+                                      for (k, v) in a.items() ] \
+                                    + [ (k, v) \
+                                            for (k, v) in b.items() \
+                                            if k not in a ])
+                elif type(a) is list:
+                    return [ _accum(a[i], b[i]) \
+                                 if len(b) > i else a[i] \
+                                 for i in range(len(a)) ] \
+                                 + [ b[i] \
+                                         for i in range(len(b)) \
+                                         if len(a) <= i ]
+                # If the first arg is integer or float type, two
+                # values are just added.
+                elif type(a) is int or type(a) is float:
+                    return a + b
+                # If the first arg is str or other types than above,
+                # then it just returns the first arg which is assumed
+                # to be the newer value.
+                return a
+            ret = {}
+            for data in statistics_data_bypid.values():
+                ret.update(_accum(data, ret))
+            return ret
+
+        # Firstly, it gets default statistics data in each spec file.
         self.update_modules()
         statistics_data = {}
         for (name, module) in self.modules.items():
             value = get_spec_defaults(module.get_statistics_spec())
             if module.validate_statistics(True, value):
                 statistics_data[name] = value
-        for (name, value) in self.statistics_data.items():
-            if name in statistics_data:
-                statistics_data[name].update(value)
-            else:
-                statistics_data[name] = value
         self.statistics_data = statistics_data
+
+        # If the "owner" and "data" arguments in this function are
+        # specified, then the variable of statistics data of each pid
+        # would be updated.
+        errors = []
         if owner and data:
-            errors = []
             try:
                 if self.modules[owner].validate_statistics(False, data, errors):
-                    self.statistics_data[owner].update(data)
-                    return
+                    if owner in self.statistics_data_bypid:
+                        if pid in self.statistics_data_bypid[owner]:
+                            self.statistics_data_bypid[owner][pid].update(data)
+                        else:
+                            self.statistics_data_bypid[owner][pid] = data
+                    else:
+                        self.statistics_data_bypid[owner] = { pid : data }
             except KeyError:
                 errors.append("unknown module name: " + str(owner))
-            return errors
+
+        # If there are inactive instances, which was actually running
+        # on the system before, their statistics data would be
+        # removed. To find inactive instances, it invokes the
+        # "show_processes" command to Boss via the cc session. Then it
+        # gets active instance list and compares its PIDs with PIDs in
+        # statistics data which it already has. If inactive instances
+        # are found, it would remove their statistics data.
+        seq = self.cc_session.group_sendmsg(
+            isc.config.ccsession.create_command("show_processes", None),
+            "Boss")
+        (answer, env) = self.cc_session.group_recvmsg(False, seq)
+        if answer:
+            (rcode, value) = isc.config.ccsession.parse_answer(answer)
+            if rcode == 0:
+                if type(value) is list and len(value) > 0 \
+                        and type(value[0]) is list and len(value[0]) > 1:
+                    mlist = [ k for k in self.statistics_data_bypid.keys() ]
+                    for m in mlist:
+                        # PID list which it has before except for -1
+                        plist1 = [ p for p in self.statistics_data_bypid[m]\
+                                       .keys() if p != -1]
+                        # PID list of active instances which is
+                        # received from Boss
+                        plist2 = [ v[0] for v in value \
+                                       if v[1].lower().find(m.lower()) \
+                                       >= 0 ]
+                        # get inactive instance list by the difference
+                        # between plist1 and plist2
+                        nplist = set(plist1).difference(set(plist2))
+                        for p in nplist:
+                            self.statistics_data_bypid[m].pop(p)
+                        if self.statistics_data_bypid[m]:
+                            if m in self.statistics_data:
+                                self.statistics_data[m].update(
+                                    _accum_bymodule(
+                                        self.statistics_data_bypid[m]))
+                        # remove statistics data of the module with no
+                        # PID
+                        else:
+                            self.statistics_data_bypid.pop(m)
+        if errors: return errors
 
     def command_status(self):
         """
@@ -379,11 +474,11 @@ class Stats:
                 1, "specified arguments are incorrect: " \
                     + "owner: " + str(owner) + ", name: " + str(name))
 
-    def command_set(self, owner, data):
+    def command_set(self, owner, pid=-1, data={}):
         """
         handle set command
         """
-        errors = self.update_statistics_data(owner, **data)
+        errors = self.update_statistics_data(owner, pid, **data)
         if errors:
             return isc.config.create_answer(
                 1, "errors while setting statistics data: " \

src/bin/stats/stats.spec

@@ -72,6 +72,13 @@
             "item_description": "module name of the owner of the statistics data"
           },
 	  {
+	    "item_name": "pid",
+            "item_type": "integer",
+            "item_optional": true,
+            "item_default": -1,
+            "item_description": "process id of the owner module"
+          },
+	  {
 	    "item_name": "data",
             "item_type": "map",
             "item_optional": false,

src/bin/stats/tests/b10-stats_test.py

@@ -373,6 +373,68 @@ class TestStats(unittest.TestCase):
         self.assertEqual(self.stats.update_statistics_data(owner='Dummy', foo='bar'),
                          ['unknown module name: Dummy'])
 
+    def test_update_statistics_data_withpid(self):
+        # one pid of Auth
+        self.stats.update_statistics_data(owner='Auth',
+                                          pid=9999,
+                                          **{'queries.tcp':1001})
+        self.assertTrue('Auth' in self.stats.statistics_data)
+        self.assertTrue('queries.tcp' in self.stats.statistics_data['Auth'])
+        self.assertEqual(self.stats.statistics_data['Auth']['queries.tcp'], 1001)
+        self.assertTrue('Auth' in self.stats.statistics_data_bypid)
+        self.assertTrue(9999 in self.stats.statistics_data_bypid['Auth'])
+        self.assertTrue('queries.tcp' in self.stats.statistics_data_bypid['Auth'][9999])
+        self.assertEqual(self.stats.statistics_data_bypid['Auth'][9999]['queries.tcp'], 1001)
+        self.assertEqual(self.stats.statistics_data_bypid,
+                         {'Auth': {9999: {'queries.tcp': 1001}}})
+        # non-existent pid of Auth, but no changes in statistics data
+        self.stats.update_statistics_data(owner='Auth',
+                                          pid=10000,
+                                          **{'queries.tcp':2001})
+        self.assertTrue('Auth' in self.stats.statistics_data)
+        self.assertTrue('queries.tcp' in self.stats.statistics_data['Auth'])
+        self.assertEqual(self.stats.statistics_data['Auth']['queries.tcp'], 1001)
+        self.assertTrue('Auth' in self.stats.statistics_data_bypid)
+        self.assertTrue(9999 in self.stats.statistics_data_bypid['Auth'])
+        self.assertTrue('queries.tcp' in self.stats.statistics_data_bypid['Auth'][9999])
+        self.assertEqual(self.stats.statistics_data_bypid['Auth'][9999]['queries.tcp'], 1001)
+        self.assertEqual(self.stats.statistics_data_bypid,
+                         {'Auth': {9999: {'queries.tcp': 1001}}})
+        # kill running Auth, then statistics is reset
+        self.assertEqual(self.base.boss.server.pid_list[0][0], 9999)
+        killed = self.base.boss.server.pid_list.pop(0)
+        self.stats.update_statistics_data()
+        self.assertTrue('Auth' in self.stats.statistics_data)
+        self.assertTrue('queries.tcp' in self.stats.statistics_data['Auth'])
+        self.assertTrue('queries.udp' in self.stats.statistics_data['Auth'])
+        self.assertEqual(self.stats.statistics_data['Auth']['queries.tcp'], 0)
+        self.assertEqual(self.stats.statistics_data['Auth']['queries.udp'], 0)
+        self.assertFalse('Auth' in self.stats.statistics_data_bypid)
+        # restore statistics data of killed auth
+        self.base.boss.server.pid_list = [ killed ] + self.base.boss.server.pid_list[:]
+        self.stats.update_statistics_data(owner='Auth',
+                                          pid=9999,
+                                          **{'queries.tcp':1001})
+        # another pid of Auth
+        self.stats.update_statistics_data(owner='Auth',
+                                          pid=9998,
+                                          **{'queries.tcp':1002,
+                                             'queries.udp':1003})
+        self.assertTrue('Auth' in self.stats.statistics_data)
+        self.assertTrue('queries.tcp' in self.stats.statistics_data['Auth'])
+        self.assertTrue('queries.udp' in self.stats.statistics_data['Auth'])
+        self.assertEqual(self.stats.statistics_data['Auth']['queries.tcp'], 2003)
+        self.assertEqual(self.stats.statistics_data['Auth']['queries.udp'], 1003)
+        self.assertTrue('Auth' in self.stats.statistics_data_bypid)
+        self.assertTrue(9999 in self.stats.statistics_data_bypid['Auth'])
+        self.assertTrue(9998 in self.stats.statistics_data_bypid['Auth'])
+        self.assertTrue('queries.tcp' in self.stats.statistics_data_bypid['Auth'][9999])
+        self.assertTrue('queries.udp' in self.stats.statistics_data_bypid['Auth'][9998])
+        self.assertTrue('queries.udp' in self.stats.statistics_data_bypid['Auth'][9998])
+        self.assertEqual(self.stats.statistics_data_bypid['Auth'][9999]['queries.tcp'], 1001)
+        self.assertEqual(self.stats.statistics_data_bypid['Auth'][9998]['queries.tcp'], 1002)
+        self.assertEqual(self.stats.statistics_data_bypid['Auth'][9998]['queries.udp'], 1003)
+
     def test_commands(self):
         # status
         self.assertEqual(self.stats.command_status(),
@@ -710,6 +772,123 @@ class TestStats(unittest.TestCase):
         self.assertRaises(stats.StatsError,
                           self.stats.command_set, owner='Stats', data={ 'dummy' : '_xxxx_yyyy_zzz_' })
 
+    def test_command_set_withpid(self):
+        # one pid of Auth
+        retval = isc.config.ccsession.parse_answer(
+            self.stats.command_set(owner='Auth',
+                                   pid=9997,
+                                   data={ 'queries.tcp' : 1001,
+                                          'queries.perzone':
+                                              [{ 'zonename': 'test1.example',
+                                                 'queries.tcp': 1 },
+                                               { 'zonename': 'test2.example',
+                                                 'queries.tcp': 2,
+                                                 'queries.udp': 3 }]}))
+        self.assertEqual(retval, (0,None))
+        self.assertTrue('Auth' in self.stats.statistics_data)
+        self.assertTrue('queries.tcp' in self.stats.statistics_data['Auth'])
+        self.assertEqual(self.stats.statistics_data['Auth']['queries.tcp'], 1001)
+        self.assertEqual(self.stats.statistics_data['Auth']['queries.perzone'],
+                         [{ 'zonename': 'test1.example',
+                            'queries.tcp': 1 },
+                          { 'zonename': 'test2.example',
+                            'queries.tcp': 2,
+                            'queries.udp': 3 }])
+        self.assertTrue('Stats' in self.stats.statistics_data)
+        self.assertTrue('last_update_time' in self.stats.statistics_data['Stats'])
+        self.assertTrue('Auth' in self.stats.statistics_data_bypid)
+        self.assertTrue(9997 in self.stats.statistics_data_bypid['Auth'])
+        self.assertTrue('queries.tcp' in self.stats.statistics_data_bypid['Auth'][9997])
+        self.assertTrue('queries.perzone' in self.stats.statistics_data_bypid['Auth'][9997])
+        self.assertEqual(self.stats.statistics_data_bypid['Auth'][9997]['queries.tcp'], 1001)
+        self.assertEqual(self.stats.statistics_data_bypid['Auth'][9997]['queries.perzone'],
+                         [{ 'zonename': 'test1.example',
+                            'queries.tcp': 1 },
+                          { 'zonename': 'test2.example',
+                            'queries.tcp': 2,
+                            'queries.udp': 3 }])
+        # non-existent pid of Auth, but no changes in statistics data
+        retval = isc.config.ccsession.parse_answer(
+            self.stats.command_set(owner='Auth',
+                                   pid=10000,
+                                   data={ 'queries.tcp' : 2001,
+                                          'queries.perzone':
+                                              [{ 'zonename': 'test1.example',
+                                                 'queries.tcp': 101 },
+                                               { 'zonename': 'test2.example',
+                                                 'queries.tcp': 102,
+                                                 'queries.udp': 103 }]}))
+        self.assertEqual(retval, (0,None))
+        self.assertTrue('Auth' in self.stats.statistics_data)
+        self.assertTrue('queries.tcp' in self.stats.statistics_data['Auth'])
+        self.assertEqual(self.stats.statistics_data['Auth']['queries.tcp'], 1001)
+        self.assertEqual(self.stats.statistics_data['Auth']['queries.perzone'],
+                         [{ 'zonename': 'test1.example',
+                            'queries.tcp': 1 },
+                          { 'zonename': 'test2.example',
+                            'queries.tcp': 2,
+                            'queries.udp': 3 }])
+        self.assertTrue('Auth' in self.stats.statistics_data_bypid)
+        self.assertTrue(9997 in self.stats.statistics_data_bypid['Auth'])
+        self.assertTrue('queries.tcp' in self.stats.statistics_data_bypid['Auth'][9997])
+        self.assertEqual(self.stats.statistics_data_bypid['Auth'][9997]['queries.tcp'], 1001)
+        self.assertEqual(self.stats.statistics_data_bypid['Auth'][9997]['queries.perzone'],
+                         [{ 'zonename': 'test1.example',
+                            'queries.tcp': 1 },
+                          { 'zonename': 'test2.example',
+                            'queries.tcp': 2,
+                            'queries.udp': 3 }])
+        # another pid of Auth
+        retval = isc.config.ccsession.parse_answer(
+            self.stats.command_set(owner='Auth',
+                                   pid=9996,
+                                   data={ 'queries.tcp' : 1002,
+                                          'queries.udp' : 1003,
+                                          'queries.perzone':
+                                              [{ 'zonename': 'test1.example',
+                                                 'queries.tcp': 10,
+                                                 'queries.udp': 11},
+                                               { 'zonename': 'test2.example',
+                                                 'queries.tcp': 12,
+                                                 'queries.udp': 13 }]}))
+        self.assertEqual(retval, (0,None))
+        self.assertTrue('Auth' in self.stats.statistics_data)
+        self.assertTrue('queries.tcp' in self.stats.statistics_data['Auth'])
+        self.assertTrue('queries.udp' in self.stats.statistics_data['Auth'])
+        self.assertTrue('queries.perzone' in self.stats.statistics_data['Auth'])
+        self.assertEqual(self.stats.statistics_data['Auth']['queries.tcp'], 2003)
+        self.assertEqual(self.stats.statistics_data['Auth']['queries.udp'], 1003)
+        self.assertEqual(self.stats.statistics_data['Auth']['queries.perzone'],
+                         [{ 'zonename': 'test1.example',
+                            'queries.tcp': 11,
+                            'queries.udp': 11},
+                          { 'zonename': 'test2.example',
+                            'queries.tcp': 14,
+                            'queries.udp': 16 }])
+        self.assertTrue('Auth' in self.stats.statistics_data_bypid)
+        self.assertTrue(9997 in self.stats.statistics_data_bypid['Auth'])
+        self.assertTrue(9996 in self.stats.statistics_data_bypid['Auth'])
+        self.assertTrue('queries.tcp' in self.stats.statistics_data_bypid['Auth'][9997])
+        self.assertTrue('queries.udp' in self.stats.statistics_data_bypid['Auth'][9996])
+        self.assertTrue('queries.udp' in self.stats.statistics_data_bypid['Auth'][9996])
+        self.assertTrue('queries.perzone' in self.stats.statistics_data_bypid['Auth'][9996])
+        self.assertEqual(self.stats.statistics_data_bypid['Auth'][9997]['queries.tcp'], 1001)
+        self.assertEqual(self.stats.statistics_data_bypid['Auth'][9997]['queries.perzone'],
+                         [{ 'zonename': 'test1.example',
+                            'queries.tcp': 1 },
+                          { 'zonename': 'test2.example',
+                            'queries.tcp': 2,
+                            'queries.udp': 3 }])
+        self.assertEqual(self.stats.statistics_data_bypid['Auth'][9996]['queries.tcp'], 1002)
+        self.assertEqual(self.stats.statistics_data_bypid['Auth'][9996]['queries.udp'], 1003)
+        self.assertEqual(self.stats.statistics_data_bypid['Auth'][9996]['queries.perzone'],
+                         [{ 'zonename': 'test1.example',
+                            'queries.tcp': 10,
+                            'queries.udp': 11},
+                          { 'zonename': 'test2.example',
+                            'queries.tcp': 12,
+                            'queries.udp': 13 }])
+
 class TestOSEnv(unittest.TestCase):
     def test_osenv(self):
         """

src/bin/stats/tests/test_utils.py

@@ -150,6 +150,11 @@ class MockBoss:
         "command_name": "sendstats",
         "command_description": "Send data to a statistics module at once",
         "command_args": []
+      },
+      {
+        "command_name": "show_processes",
+        "command_description": "List the running BIND 10 processes",
+        "command_args": []
       }
     ],
     "statistics": [
@@ -180,6 +185,10 @@ class MockBoss:
         self.spec_file.close()
         self.cc_session = self.mccs._session
         self.got_command_name = ''
+        self.pid_list = [[ 9999, "b10-auth"   ],
+                         [ 9998, "b10-auth-2" ],
+                         [ 9997, "b10-auth-3" ],
+                         [ 9996, "b10-auth-4" ]]
 
     def run(self):
         self.mccs.start()
@@ -210,6 +219,10 @@ class MockBoss:
             return isc.config.create_answer(0)
         elif command == 'getstats':
             return isc.config.create_answer(0, params)
+        elif command == 'show_processes':
+            # Return dummy pids
+            return isc.config.create_answer(
+                0, self.pid_list)
         return isc.config.create_answer(1, "Unknown Command")
 
 class MockAuth:
@@ -340,7 +353,7 @@ class MockAuth:
             params = { "owner": "Auth",
                        "data": { 'queries.tcp': self.queries_tcp,
                                  'queries.udp': self.queries_udp,
-                                 'queries.per-zone' : self.queries_per_zone } }
+                                 'queries.perzone' : self.queries_per_zone } }
             return send_command("set", "Stats", params=params, session=self.cc_session)
         return isc.config.create_answer(1, "Unknown Command")
 

tests/system/bindctl/tests.sh

@@ -25,9 +25,12 @@ status=0
 n=0
 
 # TODO: consider consistency with statistics definition in auth.spec
-auth_queries_tcp="\<queries\.tcp\>"
-auth_queries_udp="\<queries\.udp\>"
-auth_opcode_queries="\<opcode\.query\>"
+cnt_name1="\<queries\.tcp\>"
+cnt_name2="\<queries\.udp\>"
+cnt_name3="\<opcode\.query\>"
+cnt_value1=0
+cnt_value2=0
+cnt_value3=0
 
 echo "I:Checking b10-auth is working by default ($n)"
 $DIG +norec @10.53.0.1 -p 53210 ns.example.com. A >dig.out.$n || status=1
@@ -45,9 +48,12 @@ echo 'Stats show
 	--csv-file-dir=$BINDCTL_CSV_DIR > bindctl.out.$n || status=1
 # the server should have received 1 UDP and 1 TCP queries (TCP query was
 # sent from the server startup script)
-grep $auth_queries_tcp".*\<1\>" bindctl.out.$n > /dev/null || status=1
-grep $auth_queries_udp".*\<1\>" bindctl.out.$n > /dev/null || status=1
-grep $auth_opcode_queries".*\<2\>" bindctl.out.$n > /dev/null || status=1
+cnt_value1=`expr $cnt_value1 + 1`
+cnt_value2=`expr $cnt_value2 + 1`
+cnt_value3=`expr $cnt_value1 + $cnt_value2`
+grep $cnt_name1".*\<"$cnt_value1"\>" bindctl.out.$n > /dev/null || status=1
+grep $cnt_name2".*\<"$cnt_value2"\>" bindctl.out.$n > /dev/null || status=1
+grep $cnt_name3".*\<"$cnt_value3"\>" bindctl.out.$n > /dev/null || status=1
 if [ $status != 0 ]; then echo "I:failed"; fi
 n=`expr $n + 1`
 
@@ -80,9 +86,12 @@ echo 'Stats show
 ' | $RUN_BINDCTL \
 	--csv-file-dir=$BINDCTL_CSV_DIR > bindctl.out.$n || status=1
 # The statistics counters should have been reset while stop/start.
-grep $auth_queries_tcp".*\<0\>" bindctl.out.$n > /dev/null || status=1
-grep $auth_queries_udp".*\<1\>" bindctl.out.$n > /dev/null || status=1
-grep $auth_opcode_queries".*\<1\>" bindctl.out.$n > /dev/null || status=1
+cnt_value1=0
+cnt_value2=1
+cnt_value3=`expr $cnt_value1 + $cnt_value2`
+grep $cnt_name1".*\<"$cnt_value1"\>" bindctl.out.$n > /dev/null || status=1
+grep $cnt_name2".*\<"$cnt_value2"\>" bindctl.out.$n > /dev/null || status=1
+grep $cnt_name3".*\<"$cnt_value3"\>" bindctl.out.$n > /dev/null || status=1
 if [ $status != 0 ]; then echo "I:failed"; fi
 n=`expr $n + 1`
 
@@ -105,11 +114,68 @@ echo 'Stats show
 ' | $RUN_BINDCTL \
 	--csv-file-dir=$BINDCTL_CSV_DIR > bindctl.out.$n || status=1
 # The statistics counters shouldn't be reset due to hot-swapping datasource.
-grep $auth_queries_tcp".*\<0\>" bindctl.out.$n > /dev/null || status=1
-grep $auth_queries_udp".*\<2\>" bindctl.out.$n > /dev/null || status=1
-grep $auth_opcode_queries".*\<2\>" bindctl.out.$n > /dev/null || status=1
+cnt_value1=`expr $cnt_value1 + 0`
+cnt_value2=`expr $cnt_value2 + 1`
+cnt_value3=`expr $cnt_value1 + $cnt_value2`
+grep $cnt_name1".*\<"$cnt_value1"\>" bindctl.out.$n > /dev/null || status=1
+grep $cnt_name2".*\<"$cnt_value2"\>" bindctl.out.$n > /dev/null || status=1
+grep $cnt_name3".*\<"$cnt_value3"\>" bindctl.out.$n > /dev/null || status=1
 if [ $status != 0 ]; then echo "I:failed"; fi
 n=`expr $n + 1`
 
+echo "I:Starting more b10-auths and checking that ($n)"
+for i in 2 3
+do
+    echo 'config add Boss/components b10-auth-'$i'
+config set Boss/components/b10-auth-'$i' { "special": "auth", "kind": "needed" }
+config commit
+quit
+' | $RUN_BINDCTL \
+	--csv-file-dir=$BINDCTL_CSV_DIR 2>&1 > /dev/null || status=1
+done
+$DIG +norec @10.53.0.1 -p 53210 ns.example.com. A >dig.out.$n || status=1
+grep 192.0.2.2 dig.out.$n > /dev/null || status=1
+if [ $status != 0 ]; then echo "I:failed"; fi
+n=`expr $n + 1`
+
+echo "I:Rechecking BIND 10 statistics consistency after a pause ($n)"
+sleep 2
+cnt_value1=`expr $cnt_value1 + 0`
+cnt_value2=`expr $cnt_value2 + 1`
+cnt_value3=`expr $cnt_value1 + $cnt_value2`
+# Rechecking some times
+for i in 1 2 3 4
+do
+    echo 'Stats show
+' | $RUN_BINDCTL \
+	--csv-file-dir=$BINDCTL_CSV_DIR > bindctl.out.$n || status=1
+    # The statistics counters should keep being consistent even while
+    # multiple b10-auths are running.
+    grep $cnt_name1".*\<"$cnt_value1"\>" bindctl.out.$n > /dev/null || status=1
+    grep $cnt_name2".*\<"$cnt_value2"\>" bindctl.out.$n > /dev/null || status=1
+    grep $cnt_name3".*\<"$cnt_value3"\>" bindctl.out.$n > /dev/null || status=1
+    if [ $status != 0 ]; then echo "I:failed "; break ; fi
+done
+n=`expr $n + 1`
+
+echo "I:Stopping extra b10-auths and checking that ($n)"
+for i in 3 2
+do
+    echo 'config remove Boss/components b10-auth-'$i'
+config commit
+quit
+' | $RUN_BINDCTL \
+	--csv-file-dir=$BINDCTL_CSV_DIR 2>&1 > /dev/null || status=1
+done
+$DIG +norec @10.53.0.1 -p 53210 ns.example.com. A >dig.out.$n || status=1
+grep 192.0.2.2 dig.out.$n > /dev/null || status=1
+if [ $status != 0 ]; then echo "I:failed"; fi
+n=`expr $n + 1`
+
+# The statistics counters can not be rechecked here because the auth
+# instance seems to hang up after one of the multiple auth instances
+# was removed via bindctl. This reason seems to be the same reason as
+# #1703.
+
 echo "I:exit status: $status"
 exit $status