Merge branch 'master' of ssh://git.bind10.isc.org/var/bind10/git/bind10

ChangeLog

@@ -1,3 +1,18 @@
+291.    [func]          naokikambe
+	Statistics items are specified by each module's spec file.
+	Stats module can read these through the config manager. Stats
+	module and stats httpd report statistics data and statistics
+	schema by each module via both bindctl and HTTP/XML.
+	(Trac #928,#929,#930,#1175, git 054699635affd9c9ecbe7a108d880829f3ba229e)
+
+290.	[func]		jinmei
+	libdns++/pydnspp: added an option parameter to the "from wire"
+	methods of the Message class.  One option is defined,
+	PRESERVE_ORDER, which specifies the parser to handle each RR
+	separately, preserving the order, and constructs RRsets in the
+	message sections so that each RRset contains only one RR.
+	(Trac #1258, git c874cb056e2a5e656165f3c160e1b34ccfe8b302)
+
 289.	[func]*		jinmei
 	b10-xfrout: ACLs for xfrout can now be configured per zone basis.
 	A per zone ACl is part of a more general zone configuration.  A

configure.ac

@@ -817,14 +817,6 @@ AC_CONFIG_FILES([Makefile
                  src/bin/zonemgr/tests/Makefile
                  src/bin/stats/Makefile
                  src/bin/stats/tests/Makefile
-                 src/bin/stats/tests/isc/Makefile
-                 src/bin/stats/tests/isc/cc/Makefile
-                 src/bin/stats/tests/isc/config/Makefile
-                 src/bin/stats/tests/isc/util/Makefile
-                 src/bin/stats/tests/isc/log/Makefile
-                 src/bin/stats/tests/isc/log_messages/Makefile
-                 src/bin/stats/tests/testdata/Makefile
-                 src/bin/stats/tests/http/Makefile
                  src/bin/usermgr/Makefile
                  src/bin/tests/Makefile
                  src/lib/Makefile

doc/guide/bind10-guide.xml

@@ -1522,24 +1522,30 @@ then change those defaults with config set Resolver/forward_addresses[0]/address
 
     <para>
 
-       This stats daemon provides commands to identify if it is running,
-       show specified or all statistics data, set values, remove data,
-       and reset data.
+       This stats daemon provides commands to identify if it is
+       running, show specified or all statistics data, show specified
+       or all statistics data schema, and set specified statistics
+       data.
 
        For example, using <command>bindctl</command>:
 
        <screen>
 &gt; <userinput>Stats show</userinput>
 {
-    "auth.queries.tcp": 1749,
-    "auth.queries.udp": 867868,
-    "bind10.boot_time": "2011-01-20T16:59:03Z",
-    "report_time": "2011-01-20T17:04:06Z",
-    "stats.boot_time": "2011-01-20T16:59:05Z",
-    "stats.last_update_time": "2011-01-20T17:04:05Z",
-    "stats.lname": "4d3869d9_a@jreed.example.net",
-    "stats.start_time": "2011-01-20T16:59:05Z",
-    "stats.timestamp": 1295543046.823504
+    "Auth": {
+        "queries.tcp": 1749,
+        "queries.udp": 867868
+    },
+    "Boss": {
+        "boot_time": "2011-01-20T16:59:03Z"
+    },
+    "Stats": {
+        "boot_time": "2011-01-20T16:59:05Z",
+        "last_update_time": "2011-01-20T17:04:05Z",
+        "lname": "4d3869d9_a@jreed.example.net",
+        "report_time": "2011-01-20T17:04:06Z",
+        "timestamp": 1295543046.823504
+    }
 }
        </screen>
     </para>

src/bin/auth/auth_messages.mes

@@ -257,4 +257,7 @@ request. The zone manager component has been informed of the request,
 but has returned an error response (which is included in the message). The
 NOTIFY request will not be honored.
 
+% AUTH_INVALID_STATISTICS_DATA invalid specification of statistics data specified
+An error was encountered when the authoritiative server specified
+statistics data which is invalid for the auth specification file.
 

src/bin/auth/auth_srv.cc

@@ -125,6 +125,10 @@ public:
 
     /// The TSIG keyring
     const shared_ptr<TSIGKeyRing>* keyring_;
+
+    /// Bind the ModuleSpec object in config_session_ with
+    /// isc:config::ModuleSpec::validateStatistics.
+    void registerStatisticsValidator();
 private:
     std::string db_file_;
 
@@ -139,6 +143,9 @@ private:
 
     /// Increment query counter
     void incCounter(const int protocol);
+
+    // validateStatistics
+    bool validateStatistics(isc::data::ConstElementPtr data) const;
 };
 
 AuthSrvImpl::AuthSrvImpl(const bool use_cache,
@@ -317,6 +324,7 @@ AuthSrv::setXfrinSession(AbstractSession* xfrin_session) {
 void
 AuthSrv::setConfigSession(ModuleCCSession* config_session) {
     impl_->config_session_ = config_session;
+    impl_->registerStatisticsValidator();
 }
 
 void
@@ -670,6 +678,22 @@ AuthSrvImpl::incCounter(const int protocol) {
     }
 }
 
+void
+AuthSrvImpl::registerStatisticsValidator() {
+    counters_.registerStatisticsValidator(
+        boost::bind(&AuthSrvImpl::validateStatistics, this, _1));
+}
+
+bool
+AuthSrvImpl::validateStatistics(isc::data::ConstElementPtr data) const {
+    if (config_session_ == NULL) {
+        return (false);
+    }
+    return (
+        config_session_->getModuleSpec().validateStatistics(
+            data, true));
+}
+
 ConstElementPtr
 AuthSrvImpl::setDbFile(ConstElementPtr config) {
     ConstElementPtr answer = isc::config::createAnswer();

src/bin/auth/statistics.cc

@@ -37,11 +37,14 @@ public:
     void inc(const AuthCounters::CounterType type);
     bool submitStatistics() const;
     void setStatisticsSession(isc::cc::AbstractSession* statistics_session);
+    void registerStatisticsValidator
+    (AuthCounters::validator_type validator);
     // Currently for testing purpose only
     uint64_t getCounter(const AuthCounters::CounterType type) const;
 private:
     std::vector<uint64_t> counters_;
     isc::cc::AbstractSession* statistics_session_;
+    AuthCounters::validator_type validator_;
 };
 
 AuthCountersImpl::AuthCountersImpl() :
@@ -67,16 +70,25 @@ AuthCountersImpl::submitStatistics() const {
     }
     std::stringstream statistics_string;
     statistics_string << "{\"command\": [\"set\","
-                      <<   "{ \"stats_data\": "
-                      <<     "{ \"auth.queries.udp\": "
+                      <<   "{ \"owner\": \"Auth\","
+                      <<   "  \"data\":"
+                      <<     "{ \"queries.udp\": "
                       <<     counters_.at(AuthCounters::COUNTER_UDP_QUERY)
-                      <<     ", \"auth.queries.tcp\": "
+                      <<     ", \"queries.tcp\": "
                       <<     counters_.at(AuthCounters::COUNTER_TCP_QUERY)
                       <<   " }"
                       <<   "}"
                       << "]}";
     isc::data::ConstElementPtr statistics_element =
         isc::data::Element::fromJSON(statistics_string);
+    // validate the statistics data before send
+    if (validator_) {
+        if (!validator_(
+                statistics_element->get("command")->get(1)->get("data"))) {
+            LOG_ERROR(auth_logger, AUTH_INVALID_STATISTICS_DATA);
+            return (false);
+        }
+    }
     try {
         // group_{send,recv}msg() can throw an exception when encountering
         // an error, and group_recvmsg() will throw an exception on timeout.
@@ -105,6 +117,13 @@ AuthCountersImpl::setStatisticsSession
     statistics_session_ = statistics_session;
 }
 
+void
+AuthCountersImpl::registerStatisticsValidator
+    (AuthCounters::validator_type validator)
+{
+    validator_ = validator;
+}
+
 // Currently for testing purpose only
 uint64_t
 AuthCountersImpl::getCounter(const AuthCounters::CounterType type) const {
@@ -139,3 +158,10 @@ uint64_t
 AuthCounters::getCounter(const AuthCounters::CounterType type) const {
     return (impl_->getCounter(type));
 }
+
+void
+AuthCounters::registerStatisticsValidator
+    (AuthCounters::validator_type validator) const
+{
+    return (impl_->registerStatisticsValidator(validator));
+}

src/bin/auth/statistics.h

@@ -131,6 +131,26 @@ public:
     /// \return the value of the counter specified by \a type.
     ///
     uint64_t getCounter(const AuthCounters::CounterType type) const;
+
+    /// \brief A type of validation function for the specification in
+    /// isc::config::ModuleSpec.
+    ///
+    /// This type might be useful for not only statistics
+    /// specificatoin but also for config_data specification and for
+    /// commnad.
+    ///
+    typedef boost::function<bool(const isc::data::ConstElementPtr&)>
+    validator_type;
+
+    /// \brief Register a function type of the statistics validation
+    /// function for AuthCounters.
+    ///
+    /// This method never throws an exception.
+    ///
+    /// \param validator A function type of the validation of
+    /// statistics specification.
+    ///
+    void registerStatisticsValidator(AuthCounters::validator_type validator) const;
 };
 
 #endif // __STATISTICS_H

src/bin/auth/tests/statistics_unittest.cc

@@ -16,6 +16,8 @@
 
 #include <gtest/gtest.h>
 
+#include <boost/bind.hpp>
+
 #include <cc/data.h>
 #include <cc/session.h>
 
@@ -76,6 +78,13 @@ protected:
     }
     MockSession statistics_session_;
     AuthCounters counters;
+    // no need to be inherited from the original class here.
+    class MockModuleSpec {
+    public:
+        bool validateStatistics(ConstElementPtr, const bool valid) const
+            { return (valid); }
+    };
+    MockModuleSpec module_spec_;
 };
 
 void
@@ -181,7 +190,7 @@ TEST_F(AuthCountersTest, submitStatisticsWithException) {
     statistics_session_.setThrowSessionTimeout(false);
 }
 
-TEST_F(AuthCountersTest, submitStatistics) {
+TEST_F(AuthCountersTest, submitStatisticsWithoutValidator) {
     // Submit statistics data.
     // Validate if it submits correct data.
 
@@ -201,12 +210,69 @@ TEST_F(AuthCountersTest, submitStatistics) {
     // Command is "set".
     EXPECT_EQ("set", statistics_session_.sent_msg->get("command")
                          ->get(0)->stringValue());
+    EXPECT_EQ("Auth", statistics_session_.sent_msg->get("command")
+                         ->get(1)->get("owner")->stringValue());
     ConstElementPtr statistics_data = statistics_session_.sent_msg
                                           ->get("command")->get(1)
-                                          ->get("stats_data");
+                                          ->get("data");
     // UDP query counter is 2 and TCP query counter is 1.
-    EXPECT_EQ(2, statistics_data->get("auth.queries.udp")->intValue());
-    EXPECT_EQ(1, statistics_data->get("auth.queries.tcp")->intValue());
+    EXPECT_EQ(2, statistics_data->get("queries.udp")->intValue());
+    EXPECT_EQ(1, statistics_data->get("queries.tcp")->intValue());
 }
 
+TEST_F(AuthCountersTest, submitStatisticsWithValidator) {
+
+    //a validator for the unittest
+    AuthCounters::validator_type validator;
+    ConstElementPtr el;
+
+    // Submit statistics data with correct statistics validator.
+    validator = boost::bind(
+        &AuthCountersTest::MockModuleSpec::validateStatistics,
+        &module_spec_, _1, true);
+
+    EXPECT_TRUE(validator(el));
+
+    // register validator to AuthCounters
+    counters.registerStatisticsValidator(validator);
+
+    // Counters should be initialized to 0.
+    EXPECT_EQ(0, counters.getCounter(AuthCounters::COUNTER_UDP_QUERY));
+    EXPECT_EQ(0, counters.getCounter(AuthCounters::COUNTER_TCP_QUERY));
+
+    // UDP query counter is set to 2.
+    counters.inc(AuthCounters::COUNTER_UDP_QUERY);
+    counters.inc(AuthCounters::COUNTER_UDP_QUERY);
+    // TCP query counter is set to 1.
+    counters.inc(AuthCounters::COUNTER_TCP_QUERY);
+
+    // checks the value returned by submitStatistics
+    EXPECT_TRUE(counters.submitStatistics());
+
+    // Destination is "Stats".
+    EXPECT_EQ("Stats", statistics_session_.msg_destination);
+    // Command is "set".
+    EXPECT_EQ("set", statistics_session_.sent_msg->get("command")
+                         ->get(0)->stringValue());
+    EXPECT_EQ("Auth", statistics_session_.sent_msg->get("command")
+                         ->get(1)->get("owner")->stringValue());
+    ConstElementPtr statistics_data = statistics_session_.sent_msg
+                                          ->get("command")->get(1)
+                                          ->get("data");
+    // UDP query counter is 2 and TCP query counter is 1.
+    EXPECT_EQ(2, statistics_data->get("queries.udp")->intValue());
+    EXPECT_EQ(1, statistics_data->get("queries.tcp")->intValue());
+
+    // Submit statistics data with incorrect statistics validator.
+    validator = boost::bind(
+        &AuthCountersTest::MockModuleSpec::validateStatistics,
+        &module_spec_, _1, false);
+
+    EXPECT_FALSE(validator(el));
+
+    counters.registerStatisticsValidator(validator);
+
+    // checks the value returned by submitStatistics
+    EXPECT_FALSE(counters.submitStatistics());
+}
 }

src/bin/bind10/bind10_messages.mes

@@ -198,3 +198,7 @@ the message channel.
 % BIND10_UNKNOWN_CHILD_PROCESS_ENDED unknown child pid %1 exited
 An unknown child process has exited. The PID is printed, but no further
 action will be taken by the boss process.
+
+% BIND10_INVALID_STATISTICS_DATA invalid specification of statistics data specified
+An error was encountered when the boss module specified
+statistics data which is invalid for the boss specification file.

src/bin/bind10/bind10_src.py.in

@@ -85,7 +85,7 @@ isc.util.process.rename(sys.argv[0])
 # number, and the overall BIND 10 version number (set in configure.ac).
 VERSION = "bind10 20110223 (BIND 10 @PACKAGE_VERSION@)"
 
-# This is for bind10.boottime of stats module
+# This is for boot_time of Boss
 _BASETIME = time.gmtime()
 
 class RestartSchedule:
@@ -308,9 +308,11 @@ class BoB:
         return process_list
 
     def _get_stats_data(self):
-        return { "stats_data": {
-                    'bind10.boot_time': time.strftime('%Y-%m-%dT%H:%M:%SZ', _BASETIME)
-               }}
+        return { "owner": "Boss",
+                 "data": { 'boot_time':
+                               time.strftime('%Y-%m-%dT%H:%M:%SZ', _BASETIME)
+                           }
+                 }
 
     def command_handler(self, command, args):
         logger.debug(DBG_COMMANDS, BIND10_RECEIVED_COMMAND, command)
@@ -325,15 +327,22 @@ class BoB:
                 answer = isc.config.ccsession.create_answer(0, self._get_stats_data())
             elif command == "sendstats":
                 # send statistics data to the stats daemon immediately
-                cmd = isc.config.ccsession.create_command(
-                    'set', self._get_stats_data())
-                seq = self.cc_session.group_sendmsg(cmd, 'Stats')
-                # Consume the answer, in case it becomes a orphan message.
-                try:
-                    self.cc_session.group_recvmsg(False, seq)
-                except isc.cc.session.SessionTimeout:
-                    pass
-                answer = isc.config.ccsession.create_answer(0)
+                stats_data = self._get_stats_data()
+                valid = self.ccs.get_module_spec().validate_statistics(
+                    True, stats_data["data"])
+                if valid:
+                    cmd = isc.config.ccsession.create_command('set', stats_data)
+                    seq = self.cc_session.group_sendmsg(cmd, 'Stats')
+                    # Consume the answer, in case it becomes a orphan message.
+                    try:
+                        self.cc_session.group_recvmsg(False, seq)
+                    except isc.cc.session.SessionTimeout:
+                        pass
+                    answer = isc.config.ccsession.create_answer(0)
+                else:
+                    logger.fatal(BIND10_INVALID_STATISTICS_DATA);
+                    answer = isc.config.ccsession.create_answer(
+                        1, "specified statistics data is invalid")
             elif command == "ping":
                 answer = isc.config.ccsession.create_answer(0, "pong")
             elif command == "show_processes":

src/bin/bind10/tests/bind10_test.py.in

@@ -137,9 +137,27 @@ class TestBoB(unittest.TestCase):
             def group_sendmsg(self, msg, group):
                 (self.msg, self.group) = (msg, group)
             def group_recvmsg(self, nonblock, seq): pass
+        class DummyModuleCCSession():
+            module_spec = isc.config.module_spec.ModuleSpec({
+                    "module_name": "Boss",
+                    "statistics": [
+                        {
+                            "item_name": "boot_time",
+                            "item_type": "string",
+                            "item_optional": False,
+                            "item_default": "1970-01-01T00:00:00Z",
+                            "item_title": "Boot time",
+                            "item_description": "A date time when bind10 process starts initially",
+                            "item_format": "date-time"
+                            }
+                        ]
+                    })
+            def get_module_spec(self):
+                return self.module_spec
         bob = BoB()
         bob.verbose = True
         bob.cc_session = DummySession()
+        bob.ccs = DummyModuleCCSession()
         # a bad command
         self.assertEqual(bob.command_handler(-1, None),
                          isc.config.ccsession.create_answer(1, "bad command"))
@@ -150,8 +168,9 @@ class TestBoB(unittest.TestCase):
         # "getstats" command
         self.assertEqual(bob.command_handler("getstats", None),
                          isc.config.ccsession.create_answer(0,
-                            { "stats_data": {
-                                'bind10.boot_time': time.strftime('%Y-%m-%dT%H:%M:%SZ', _BASETIME)
+                            { "owner": "Boss",
+                              "data": {
+                                'boot_time': time.strftime('%Y-%m-%dT%H:%M:%SZ', _BASETIME)
                             }}))
         # "sendstats" command
         self.assertEqual(bob.command_handler("sendstats", None),
@@ -159,8 +178,9 @@ class TestBoB(unittest.TestCase):
         self.assertEqual(bob.cc_session.group, "Stats")
         self.assertEqual(bob.cc_session.msg,
                          isc.config.ccsession.create_command(
-                'set', { "stats_data": {
-                        'bind10.boot_time': time.strftime('%Y-%m-%dT%H:%M:%SZ', _BASETIME)
+                "set", { "owner": "Boss",
+                         "data": {
+                        "boot_time": time.strftime("%Y-%m-%dT%H:%M:%SZ", _BASETIME)
                         }}))
         # "ping" command
         self.assertEqual(bob.command_handler("ping", None),

src/bin/stats/Makefile.am

@@ -5,7 +5,7 @@ pkglibexecdir = $(libexecdir)/@PACKAGE@
 pkglibexec_SCRIPTS = b10-stats b10-stats-httpd
 
 b10_statsdir = $(pkgdatadir)
-b10_stats_DATA = stats.spec stats-httpd.spec stats-schema.spec
+b10_stats_DATA = stats.spec stats-httpd.spec
 b10_stats_DATA += stats-httpd-xml.tpl stats-httpd-xsd.tpl stats-httpd-xsl.tpl
 
 nodist_pylogmessage_PYTHON = $(PYTHON_LOGMSGPKG_DIR)/work/stats_messages.py
@@ -21,7 +21,7 @@ CLEANFILES += $(PYTHON_LOGMSGPKG_DIR)/work/stats_httpd_messages.pyc
 
 man_MANS = b10-stats.8 b10-stats-httpd.8
 EXTRA_DIST = $(man_MANS) b10-stats.xml b10-stats-httpd.xml
-EXTRA_DIST += stats.spec stats-httpd.spec stats-schema.spec
+EXTRA_DIST += stats.spec stats-httpd.spec
 EXTRA_DIST += stats-httpd-xml.tpl stats-httpd-xsd.tpl stats-httpd-xsl.tpl
 EXTRA_DIST += stats_messages.mes stats_httpd_messages.mes
 

src/bin/stats/b10-stats-httpd.xml

@@ -57,7 +57,7 @@
       by the BIND 10 boss process (<command>bind10</command>) and eventually
       exited by it.  The server is intended to be server requests by HTTP
       clients like web browsers and third-party modules. When the server is
-      asked, it requests BIND 10 statistics data from
+      asked, it requests BIND 10 statistics data or its schema from
       <command>b10-stats</command>, and it sends the data back in Python
       dictionary format and the server converts it into XML format. The server
       sends it to the HTTP client. The server can send three types of document,
@@ -112,12 +112,6 @@
       of <refentrytitle>bindctl</refentrytitle><manvolnum>1</manvolnum> about
       how to configure the settings.
     </para>
-    <para><filename>/usr/local/share/bind10-devel/stats-schema.spec</filename>
-      <!--TODO: The filename should be computed from prefix-->
-      &mdash; This is a spec file for data schema of
-      of BIND 10 statistics. This schema cannot be configured 
-      via <refentrytitle>bindctl</refentrytitle><manvolnum>1</manvolnum>.
-    </para>
     <para>
       <filename>/usr/local/share/bind10-devel/stats-httpd-xml.tpl</filename>
       <!--TODO: The filename should be computed from prefix-->
@@ -138,7 +132,7 @@
   <refsect1>
     <title>CONFIGURATION AND COMMANDS</title>
     <para>
-      The configurable setting in 
+      The configurable setting in
       <filename>stats-httpd.spec</filename> is:
     </para>
     <variablelist>

src/bin/stats/b10-stats.8

@@ -135,10 +135,6 @@ See other manual pages for explanations for their statistics that are kept track
 \fBb10\-stats\fR\&. It contains commands for
 \fBb10\-stats\fR\&. They can be invoked via
 bindctl(1)\&.
-.PP
-/usr/local/share/bind10\-devel/stats\-schema\&.spec
-\(em This is a spec file for data schema of of BIND 10 statistics\&. This schema cannot be configured via
-bindctl(1)\&.
 .SH "SEE ALSO"
 .PP
 

src/bin/stats/b10-stats.xml

@@ -213,12 +213,6 @@
       invoked
       via <refentrytitle>bindctl</refentrytitle><manvolnum>1</manvolnum>.
     </para>
-    <para><filename>/usr/local/share/bind10-devel/stats-schema.spec</filename>
-      <!--TODO: The filename should be computed from prefix-->
-      &mdash; This is a spec file for data schema of
-      of BIND 10 statistics. This schema cannot be configured 
-      via <refentrytitle>bindctl</refentrytitle><manvolnum>1</manvolnum>.
-    </para>
   </refsect1>
 
   <refsect1>

src/bin/stats/stats-httpd-xsl.tpl

@@ -44,6 +44,7 @@ td.title {
         <h1>BIND 10 Statistics</h1>
         <table>
           <tr>
+            <th>Owner</th>
             <th>Title</th>
             <th>Value</th>
           </tr>

src/bin/stats/stats-schema.spec

@@ -1,86 +0,0 @@
-{
-  "module_spec": {
-    "module_name": "Stats",
-    "module_description": "Statistics data schema",
-    "config_data": [
-      {
-        "item_name": "report_time",
-        "item_type": "string",
-        "item_optional": false,
-        "item_default": "1970-01-01T00:00:00Z",
-        "item_title": "Report time",
-        "item_description": "A date time when stats module reports",
-        "item_format": "date-time"
-      },
-      {
-        "item_name": "bind10.boot_time",
-        "item_type": "string",
-        "item_optional": false,
-        "item_default": "1970-01-01T00:00:00Z",
-        "item_title": "bind10.BootTime",
-        "item_description": "A date time when bind10 process starts initially",
-        "item_format": "date-time"
-      },
-      {
-        "item_name": "stats.boot_time",
-        "item_type": "string",
-        "item_optional": false,
-        "item_default": "1970-01-01T00:00:00Z",
-        "item_title": "stats.BootTime",
-        "item_description": "A date time when the stats module starts initially or when the stats module restarts",
-        "item_format": "date-time"
-      },
-      {
-        "item_name": "stats.start_time",
-        "item_type": "string",
-        "item_optional": false,
-        "item_default": "1970-01-01T00:00:00Z",
-        "item_title": "stats.StartTime",
-        "item_description": "A date time when the stats module starts collecting data or resetting values last time",
-        "item_format": "date-time"
-      },
-      {
-        "item_name": "stats.last_update_time",
-        "item_type": "string",
-        "item_optional": false,
-        "item_default": "1970-01-01T00:00:00Z",
-        "item_title": "stats.LastUpdateTime",
-        "item_description": "The latest date time when the stats module receives from other modules like auth server or boss process and so on",
-        "item_format": "date-time"
-      },
-      {
-        "item_name": "stats.timestamp",
-        "item_type": "real",
-        "item_optional": false,
-        "item_default": 0.0,
-        "item_title": "stats.Timestamp",
-        "item_description": "A current time stamp since epoch time (1970-01-01T00:00:00Z)"
-      },
-      {
-        "item_name": "stats.lname",
-        "item_type": "string",
-        "item_optional": false,
-        "item_default": "",
-        "item_title": "stats.LocalName",
-        "item_description": "A localname of stats module given via CC protocol"
-      },
-      {
-        "item_name": "auth.queries.tcp",
-        "item_type": "integer",
-        "item_optional": false,
-        "item_default": 0,
-        "item_title": "auth.queries.tcp",
-        "item_description": "A number of total query counts which all auth servers receive over TCP since they started initially"
-      },
-      {
-        "item_name": "auth.queries.udp",
-        "item_type": "integer",
-        "item_optional": false,
-        "item_default": 0,
-        "item_title": "auth.queries.udp",
-        "item_description": "A number of total query counts which all auth servers receive over UDP since they started initially"
-      }
-    ],
-    "commands": []
-  }
-}

src/bin/stats/stats.py.in

@@ -15,16 +15,17 @@
 # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
 # WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
+"""
+Statistics daemon in BIND 10
+
+"""
 import sys; sys.path.append ('@@PYTHONPATH@@')
 import os
-import signal
-import select
 from time import time, strftime, gmtime
 from optparse import OptionParser, OptionValueError
-from collections import defaultdict
-from isc.config.ccsession import ModuleCCSession, create_answer
-from isc.cc import Session, SessionError
 
+import isc
+import isc.util.process
 import isc.log
 from isc.log_messages.stats_messages import *
 
@@ -35,226 +36,157 @@ logger = isc.log.Logger("stats")
 # have #1074
 DBG_STATS_MESSAGING = 30
 
+# This is for boot_time of Stats
+_BASETIME = gmtime()
+
 # for setproctitle
-import isc.util.process
 isc.util.process.rename()
 
 # If B10_FROM_SOURCE is set in the environment, we use data files
 # from a directory relative to that, otherwise we use the ones
 # installed on the system
 if "B10_FROM_SOURCE" in os.environ:
-    BASE_LOCATION = os.environ["B10_FROM_SOURCE"] + os.sep + \
-        "src" + os.sep + "bin" + os.sep + "stats"
+    SPECFILE_LOCATION = os.environ["B10_FROM_SOURCE"] + os.sep + \
+        "src" + os.sep + "bin" + os.sep + "stats" + os.sep + "stats.spec"
 else:
     PREFIX = "@prefix@"
     DATAROOTDIR = "@datarootdir@"
-    BASE_LOCATION = "@datadir@" + os.sep + "@PACKAGE@"
-    BASE_LOCATION = BASE_LOCATION.replace("${datarootdir}", DATAROOTDIR).replace("${prefix}", PREFIX)
-SPECFILE_LOCATION = BASE_LOCATION + os.sep + "stats.spec"
-SCHEMA_SPECFILE_LOCATION = BASE_LOCATION + os.sep + "stats-schema.spec"
+    SPECFILE_LOCATION = "@datadir@" + os.sep + "@PACKAGE@" + os.sep + "stats.spec"
+    SPECFILE_LOCATION = SPECFILE_LOCATION.replace("${datarootdir}", DATAROOTDIR)\
+        .replace("${prefix}", PREFIX)
 
-class Singleton(type):
+def get_timestamp():
     """
-    A abstract class of singleton pattern
+    get current timestamp
     """
-    # Because of singleton pattern: 
-    #   At the beginning of coding, one UNIX domain socket is needed
-    #  for config manager, another socket is needed for stats module,
-    #  then stats module might need two sockets. So I adopted the
-    #  singleton pattern because I avoid creating multiple sockets in
-    #  one stats module. But in the initial version stats module
-    #  reports only via bindctl, so just one socket is needed. To use
-    #  the singleton pattern is not important now. :(
+    return time()
 
-    def __init__(self, *args, **kwargs):
-        type.__init__(self, *args, **kwargs)
-        self._instances = {}
+def get_datetime(gmt=None):
+    """
+    get current datetime
+    """
+    if not gmt: gmt = gmtime()
+    return strftime("%Y-%m-%dT%H:%M:%SZ", gmt)
 
-    def __call__(self, *args, **kwargs):
-        if args not in self._instances:
-            self._instances[args]={}
-        kw = tuple(kwargs.items())
-        if  kw not in self._instances[args]:
-            self._instances[args][kw] = type.__call__(self, *args, **kwargs)
-        return self._instances[args][kw]
+def get_spec_defaults(spec):
+    """
+    extracts the default values of the items from spec specified in
+    arg, and returns the dict-type variable which is a set of the item
+    names and the default values
+    """
+    if type(spec) is not list: return {}
+    def _get_spec_defaults(spec):
+        item_type = spec['item_type']
+        if item_type == "integer":
+            return int(spec.get('item_default', 0))
+        elif item_type == "real":
+            return float(spec.get('item_default', 0.0))
+        elif item_type == "boolean":
+            return bool(spec.get('item_default', False))
+        elif item_type == "string":
+            return str(spec.get('item_default', ""))
+        elif item_type == "list":
+            return spec.get(
+                    "item_default",
+                    [ _get_spec_defaults(spec["list_item_spec"]) ])
+        elif item_type == "map":
+            return spec.get(
+                    "item_default",
+                    dict([ (s["item_name"], _get_spec_defaults(s)) for s in spec["map_item_spec"] ]) )
+        else:
+            return spec.get("item_default", None)
+    return dict([ (s['item_name'], _get_spec_defaults(s)) for s in spec ])
 
 class Callback():
     """
     A Callback handler class
     """
-    def __init__(self, name=None, callback=None, args=(), kwargs={}):
-        self.name = name
-        self.callback = callback
+    def __init__(self, command=None, args=(), kwargs={}):
+        self.command = command
         self.args = args
         self.kwargs = kwargs
 
     def __call__(self, *args, **kwargs):
-        if not args:
-            args = self.args
-        if not kwargs:
-            kwargs = self.kwargs
-        if self.callback:
-            return self.callback(*args, **kwargs)
+        if not args: args = self.args
+        if not kwargs: kwargs = self.kwargs
+        if self.command: return self.command(*args, **kwargs)
 
-class Subject():
-    """
-    A abstract subject class of observer pattern
-    """
-    # Because of observer pattern:
-    #   In the initial release, I'm also sure that observer pattern
-    #  isn't definitely needed because the interface between gathering
-    #  and reporting statistics data is single.  However in the future
-    #  release, the interfaces may be multiple, that is, multiple
-    #  listeners may be needed. For example, one interface, which
-    #  stats module has, is for between ''config manager'' and stats
-    #  module, another interface is for between ''HTTP server'' and
-    #  stats module, and one more interface is for between ''SNMP
-    #  server'' and stats module. So by considering that stats module
-    #  needs multiple interfaces in the future release, I adopted the
-    #  observer pattern in stats module. But I don't have concrete
-    #  ideas in case of multiple listener currently.
-
-    def __init__(self):
-        self._listeners = []
-
-    def attach(self, listener):
-        if not listener in self._listeners:
-            self._listeners.append(listener)
-
-    def detach(self, listener):
-        try:
-            self._listeners.remove(listener)
-        except ValueError:
-            pass
+class StatsError(Exception):
+    """Exception class for Stats class"""
+    pass
 
-    def notify(self, event, modifier=None):
-        for listener in self._listeners:
-            if modifier != listener:
-                listener.update(event)
-
-class Listener():
+class Stats:
     """
-    A abstract listener class of observer pattern
+    Main class of stats module
     """
-    def __init__(self, subject):
-        self.subject = subject
-        self.subject.attach(self)
-        self.events = {}
-
-    def update(self, name):
-        if name in self.events:
-            callback = self.events[name]
-            return callback()
-
-    def add_event(self, event):
-        self.events[event.name]=event
-
-class SessionSubject(Subject, metaclass=Singleton):
-    """
-    A concrete subject class which creates CC session object
-    """
-    def __init__(self, session=None):
-        Subject.__init__(self)
-        self.session=session
-        self.running = False
-
-    def start(self):
-        self.running = True
-        self.notify('start')
-
-    def stop(self):
+    def __init__(self):
         self.running = False
-        self.notify('stop')
-
-    def check(self):
-        self.notify('check')
-
-class CCSessionListener(Listener):
-    """
-    A concrete listener class which creates SessionSubject object and
-    ModuleCCSession object
-    """
-    def __init__(self, subject):
-        Listener.__init__(self, subject)
-        self.session = subject.session
-        self.boot_time = get_datetime()
-
         # create ModuleCCSession object
-        self.cc_session = ModuleCCSession(SPECFILE_LOCATION,
-                                          self.config_handler,
-                                          self.command_handler,
-                                          self.session)
-
-        self.session = self.subject.session = self.cc_session._session
-
-        # initialize internal data
-        self.stats_spec = isc.config.module_spec_from_file(SCHEMA_SPECFILE_LOCATION).get_config_spec()
-        self.stats_data = self.initialize_data(self.stats_spec)
-
-        # add event handler invoked via SessionSubject object
-        self.add_event(Callback('start', self.start))
-        self.add_event(Callback('stop', self.stop))
-        self.add_event(Callback('check', self.check))
-        # don't add 'command_' suffix to the special commands in
-        # order to prevent executing internal command via bindctl
-
+        self.mccs = isc.config.ModuleCCSession(SPECFILE_LOCATION,
+                                               self.config_handler,
+                                               self.command_handler)
+        self.cc_session = self.mccs._session
+        # get module spec
+        self.module_name = self.mccs.get_module_spec().get_module_name()
+        self.modules = {}
+        self.statistics_data = {}
         # get commands spec
-        self.commands_spec = self.cc_session.get_module_spec().get_commands_spec()
-
+        self.commands_spec = self.mccs.get_module_spec().get_commands_spec()
         # add event handler related command_handler of ModuleCCSession
-        # invoked via bindctl
+        self.callbacks = {}
         for cmd in self.commands_spec:
+            # add prefix "command_"
+            name = "command_" + cmd["command_name"]
             try:
-                # add prefix "command_"
-                name = "command_" + cmd["command_name"]
                 callback = getattr(self, name)
-                kwargs = self.initialize_data(cmd["command_args"])
-                self.add_event(Callback(name=name, callback=callback, args=(), kwargs=kwargs))
-            except AttributeError as ae:
-                logger.error(STATS_UNKNOWN_COMMAND_IN_SPEC, cmd["command_name"])
-
-    def _update_stats_data(self, args):
-        # 'args' must be dictionary type
-        if isinstance(args, dict) and isinstance(args.get('stats_data'), dict):
-            self.stats_data.update(args['stats_data'])
-
-        # overwrite "stats.LastUpdateTime"
-        self.stats_data['stats.last_update_time'] = get_datetime()
+                kwargs = get_spec_defaults(cmd["command_args"])
+                self.callbacks[name] = Callback(command=callback, kwargs=kwargs)
+            except AttributeError:
+                raise StatsError(STATS_UNKNOWN_COMMAND_IN_SPEC, cmd["command_name"])
+        self.mccs.start()
 
     def start(self):
         """
-        start the cc chanel
+        Start stats module
         """
-        # set initial value
-        self.stats_data['stats.boot_time'] = self.boot_time
-        self.stats_data['stats.start_time'] = get_datetime()
-        self.stats_data['stats.last_update_time'] = get_datetime()
-        self.stats_data['stats.lname'] = self.session.lname
-        self.cc_session.start()
+        self.running = True
+        logger.info(STATS_STARTING)
+
         # request Bob to send statistics data
         logger.debug(DBG_STATS_MESSAGING, STATS_SEND_REQUEST_BOSS)
         cmd = isc.config.ccsession.create_command("getstats", None)
-        seq = self.session.group_sendmsg(cmd, 'Boss')
+        seq = self.cc_session.group_sendmsg(cmd, 'Boss')
         try:
-            answer, env = self.session.group_recvmsg(False, seq)
+            answer, env = self.cc_session.group_recvmsg(False, seq)
             if answer:
-                rcode, arg = isc.config.ccsession.parse_answer(answer)
+                rcode, args = isc.config.ccsession.parse_answer(answer)
                 if rcode == 0:
-                    self._update_stats_data(arg)
+                    errors = self.update_statistics_data(
+                        args["owner"], **args["data"])
+                    if errors:
+                        raise StatsError("boss spec file is incorrect: "
+                                         + ", ".join(errors))
+                    errors = self.update_statistics_data(
+                                self.module_name,
+                                last_update_time=get_datetime())
+                    if errors:
+                        raise StatsError("stats spec file is incorrect: "
+                                         + ", ".join(errors))
         except isc.cc.session.SessionTimeout:
             pass
 
-    def stop(self):
-        """
-        stop the cc chanel
-        """
-        return self.cc_session.close()
+        # initialized Statistics data
+        errors = self.update_statistics_data(
+            self.module_name,
+            lname=self.cc_session.lname,
+            boot_time=get_datetime(_BASETIME)
+            )
+        if errors:
+            raise StatsError("stats spec file is incorrect: "
+                             + ", ".join(errors))
 
-    def check(self):
-        """
-        check the cc chanel
-        """
-        return self.cc_session.check_command(False)
+        while self.running:
+            self.mccs.check_command(False)
 
     def config_handler(self, new_config):
         """
@@ -262,169 +194,222 @@ class CCSessionListener(Listener):
         """
         logger.debug(DBG_STATS_MESSAGING, STATS_RECEIVED_NEW_CONFIG,
                      new_config)
-
         # do nothing currently
-        return create_answer(0)
+        return isc.config.create_answer(0)
 
-    def command_handler(self, command, *args, **kwargs):
+    def command_handler(self, command, kwargs):
         """
         handle commands from the cc channel
         """
-        # add 'command_' suffix in order to executing command via bindctl
         name = 'command_' + command
-        
-        if name in self.events:
-            event = self.events[name]
-            return event(*args, **kwargs)
+        if name in self.callbacks:
+            callback = self.callbacks[name]
+            if kwargs:
+                return callback(**kwargs)
+            else:
+                return callback()
         else:
-            return self.command_unknown(command, args)
+            logger.error(STATS_RECEIVED_UNKNOWN_COMMAND, command)
+            return isc.config.create_answer(1, "Unknown command: '"+str(command)+"'")
 
-    def command_shutdown(self, args):
+    def update_modules(self):
         """
-        handle shutdown command
+        updates information of each module. This method gets each
+        module's information from the config manager and sets it into
+        self.modules. If its getting from the config manager fails, it
+        raises StatsError.
         """
-        logger.info(STATS_RECEIVED_SHUTDOWN_COMMAND)
-        self.subject.running = False
-        return create_answer(0)
+        modules = {}
+        seq = self.cc_session.group_sendmsg(
+            isc.config.ccsession.create_command(
+                isc.config.ccsession.COMMAND_GET_STATISTICS_SPEC),
+            'ConfigManager')
+        (answer, env) = self.cc_session.group_recvmsg(False, seq)
+        if answer:
+            (rcode, value) = isc.config.ccsession.parse_answer(answer)
+            if rcode == 0:
+                for mod in value:
+                    spec = { "module_name" : mod }
+                    if value[mod] and type(value[mod]) is list:
+                        spec["statistics"] = value[mod]
+                    modules[mod] = isc.config.module_spec.ModuleSpec(spec)
+            else:
+                raise StatsError("Updating module spec fails: " + str(value))
+        modules[self.module_name] = self.mccs.get_module_spec()
+        self.modules = modules
 
-    def command_set(self, args, stats_data={}):
+    def get_statistics_data(self, owner=None, name=None):
         """
-        handle set command
+        returns statistics data which stats module has of each
+        module. If it can't find specified statistics data, it raises
+        StatsError.
         """
-        self._update_stats_data(args)
-        return create_answer(0)
+        self.update_statistics_data()
+        if owner and name:
+            try:
+                return self.statistics_data[owner][name]
+            except KeyError:
+                pass
+        elif owner:
+            try:
+                return self.statistics_data[owner]
+            except KeyError:
+                pass
+        elif name:
+            pass
+        else:
+            return self.statistics_data
+        raise StatsError("No statistics data found: "
+                         + "owner: " + str(owner) + ", "
+                         + "name: " + str(name))
 
-    def command_remove(self, args, stats_item_name=''):
+    def update_statistics_data(self, owner=None, **data):
         """
-        handle remove command
+        change statistics date of specified module into specified
+        data. It updates information of each module first, and it
+        updates statistics data. If specified data is invalid for
+        statistics spec of specified owner, it returns a list of error
+        messeges. If there is no error or if neither owner nor data is
+        specified in args, it returns None.
         """
-
-        # 'args' must be dictionary type
-        if args and args['stats_item_name'] in self.stats_data:
-            stats_item_name = args['stats_item_name']
-
-        logger.debug(DBG_STATS_MESSAGING, STATS_RECEIVED_REMOVE_COMMAND,
-                     stats_item_name)
-
-        # just remove one item
-        self.stats_data.pop(stats_item_name)
-
-        return create_answer(0)
-
-    def command_show(self, args, stats_item_name=''):
+        self.update_modules()
+        statistics_data = {}
+        for (name, module) in self.modules.items():
+            value = get_spec_defaults(module.get_statistics_spec())
+            if module.validate_statistics(True, value):
+                statistics_data[name] = value
+        for (name, value) in self.statistics_data.items():
+            if name in statistics_data:
+                statistics_data[name].update(value)
+            else:
+                statistics_data[name] = value
+        self.statistics_data = statistics_data
+        if owner and data:
+            errors = []
+            try:
+                if self.modules[owner].validate_statistics(False, data, errors):
+                    self.statistics_data[owner].update(data)
+                    return
+            except KeyError:
+                errors.append("unknown module name: " + str(owner))
+            return errors
+
+    def command_status(self):
         """
-        handle show command
+        handle status command
         """
+        logger.debug(DBG_STATS_MESSAGING, STATS_RECEIVED_STATUS_COMMAND)
+        return isc.config.create_answer(
+            0, "Stats is up. (PID " + str(os.getpid()) + ")")
 
-        # always overwrite 'report_time' and 'stats.timestamp'
-        # if "show" command invoked
-        self.stats_data['report_time'] = get_datetime()
-        self.stats_data['stats.timestamp'] = get_timestamp()
-
-        # if with args
-        if args and args['stats_item_name'] in self.stats_data:
-            stats_item_name = args['stats_item_name']
-            logger.debug(DBG_STATS_MESSAGING,
-                         STATS_RECEIVED_SHOW_NAME_COMMAND,
-                         stats_item_name)
-            return create_answer(0, {stats_item_name: self.stats_data[stats_item_name]})
-
-        logger.debug(DBG_STATS_MESSAGING,
-                     STATS_RECEIVED_SHOW_ALL_COMMAND)
-        return create_answer(0, self.stats_data)
-
-    def command_reset(self, args):
+    def command_shutdown(self):
         """
-        handle reset command
+        handle shutdown command
         """
-        logger.debug(DBG_STATS_MESSAGING,
-                     STATS_RECEIVED_RESET_COMMAND)
-
-        # re-initialize internal variables
-        self.stats_data = self.initialize_data(self.stats_spec)
-
-        # reset initial value
-        self.stats_data['stats.boot_time'] = self.boot_time
-        self.stats_data['stats.start_time'] = get_datetime()
-        self.stats_data['stats.last_update_time'] = get_datetime()
-        self.stats_data['stats.lname'] = self.session.lname
-
-        return create_answer(0)
+        logger.info(STATS_RECEIVED_SHUTDOWN_COMMAND)
+        self.running = False
+        return isc.config.create_answer(0)
 
-    def command_status(self, args):
+    def command_show(self, owner=None, name=None):
         """
-        handle status command
+        handle show command
         """
-        logger.debug(DBG_STATS_MESSAGING, STATS_RECEIVED_STATUS_COMMAND)
-        # just return "I'm alive."
-        return create_answer(0, "I'm alive.")
-
-    def command_unknown(self, command, args):
+        if owner or name:
+            logger.debug(DBG_STATS_MESSAGING,
+                         STATS_RECEIVED_SHOW_NAME_COMMAND,
+                         str(owner)+", "+str(name))
+        else:
+            logger.debug(DBG_STATS_MESSAGING,
+                         STATS_RECEIVED_SHOW_ALL_COMMAND)
+        errors = self.update_statistics_data(
+            self.module_name,
+            timestamp=get_timestamp(),
+            report_time=get_datetime()
+            )
+        if errors:
+            raise StatsError("stats spec file is incorrect: "
+                             + ", ".join(errors))
+        try:
+            return isc.config.create_answer(
+                0, self.get_statistics_data(owner, name))
+        except StatsError:
+            return isc.config.create_answer(
+                1, "specified arguments are incorrect: " \
+                    + "owner: " + str(owner) + ", name: " + str(name))
+
+    def command_showschema(self, owner=None, name=None):
         """
-        handle an unknown command
+        handle show command
         """
-        logger.error(STATS_RECEIVED_UNKNOWN_COMMAND, command)
-        return create_answer(1, "Unknown command: '"+str(command)+"'")
-
+        if owner or name:
+            logger.debug(DBG_STATS_MESSAGING,
+                         STATS_RECEIVED_SHOWSCHEMA_NAME_COMMAND,
+                         str(owner)+", "+str(name))
+        else:
+            logger.debug(DBG_STATS_MESSAGING,
+                         STATS_RECEIVED_SHOWSCHEMA_ALL_COMMAND)
+        self.update_modules()
+        schema = {}
+        schema_byname = {}
+        for mod in self.modules:
+            spec = self.modules[mod].get_statistics_spec()
+            schema_byname[mod] = {}
+            if spec:
+                schema[mod] = spec
+                for item in spec:
+                    schema_byname[mod][item['item_name']] = item
+        if owner:
+            try:
+                if name:
+                    return isc.config.create_answer(0, schema_byname[owner][name])
+                else:
+                    return isc.config.create_answer(0, schema[owner])
+            except KeyError:
+                pass
+        else:
+            if name:
+                return isc.config.create_answer(1, "module name is not specified")
+            else:
+                return isc.config.create_answer(0, schema)
+        return isc.config.create_answer(
+                1, "specified arguments are incorrect: " \
+                    + "owner: " + str(owner) + ", name: " + str(name))
 
-    def initialize_data(self, spec):
+    def command_set(self, owner, data):
         """
-        initialize stats data
+        handle set command
         """
-        def __get_init_val(spec):
-            if spec['item_type'] == 'null':
-                return None
-            elif spec['item_type'] == 'boolean':
-                return bool(spec.get('item_default', False))
-            elif spec['item_type'] == 'string':
-                return str(spec.get('item_default', ''))
-            elif spec['item_type'] in set(['number', 'integer']):
-                return int(spec.get('item_default', 0))
-            elif spec['item_type'] in set(['float', 'double', 'real']):
-                return float(spec.get('item_default', 0.0))
-            elif spec['item_type'] in set(['list', 'array']):
-                return spec.get('item_default',
-                                [ __get_init_val(s) for s in spec['list_item_spec'] ])
-            elif spec['item_type'] in set(['map', 'object']):
-                return spec.get('item_default',
-                                dict([ (s['item_name'], __get_init_val(s)) for s in spec['map_item_spec'] ]) )
-            else:
-                return spec.get('item_default')
-        return dict([ (s['item_name'], __get_init_val(s)) for s in spec ])
+        errors = self.update_statistics_data(owner, **data)
+        if errors:
+            return isc.config.create_answer(
+                1, "errors while setting statistics data: " \
+                    + ", ".join(errors))
+        errors = self.update_statistics_data(
+            self.module_name, last_update_time=get_datetime() )
+        if errors:
+            raise StatsError("stats spec file is incorrect: "
+                             + ", ".join(errors))
+        return isc.config.create_answer(0)
 
-def get_timestamp():
-    """
-    get current timestamp
-    """
-    return time()
-
-def get_datetime():
-    """
-    get current datetime
-    """
-    return strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())
-
-def main(session=None):
+if __name__ == "__main__":
     try:
         parser = OptionParser()
-        parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
-                      help="display more about what is going on")
+        parser.add_option(
+            "-v", "--verbose", dest="verbose", action="store_true",
+            help="display more about what is going on")
         (options, args) = parser.parse_args()
         if options.verbose:
             isc.log.init("b10-stats", "DEBUG", 99)
-        subject = SessionSubject(session=session)
-        listener = CCSessionListener(subject)
-        subject.start()
-        while subject.running:
-            subject.check()
-        subject.stop()
-
+        stats = Stats()
+        stats.start()
     except OptionValueError as ove:
         logger.fatal(STATS_BAD_OPTION_VALUE, ove)
-    except SessionError as se:
+        sys.exit(1)
+    except isc.cc.session.SessionError as se:
         logger.fatal(STATS_CC_SESSION_ERROR, se)
+        sys.exit(1)
+    except StatsError as se:
+        logger.fatal(STATS_START_ERROR, se)
+        sys.exit(1)
     except KeyboardInterrupt as kie:
         logger.info(STATS_STOPPED_BY_KEYBOARD)
-
-if __name__ == "__main__":
-    main()

src/bin/stats/stats.spec

@@ -6,55 +6,74 @@
     "commands": [
       {
         "command_name": "status",
-        "command_description": "identify whether stats module is alive or not",
+        "command_description": "Show status of the stats daemon",
+        "command_args": []
+      },
+      {
+        "command_name": "shutdown",
+        "command_description": "Shut down the stats module",
         "command_args": []
       },
       {
         "command_name": "show",
-        "command_description": "show the specified/all statistics data",
+        "command_description": "Show the specified/all statistics data",
         "command_args": [
           {
-            "item_name": "stats_item_name",
+            "item_name": "owner",
+            "item_type": "string",
+            "item_optional": true,
+            "item_default": "",
+            "item_description": "module name of the owner of the statistics data"
+          },
+	  {
+	    "item_name": "name",
             "item_type": "string",
             "item_optional": true,
-            "item_default": ""
+            "item_default": "",
+            "item_description": "statistics item name of the owner"
           }
         ]
       },
       {
-        "command_name": "set",
-        "command_description": "set the value of specified name in statistics data",
+        "command_name": "showschema",
+        "command_description": "show the specified/all statistics shema",
         "command_args": [
           {
-            "item_name": "stats_data",
-            "item_type": "map",
-            "item_optional": false,
-            "item_default": {},
-            "map_item_spec": []
+            "item_name": "owner",
+            "item_type": "string",
+            "item_optional": true,
+            "item_default": "",
+            "item_description": "module name of the owner of the statistics data"
+          },
+	  {
+	    "item_name": "name",
+            "item_type": "string",
+            "item_optional": true,
+            "item_default": "",
+            "item_description": "statistics item name of the owner"
           }
         ]
       },
       {
-        "command_name": "remove",
-        "command_description": "remove the specified name from statistics data",
+        "command_name": "set",
+        "command_description": "set the value of specified name in statistics data",
         "command_args": [
           {
-            "item_name": "stats_item_name",
+            "item_name": "owner",
             "item_type": "string",
             "item_optional": false,
-            "item_default": ""
+            "item_default": "",
+            "item_description": "module name of the owner of the statistics data"
+          },
+	  {
+	    "item_name": "data",
+            "item_type": "map",
+            "item_optional": false,
+            "item_default": {},
+            "item_description": "statistics data set of the owner",
+            "map_item_spec": []
           }
         ]
-      },
-      {
-        "command_name": "reset",
-        "command_description": "reset all statistics data to default values except for several constant names",
-        "command_args": []
-      },
-      {
-        "command_name": "shutdown",
-        "command_description": "Shut down the stats module",
-        "command_args": []
       }
     ],
     "statistics": [
@@ -100,7 +119,7 @@
         "item_default": "",
         "item_title": "Local Name",
         "item_description": "A localname of stats module given via CC protocol"
-       }
+      }
     ]
   }
 }

src/bin/stats/stats_httpd.py.in

@@ -57,7 +57,6 @@ else:
     BASE_LOCATION = "@datadir@" + os.sep + "@PACKAGE@"
     BASE_LOCATION = BASE_LOCATION.replace("${datarootdir}", DATAROOTDIR).replace("${prefix}", PREFIX)
 SPECFILE_LOCATION = BASE_LOCATION + os.sep + "stats-httpd.spec"
-SCHEMA_SPECFILE_LOCATION = BASE_LOCATION + os.sep + "stats-schema.spec"
 XML_TEMPLATE_LOCATION = BASE_LOCATION + os.sep + "stats-httpd-xml.tpl"
 XSD_TEMPLATE_LOCATION = BASE_LOCATION + os.sep + "stats-httpd-xsd.tpl"
 XSL_TEMPLATE_LOCATION = BASE_LOCATION + os.sep + "stats-httpd-xsl.tpl"
@@ -69,7 +68,6 @@ XSD_URL_PATH = '/bind10/statistics/xsd'
 XSL_URL_PATH = '/bind10/statistics/xsl'
 # TODO: This should be considered later.
 XSD_NAMESPACE = 'http://bind10.isc.org' + XSD_URL_PATH
-DEFAULT_CONFIG = dict(listen_on=[('127.0.0.1', 8000)])
 
 # Assign this process name
 isc.util.process.rename()
@@ -160,8 +158,10 @@ class StatsHttpd:
         self.mccs = None
         self.httpd = []
         self.open_mccs()
+        self.config = {}
         self.load_config()
-        self.load_templates()
+        self.http_addrs = []
+        self.mccs.start()
         self.open_httpd()
 
     def open_mccs(self):
@@ -171,10 +171,6 @@ class StatsHttpd:
         self.mccs = isc.config.ModuleCCSession(
             SPECFILE_LOCATION, self.config_handler, self.command_handler)
         self.cc_session = self.mccs._session
-        # read spec file of stats module and subscribe 'Stats'
-        self.stats_module_spec = isc.config.module_spec_from_file(SCHEMA_SPECFILE_LOCATION)
-        self.stats_config_spec = self.stats_module_spec.get_config_spec()
-        self.stats_module_name = self.stats_module_spec.get_module_name()
 
     def close_mccs(self):
         """Closes a ModuleCCSession object"""
@@ -189,18 +185,19 @@ class StatsHttpd:
         """Loads configuration from spec file or new configuration
         from the config manager"""
         # load config
-        if len(new_config) > 0:
-            self.config.update(new_config)
-        else:
-            self.config = DEFAULT_CONFIG
-            self.config.update(
-                dict([
-                        (itm['item_name'], self.mccs.get_value(itm['item_name'])[0])
-                        for itm in self.mccs.get_module_spec().get_config_spec()
-                        ])
-                )
+        if len(self.config) == 0:
+            self.config = dict([
+                (itm['item_name'], self.mccs.get_value(itm['item_name'])[0])
+                for itm in self.mccs.get_module_spec().get_config_spec()
+                ])
+        self.config.update(new_config)
         # set addresses and ports for HTTP
-        self.http_addrs = [ (cf['address'], cf['port']) for cf in self.config['listen_on'] ]
+        addrs = []
+        if 'listen_on' in self.config:
+            for cf in self.config['listen_on']:
+                if 'address' in cf and 'port' in cf:
+                    addrs.append((cf['address'], cf['port']))
+        self.http_addrs = addrs
 
     def open_httpd(self):
         """Opens sockets for HTTP. Iterating each HTTP address to be
@@ -208,46 +205,44 @@ class StatsHttpd:
         for addr in self.http_addrs:
             self.httpd.append(self._open_httpd(addr))
 
-    def _open_httpd(self, server_address, address_family=None):
+    def _open_httpd(self, server_address):
+        httpd = None
         try:
-            # try IPv6 at first
-            if address_family is not None:
-                HttpServer.address_family = address_family
-            elif socket.has_ipv6:
-                HttpServer.address_family = socket.AF_INET6
+            # get address family for the server_address before
+            # creating HttpServer object. If a specified address is
+            # not numerical, gaierror may be thrown.
+            address_family = socket.getaddrinfo(
+                server_address[0], server_address[1], 0,
+                socket.SOCK_STREAM, socket.IPPROTO_TCP, socket.AI_NUMERICHOST
+                )[0][0]
+            HttpServer.address_family = address_family
             httpd = HttpServer(
                 server_address, HttpHandler,
                 self.xml_handler, self.xsd_handler, self.xsl_handler,
                 self.write_log)
-        except (socket.gaierror, socket.error,
-                OverflowError, TypeError) as err:
-            # try IPv4 next
-            if HttpServer.address_family == socket.AF_INET6:
-                httpd = self._open_httpd(server_address, socket.AF_INET)
-            else:
-                raise HttpServerError(
-                    "Invalid address %s, port %s: %s: %s" %
-                    (server_address[0], server_address[1],
-                     err.__class__.__name__, err))
-        else:
             logger.info(STATHTTPD_STARTED, server_address[0],
                         server_address[1])
-        return httpd
+            return httpd
+        except (socket.gaierror, socket.error,
+                OverflowError, TypeError) as err:
+           if httpd:
+                httpd.server_close()
+           raise HttpServerError(
+               "Invalid address %s, port %s: %s: %s" %
+               (server_address[0], server_address[1],
+                err.__class__.__name__, err))
 
     def close_httpd(self):
         """Closes sockets for HTTP"""
-        if len(self.httpd) == 0:
-            return
-        for ht in self.httpd:
+        while len(self.httpd)>0:
+            ht = self.httpd.pop()
             logger.info(STATHTTPD_CLOSING, ht.server_address[0],
                         ht.server_address[1])
             ht.server_close()
-        self.httpd = []
 
     def start(self):
         """Starts StatsHttpd objects to run. Waiting for client
         requests by using select.select functions"""
-        self.mccs.start()
         self.running = True
         while self.running:
             try:
@@ -280,6 +275,7 @@ class StatsHttpd:
         logger.info(STATHTTPD_SHUTDOWN)
         self.close_httpd()
         self.close_mccs()
+        self.running = False
 
     def get_sockets(self):
         """Returns sockets to select.select"""
@@ -296,23 +292,27 @@ class StatsHttpd:
         addresses and ports to listen HTTP requests on."""
         logger.debug(DBG_STATHTTPD_MESSAGING, STATHTTPD_HANDLE_CONFIG,
                    new_config)
-        for key in new_config.keys():
-            if key not in DEFAULT_CONFIG and key != "version":
-                logger.error(STATHTTPD_UNKNOWN_CONFIG_ITEM, key)
+        errors = []
+        if not self.mccs.get_module_spec().\
+                validate_config(False, new_config, errors):
                 return isc.config.ccsession.create_answer(
-                    1, "Unknown known config: %s" % key)
+                    1, ", ".join(errors))
         # backup old config
         old_config = self.config.copy()
-        self.close_httpd()
         self.load_config(new_config)
+        # If the http sockets aren't opened or
+        # if new_config doesn't have'listen_on', it returns
+        if len(self.httpd) == 0 or 'listen_on' not in new_config:
+            return isc.config.ccsession.create_answer(0)
+        self.close_httpd()
         try:
             self.open_httpd()
         except HttpServerError as err:
             logger.error(STATHTTPD_SERVER_ERROR, err)
             # restore old config
-            self.config_handler(old_config)
-            return isc.config.ccsession.create_answer(
-                1, "[b10-stats-httpd] %s" % err)
+            self.load_config(old_config)
+            self.open_httpd()
+            return isc.config.ccsession.create_answer(1, str(err))
         else:
             return isc.config.ccsession.create_answer(0)
 
@@ -328,8 +328,7 @@ class StatsHttpd:
             logger.debug(DBG_STATHTTPD_MESSAGING,
                          STATHTTPD_RECEIVED_SHUTDOWN_COMMAND)
             self.running = False
-            return isc.config.ccsession.create_answer(
-                0, "Stats Httpd is shutting down.")
+            return isc.config.ccsession.create_answer(0)
         else:
             logger.debug(DBG_STATHTTPD_MESSAGING,
                          STATHTTPD_RECEIVED_UNKNOWN_COMMAND, command)
@@ -341,8 +340,7 @@ class StatsHttpd:
         the data which obtains from it"""
         try:
             seq = self.cc_session.group_sendmsg(
-                isc.config.ccsession.create_command('show'),
-                self.stats_module_name)
+                isc.config.ccsession.create_command('show'), 'Stats')
             (answer, env) = self.cc_session.group_recvmsg(False, seq)
             if answer:
                 (rcode, value) = isc.config.ccsession.parse_answer(answer)
@@ -357,34 +355,82 @@ class StatsHttpd:
                 raise StatsHttpdError("Stats module: %s" % str(value))
 
     def get_stats_spec(self):
-        """Just returns spec data"""
-        return self.stats_config_spec
-
-    def load_templates(self):
-        """Setup the bodies of XSD and XSL documents to be responds to
-        HTTP clients. Before that it also creates XML tag structures by
-        using xml.etree.ElementTree.Element class and substitutes
-        concrete strings with parameters embed in the string.Template
-        object."""
+        """Requests statistics data to the Stats daemon and returns
+        the data which obtains from it"""
+        try:
+            seq = self.cc_session.group_sendmsg(
+                isc.config.ccsession.create_command('showschema'), 'Stats')
+            (answer, env) = self.cc_session.group_recvmsg(False, seq)
+            if answer:
+                (rcode, value) = isc.config.ccsession.parse_answer(answer)
+                if rcode == 0:
+                    return value
+                else:
+                    raise StatsHttpdError("Stats module: %s" % str(value))
+        except (isc.cc.session.SessionTimeout,
+                isc.cc.session.SessionError) as err:
+            raise StatsHttpdError("%s: %s" %
+                                  (err.__class__.__name__, err))
+
+    def xml_handler(self):
+        """Handler which requests to Stats daemon to obtain statistics
+        data and returns the body of XML document"""
+        xml_list=[]
+        for (mod, spec) in self.get_stats_data().items():
+            if not spec: continue
+            elem1 = xml.etree.ElementTree.Element(str(mod))
+            for (k, v) in spec.items():
+                elem2 = xml.etree.ElementTree.Element(str(k))
+                elem2.text = str(v)
+                elem1.append(elem2)
+            # The coding conversion is tricky. xml..tostring() of Python 3.2
+            # returns bytes (not string) regardless of the coding, while
+            # tostring() of Python 3.1 returns a string.  To support both
+            # cases transparently, we first make sure tostring() returns
+            # bytes by specifying utf-8 and then convert the result to a
+            # plain string (code below assume it).
+            xml_list.append(
+                str(xml.etree.ElementTree.tostring(elem1, encoding='utf-8'),
+                    encoding='us-ascii'))
+        xml_string = "".join(xml_list)
+        self.xml_body = self.open_template(XML_TEMPLATE_LOCATION).substitute(
+            xml_string=xml_string,
+            xsd_namespace=XSD_NAMESPACE,
+            xsd_url_path=XSD_URL_PATH,
+            xsl_url_path=XSL_URL_PATH)
+        assert self.xml_body is not None
+        return self.xml_body
+
+    def xsd_handler(self):
+        """Handler which just returns the body of XSD document"""
         # for XSD
         xsd_root = xml.etree.ElementTree.Element("all") # started with "all" tag
-        for item in self.get_stats_spec():
-            element = xml.etree.ElementTree.Element(
-                "element",
-                dict( name=item["item_name"],
-                      type=item["item_type"] if item["item_type"].lower() != 'real' else 'float',
-                      minOccurs="1",
-                      maxOccurs="1" ),
-                )
-            annotation = xml.etree.ElementTree.Element("annotation")
-            appinfo = xml.etree.ElementTree.Element("appinfo")
-            documentation = xml.etree.ElementTree.Element("documentation")
-            appinfo.text = item["item_title"]
-            documentation.text = item["item_description"]
-            annotation.append(appinfo)
-            annotation.append(documentation)
-            element.append(annotation)
-            xsd_root.append(element)
+        for (mod, spec) in self.get_stats_spec().items():
+            if not spec: continue
+            alltag = xml.etree.ElementTree.Element("all")
+            for item in spec:
+                element = xml.etree.ElementTree.Element(
+                    "element",
+                    dict( name=item["item_name"],
+                          type=item["item_type"] if item["item_type"].lower() != 'real' else 'float',
+                          minOccurs="1",
+                          maxOccurs="1" ),
+                    )
+                annotation = xml.etree.ElementTree.Element("annotation")
+                appinfo = xml.etree.ElementTree.Element("appinfo")
+                documentation = xml.etree.ElementTree.Element("documentation")
+                appinfo.text = item["item_title"]
+                documentation.text = item["item_description"]
+                annotation.append(appinfo)
+                annotation.append(documentation)
+                element.append(annotation)
+                alltag.append(element)
+
+            complextype = xml.etree.ElementTree.Element("complexType")
+            complextype.append(alltag)
+            mod_element = xml.etree.ElementTree.Element("element", { "name" : mod })
+            mod_element.append(complextype)
+            xsd_root.append(mod_element)
         # The coding conversion is tricky. xml..tostring() of Python 3.2
         # returns bytes (not string) regardless of the coding, while
         # tostring() of Python 3.1 returns a string.  To support both
@@ -398,25 +444,33 @@ class StatsHttpd:
             xsd_namespace=XSD_NAMESPACE
             )
         assert self.xsd_body is not None
+        return self.xsd_body
 
+    def xsl_handler(self):
+        """Handler which just returns the body of XSL document"""
         # for XSL
         xsd_root = xml.etree.ElementTree.Element(
             "xsl:template",
             dict(match="*")) # started with xml:template tag
-        for item in self.get_stats_spec():
-            tr = xml.etree.ElementTree.Element("tr")
-            td1 = xml.etree.ElementTree.Element(
-                "td", { "class" : "title",
-                        "title" : item["item_description"] })
-            td1.text = item["item_title"]
-            td2 = xml.etree.ElementTree.Element("td")
-            xsl_valueof = xml.etree.ElementTree.Element(
-                "xsl:value-of",
-                dict(select=item["item_name"]))
-            td2.append(xsl_valueof)
-            tr.append(td1)
-            tr.append(td2)
-            xsd_root.append(tr)
+        for (mod, spec) in self.get_stats_spec().items():
+            if not spec: continue
+            for item in spec:
+                tr = xml.etree.ElementTree.Element("tr")
+                td0 = xml.etree.ElementTree.Element("td")
+                td0.text = str(mod)
+                td1 = xml.etree.ElementTree.Element(
+                    "td", { "class" : "title",
+                            "title" : item["item_description"] })
+                td1.text = item["item_title"]
+                td2 = xml.etree.ElementTree.Element("td")
+                xsl_valueof = xml.etree.ElementTree.Element(
+                    "xsl:value-of",
+                    dict(select=mod+'/'+item["item_name"]))
+                td2.append(xsl_valueof)
+                tr.append(td0)
+                tr.append(td1)
+                tr.append(td2)
+                xsd_root.append(tr)
         # The coding conversion is tricky. xml..tostring() of Python 3.2
         # returns bytes (not string) regardless of the coding, while
         # tostring() of Python 3.1 returns a string.  To support both
@@ -429,47 +483,15 @@ class StatsHttpd:
             xsl_string=xsl_string,
             xsd_namespace=XSD_NAMESPACE)
         assert self.xsl_body is not None
-
-    def xml_handler(self):
-        """Handler which requests to Stats daemon to obtain statistics
-        data and returns the body of XML document"""
-        xml_list=[]
-        for (k, v) in self.get_stats_data().items():
-            (k, v) = (str(k), str(v))
-            elem = xml.etree.ElementTree.Element(k)
-            elem.text = v
-            # The coding conversion is tricky. xml..tostring() of Python 3.2
-            # returns bytes (not string) regardless of the coding, while
-            # tostring() of Python 3.1 returns a string.  To support both
-            # cases transparently, we first make sure tostring() returns
-            # bytes by specifying utf-8 and then convert the result to a
-            # plain string (code below assume it).
-            xml_list.append(
-                str(xml.etree.ElementTree.tostring(elem, encoding='utf-8'),
-                    encoding='us-ascii'))
-        xml_string = "".join(xml_list)
-        self.xml_body = self.open_template(XML_TEMPLATE_LOCATION).substitute(
-            xml_string=xml_string,
-            xsd_namespace=XSD_NAMESPACE,
-            xsd_url_path=XSD_URL_PATH,
-            xsl_url_path=XSL_URL_PATH)
-        assert self.xml_body is not None
-        return self.xml_body
-
-    def xsd_handler(self):
-        """Handler which just returns the body of XSD document"""
-        return self.xsd_body
-
-    def xsl_handler(self):
-        """Handler which just returns the body of XSL document"""
         return self.xsl_body
 
     def open_template(self, file_name):
         """It opens a template file, and it loads all lines to a
         string variable and returns string. Template object includes
         the variable. Limitation of a file size isn't needed there."""
-        lines = "".join(
-            open(file_name, 'r').readlines())
+        f = open(file_name, 'r')
+        lines = "".join(f.readlines())
+        f.close()
         assert lines is not None
         return string.Template(lines)
 
@@ -491,7 +513,7 @@ if __name__ == "__main__":
         logger.fatal(STATHTTPD_CC_SESSION_ERROR, se)
         sys.exit(1)
     except HttpServerError as hse:
-        logger.fatal(STATHTTPD_START_SERVER_ERROR, hse)
+        logger.fatal(STATHTTPD_START_SERVER_INIT_ERROR, hse)
         sys.exit(1)
     except KeyboardInterrupt as kie:
         logger.info(STATHTTPD_STOPPED_BY_KEYBOARD)

src/bin/stats/stats_messages.mes

@@ -28,16 +28,6 @@ control bus. A likely problem is that the message bus daemon
 This debug message is printed when the stats module has received a
 configuration update from the configuration manager.
 
-% STATS_RECEIVED_REMOVE_COMMAND received command to remove %1
-A remove command for the given name was sent to the stats module, and
-the given statistics value will now be removed. It will not appear in
-statistics reports until it appears in a statistics update from a
-module again.
-
-% STATS_RECEIVED_RESET_COMMAND received command to reset all statistics
-The stats module received a command to clear all collected statistics.
-The data is cleared until it receives an update from the modules again.
-
 % STATS_RECEIVED_SHOW_ALL_COMMAND received command to show all statistics
 The stats module received a command to show all statistics that it has
 collected.
@@ -72,4 +62,15 @@ installation problem, where the specification file stats.spec is
 from a different version of BIND 10 than the stats module itself.
 Please check your installation.
 
+% STATS_STARTING starting
+The stats module will be now starting.
+
+% STATS_RECEIVED_SHOWSCHEMA_ALL_COMMAND received command to show all statistics schema
+The stats module received a command to show all statistics schemas of all modules.
+
+% STATS_RECEIVED_SHOWSCHEMA_NAME_COMMAND received command to show statistics schema for %1
+The stats module received a command to show the specified statistics schema of the specified module.
 
+% STATS_START_ERROR stats module error: %1
+An internal error occurred while starting the stats module. The stats
+module will be now shutting down.

src/bin/stats/tests/Makefile.am

@@ -1,8 +1,7 @@
-SUBDIRS = isc http testdata
 PYCOVERAGE_RUN = @PYCOVERAGE_RUN@
 PYTESTS = b10-stats_test.py b10-stats-httpd_test.py
-EXTRA_DIST = $(PYTESTS) fake_time.py fake_socket.py fake_select.py
-CLEANFILES = fake_time.pyc fake_socket.pyc fake_select.pyc
+EXTRA_DIST = $(PYTESTS) test_utils.py
+CLEANFILES = test_utils.pyc
 
 # If necessary (rare cases), explicitly specify paths to dynamic libraries
 # required by loadable python modules.
@@ -14,15 +13,16 @@ endif
 # test using command-line arguments, so use check-local target instead of TESTS
 check-local:
 if ENABLE_PYTHON_COVERAGE
-	touch $(abs_top_srcdir)/.coverage 
+	touch $(abs_top_srcdir)/.coverage
 	rm -f .coverage
 	${LN_S} $(abs_top_srcdir)/.coverage .coverage
 endif
 	for pytest in $(PYTESTS) ; do \
 	echo Running test: $$pytest ; \
 	$(LIBRARY_PATH_PLACEHOLDER) \
-	PYTHONPATH=$(COMMON_PYTHON_PATH):$(abs_top_builddir)/src/bin/stats:$(abs_top_builddir)/src/bin/stats/tests \
+	PYTHONPATH=$(COMMON_PYTHON_PATH):$(abs_top_builddir)/src/bin/stats:$(abs_top_builddir)/src/bin/stats/tests:$(abs_top_builddir)/src/bin/msgq:$(abs_top_builddir)/src/lib/python/isc/config \
 	B10_FROM_SOURCE=$(abs_top_srcdir) \
+	CONFIG_TESTDATA_PATH=$(abs_top_srcdir)/src/lib/config/tests/testdata \
 	$(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \
 	done
 

src/bin/stats/tests/b10-stats-httpd_test.py

@@ -13,147 +13,269 @@
 # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
 # WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
+"""
+In each of these tests we start several virtual components. They are
+not the real components, no external processes are started. They are
+just simple mock objects running each in its own thread and pretending
+to be bind10 modules. This helps testing the stats http server in a
+close to real environment.
+"""
+
 import unittest
 import os
-import http.server
-import string
-import fake_select
 import imp
-import sys
-import fake_socket
-
-import isc.cc
+import socket
+import errno
+import select
+import string
+import time
+import threading
+import http.client
+import xml.etree.ElementTree
+import random
 
+import isc
 import stats_httpd
-stats_httpd.socket = fake_socket
-stats_httpd.select = fake_select
+import stats
+from test_utils import BaseModules, ThreadingServerManager, MyStats, MyStatsHttpd, SignalHandler, send_command, send_shutdown
 
 DUMMY_DATA = {
-    "auth.queries.tcp": 10000,
-    "auth.queries.udp": 12000,
-    "bind10.boot_time": "2011-03-04T11:59:05Z",
-    "report_time": "2011-03-04T11:59:19Z",
-    "stats.boot_time": "2011-03-04T11:59:06Z",
-    "stats.last_update_time": "2011-03-04T11:59:07Z",
-    "stats.lname": "4d70d40a_c@host",
-    "stats.start_time": "2011-03-04T11:59:06Z",
-    "stats.timestamp": 1299239959.560846
+    'Boss' : {
+        "boot_time": "2011-03-04T11:59:06Z"
+        },
+    'Auth' : {
+        "queries.tcp": 2,
+        "queries.udp": 3
+        },
+    'Stats' : {
+        "report_time": "2011-03-04T11:59:19Z",
+        "boot_time": "2011-03-04T11:59:06Z",
+        "last_update_time": "2011-03-04T11:59:07Z",
+        "lname": "4d70d40a_c@host",
+        "timestamp": 1299239959.560846
+        }
     }
 
-def push_answer(stats_httpd):
-    stats_httpd.cc_session.group_sendmsg(
-        { 'result': 
-          [ 0, DUMMY_DATA ] }, "Stats")
-
-def pull_query(stats_httpd):
-    (msg, env) = stats_httpd.cc_session.group_recvmsg()
-    if 'result' in msg:
-        (ret, arg) = isc.config.ccsession.parse_answer(msg)
-    else:
-        (ret, arg) = isc.config.ccsession.parse_command(msg)
-    return (ret, arg, env)
+def get_availaddr(address='127.0.0.1', port=8001):
+    """returns a tuple of address and port which is available to
+    listen on the platform. The first argument is a address for
+    search. The second argument is a port for search. If a set of
+    address and port is failed on the search for the availability, the
+    port number is increased and it goes on the next trial until the
+    available set of address and port is looked up. If the port number
+    reaches over 65535, it may stop the search and raise a
+    OverflowError exception."""
+    while True:
+        for addr in socket.getaddrinfo(
+            address, port, 0,
+            socket.SOCK_STREAM, socket.IPPROTO_TCP):
+            sock = socket.socket(addr[0], socket.SOCK_STREAM)
+            try:
+                sock.bind((address, port))
+                return (address, port)
+            except socket.error:
+                continue
+            finally:
+                if sock: sock.close()
+        # This address and port number are already in use.
+        # next port number is added
+        port = port + 1
+
+def is_ipv6_enabled(address='::1', port=8001):
+    """checks IPv6 enabled on the platform. address for check is '::1'
+    and port for check is random number between 8001 and
+    65535. Retrying is 3 times even if it fails. The built-in socket
+    module provides a 'has_ipv6' parameter, but it's not used here
+    because there may be a situation where the value is True on an
+    environment where the IPv6 config is disabled."""
+    for p in random.sample(range(port, 65535), 3):
+        try:
+            sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+            sock.bind((address, p))
+            return True
+        except socket.error:
+            continue
+        finally:
+            if sock: sock.close()
+    return False
 
 class TestHttpHandler(unittest.TestCase):
     """Tests for HttpHandler class"""
-
     def setUp(self):
-        self.stats_httpd = stats_httpd.StatsHttpd()
-        self.assertTrue(type(self.stats_httpd.httpd) is list)
-        self.httpd = self.stats_httpd.httpd
+        # set the signal handler for deadlock
+        self.sig_handler = SignalHandler(self.fail)
+        self.base = BaseModules()
+        self.stats_server = ThreadingServerManager(MyStats)
+        self.stats = self.stats_server.server
+        self.stats_server.run()
+        (self.address, self.port) = get_availaddr()
+        self.stats_httpd_server = ThreadingServerManager(MyStatsHttpd, (self.address, self.port))
+        self.stats_httpd = self.stats_httpd_server.server
+        self.stats_httpd_server.run()
+        self.client = http.client.HTTPConnection(self.address, self.port)
+        self.client._http_vsn_str = 'HTTP/1.0\n'
+        self.client.connect()
 
-    def test_do_GET(self):
-        for ht in self.httpd:
-            self._test_do_GET(ht._handler)
+    def tearDown(self):
+        self.client.close()
+        self.stats_httpd_server.shutdown()
+        self.stats_server.shutdown()
+        self.base.shutdown()
+        # reset the signal handler
+        self.sig_handler.reset()
 
-    def _test_do_GET(self, handler):
+    def test_do_GET(self):
+        self.assertTrue(type(self.stats_httpd.httpd) is list)
+        self.assertEqual(len(self.stats_httpd.httpd), 1)
+        self.assertEqual((self.address, self.port), self.stats_httpd.http_addrs[0])
 
         # URL is '/bind10/statistics/xml'
-        handler.path = stats_httpd.XML_URL_PATH
-        push_answer(self.stats_httpd)
-        handler.do_GET()
-        (ret, arg, env) = pull_query(self.stats_httpd)
-        self.assertEqual(ret, "show")
-        self.assertIsNone(arg)
-        self.assertTrue('group' in env)
-        self.assertEqual(env['group'], 'Stats')
-        self.assertEqual(handler.response.code, 200)
-        self.assertEqual(handler.response.headers["Content-type"], "text/xml")
-        self.assertTrue(handler.response.headers["Content-Length"] > 0)
-        self.assertTrue(handler.response.wrote_headers)
-        self.assertTrue(handler.response.body.find(stats_httpd.XSD_NAMESPACE)>0)
-        self.assertTrue(handler.response.body.find(stats_httpd.XSD_URL_PATH)>0)
-        for (k, v) in DUMMY_DATA.items():
-            self.assertTrue(handler.response.body.find(str(k))>0)
-            self.assertTrue(handler.response.body.find(str(v))>0)
+        self.client.putrequest('GET', stats_httpd.XML_URL_PATH)
+        self.client.endheaders()
+        response = self.client.getresponse()
+        self.assertEqual(response.getheader("Content-type"), "text/xml")
+        self.assertTrue(int(response.getheader("Content-Length")) > 0)
+        self.assertEqual(response.status, 200)
+        root = xml.etree.ElementTree.parse(response).getroot()
+        self.assertTrue(root.tag.find('stats_data') > 0)
+        for (k,v) in root.attrib.items():
+            if k.find('schemaLocation') > 0:
+                self.assertEqual(v, stats_httpd.XSD_NAMESPACE + ' ' + stats_httpd.XSD_URL_PATH)
+        for mod in DUMMY_DATA:
+            for (item, value) in DUMMY_DATA[mod].items():
+                self.assertIsNotNone(root.find(mod + '/' + item))
 
         # URL is '/bind10/statitics/xsd'
-        handler.path = stats_httpd.XSD_URL_PATH
-        handler.do_GET()
-        self.assertEqual(handler.response.code, 200)
-        self.assertEqual(handler.response.headers["Content-type"], "text/xml")
-        self.assertTrue(handler.response.headers["Content-Length"] > 0)
-        self.assertTrue(handler.response.wrote_headers)
-        self.assertTrue(handler.response.body.find(stats_httpd.XSD_NAMESPACE)>0)
-        for (k, v) in DUMMY_DATA.items():
-            self.assertTrue(handler.response.body.find(str(k))>0)
+        self.client.putrequest('GET', stats_httpd.XSD_URL_PATH)
+        self.client.endheaders()
+        response = self.client.getresponse()
+        self.assertEqual(response.getheader("Content-type"), "text/xml")
+        self.assertTrue(int(response.getheader("Content-Length")) > 0)
+        self.assertEqual(response.status, 200)
+        root = xml.etree.ElementTree.parse(response).getroot()
+        url_xmlschema = '{http://www.w3.org/2001/XMLSchema}'
+        tags = [ url_xmlschema + t for t in [ 'element', 'complexType', 'all', 'element' ] ]
+        xsdpath = '/'.join(tags)
+        self.assertTrue(root.tag.find('schema') > 0)
+        self.assertTrue(hasattr(root, 'attrib'))
+        self.assertTrue('targetNamespace' in root.attrib)
+        self.assertEqual(root.attrib['targetNamespace'],
+                         stats_httpd.XSD_NAMESPACE)
+        for elm in root.findall(xsdpath):
+            self.assertIsNotNone(elm.attrib['name'])
+            self.assertTrue(elm.attrib['name'] in DUMMY_DATA)
 
         # URL is '/bind10/statitics/xsl'
-        handler.path = stats_httpd.XSL_URL_PATH
-        handler.do_GET()
-        self.assertEqual(handler.response.code, 200)
-        self.assertEqual(handler.response.headers["Content-type"], "text/xml")
-        self.assertTrue(handler.response.headers["Content-Length"] > 0)
-        self.assertTrue(handler.response.wrote_headers)
-        self.assertTrue(handler.response.body.find(stats_httpd.XSD_NAMESPACE)>0)
-        for (k, v) in DUMMY_DATA.items():
-            self.assertTrue(handler.response.body.find(str(k))>0)
+        self.client.putrequest('GET', stats_httpd.XSL_URL_PATH)
+        self.client.endheaders()
+        response = self.client.getresponse()
+        self.assertEqual(response.getheader("Content-type"), "text/xml")
+        self.assertTrue(int(response.getheader("Content-Length")) > 0)
+        self.assertEqual(response.status, 200)
+        root = xml.etree.ElementTree.parse(response).getroot()
+        url_trans = '{http://www.w3.org/1999/XSL/Transform}'
+        url_xhtml = '{http://www.w3.org/1999/xhtml}'
+        xslpath = url_trans + 'template/' + url_xhtml + 'tr'
+        self.assertEqual(root.tag, url_trans + 'stylesheet')
+        for tr in root.findall(xslpath):
+            tds = tr.findall(url_xhtml + 'td')
+            self.assertIsNotNone(tds)
+            self.assertEqual(type(tds), list)
+            self.assertTrue(len(tds) > 2)
+            self.assertTrue(hasattr(tds[0], 'text'))
+            self.assertTrue(tds[0].text in DUMMY_DATA)
+            valueof = tds[2].find(url_trans + 'value-of')
+            self.assertIsNotNone(valueof)
+            self.assertTrue(hasattr(valueof, 'attrib'))
+            self.assertIsNotNone(valueof.attrib)
+            self.assertTrue('select' in valueof.attrib)
+            self.assertTrue(valueof.attrib['select'] in \
+                                [ tds[0].text+'/'+item for item in DUMMY_DATA[tds[0].text].keys() ])
 
         # 302 redirect
-        handler.path = '/'
-        handler.headers = {'Host': 'my.host.domain'}
-        handler.do_GET()
-        self.assertEqual(handler.response.code, 302)
-        self.assertEqual(handler.response.headers["Location"],
-                         "http://my.host.domain%s" % stats_httpd.XML_URL_PATH)
+        self.client._http_vsn_str = 'HTTP/1.1'
+        self.client.putrequest('GET', '/')
+        self.client.putheader('Host', self.address)
+        self.client.endheaders()
+        response = self.client.getresponse()
+        self.assertEqual(response.status, 302)
+        self.assertEqual(response.getheader('Location'),
+                         "http://%s:%d%s" % (self.address, self.port, stats_httpd.XML_URL_PATH))
 
         # 404 NotFound
-        handler.path = '/path/to/foo/bar'
-        handler.headers = {}
-        handler.do_GET()
-        self.assertEqual(handler.response.code, 404)
-
-        # failure case(connection with Stats is down)
-        handler.path = stats_httpd.XML_URL_PATH
-        push_answer(self.stats_httpd)
-        self.assertFalse(self.stats_httpd.cc_session._socket._closed)
-        self.stats_httpd.cc_session._socket._closed = True
-        handler.do_GET()
-        self.stats_httpd.cc_session._socket._closed = False
-        self.assertEqual(handler.response.code, 500)
-        self.stats_httpd.cc_session._clear_queues()
-
-        # failure case(Stats module returns err)
-        handler.path = stats_httpd.XML_URL_PATH
-        self.stats_httpd.cc_session.group_sendmsg(
-            { 'result': [ 1, "I have an error." ] }, "Stats")
-        self.assertFalse(self.stats_httpd.cc_session._socket._closed)
-        self.stats_httpd.cc_session._socket._closed = False
-        handler.do_GET()
-        self.assertEqual(handler.response.code, 500)
-        self.stats_httpd.cc_session._clear_queues()
+        self.client._http_vsn_str = 'HTTP/1.0'
+        self.client.putrequest('GET', '/path/to/foo/bar')
+        self.client.endheaders()
+        response = self.client.getresponse()
+        self.assertEqual(response.status, 404)
+
+
+    def test_do_GET_failed1(self):
+        # checks status
+        self.assertEqual(send_command("status", "Stats"),
+                         (0, "Stats is up. (PID " + str(os.getpid()) + ")"))
+        # failure case(Stats is down)
+        self.assertTrue(self.stats.running)
+        self.assertEqual(send_shutdown("Stats"), (0, None)) # Stats is down
+        self.assertFalse(self.stats.running)
+        self.stats_httpd.cc_session.set_timeout(milliseconds=100)
+
+        # request XML
+        self.client.putrequest('GET', stats_httpd.XML_URL_PATH)
+        self.client.endheaders()
+        response = self.client.getresponse()
+        self.assertEqual(response.status, 500)
+
+        # request XSD
+        self.client.putrequest('GET', stats_httpd.XSD_URL_PATH)
+        self.client.endheaders()
+        response = self.client.getresponse()
+        self.assertEqual(response.status, 500)
+
+        # request XSL
+        self.client.putrequest('GET', stats_httpd.XSL_URL_PATH)
+        self.client.endheaders()
+        response = self.client.getresponse()
+        self.assertEqual(response.status, 500)
+
+    def test_do_GET_failed2(self):
+        # failure case(Stats replies an error)
+        self.stats.mccs.set_command_handler(
+            lambda cmd, args: \
+                isc.config.ccsession.create_answer(1, "I have an error.")
+            )
+
+        # request XML
+        self.client.putrequest('GET', stats_httpd.XML_URL_PATH)
+        self.client.endheaders()
+        response = self.client.getresponse()
+        self.assertEqual(response.status, 500)
+
+        # request XSD
+        self.client.putrequest('GET', stats_httpd.XSD_URL_PATH)
+        self.client.endheaders()
+        response = self.client.getresponse()
+        self.assertEqual(response.status, 500)
+
+        # request XSL
+        self.client.putrequest('GET', stats_httpd.XSL_URL_PATH)
+        self.client.endheaders()
+        response = self.client.getresponse()
+        self.assertEqual(response.status, 500)
 
     def test_do_HEAD(self):
-        for ht in self.httpd:
-            self._test_do_HEAD(ht._handler)
+        self.client.putrequest('HEAD', stats_httpd.XML_URL_PATH)
+        self.client.endheaders()
+        response = self.client.getresponse()
+        self.assertEqual(response.status, 200)
 
-    def _test_do_HEAD(self, handler):
-        handler.path = '/path/to/foo/bar'
-        handler.do_HEAD()
-        self.assertEqual(handler.response.code, 404)
+        self.client.putrequest('HEAD', '/path/to/foo/bar')
+        self.client.endheaders()
+        response = self.client.getresponse()
+        self.assertEqual(response.status, 404)
 
 class TestHttpServerError(unittest.TestCase):
     """Tests for HttpServerError exception"""
-
     def test_raises(self):
         try:
             raise stats_httpd.HttpServerError('Nothing')
@@ -162,17 +284,24 @@ class TestHttpServerError(unittest.TestCase):
 
 class TestHttpServer(unittest.TestCase):
     """Tests for HttpServer class"""
+    def setUp(self):
+        # set the signal handler for deadlock
+        self.sig_handler = SignalHandler(self.fail)
+        self.base = BaseModules()
+
+    def tearDown(self):
+        if hasattr(self, "stats_httpd"):
+            self.stats_httpd.stop()
+        self.base.shutdown()
+        # reset the signal handler
+        self.sig_handler.reset()
 
     def test_httpserver(self):
-        self.stats_httpd = stats_httpd.StatsHttpd()
-        for ht in self.stats_httpd.httpd:
-            self.assertTrue(ht.server_address in self.stats_httpd.http_addrs)
-            self.assertEqual(ht.xml_handler, self.stats_httpd.xml_handler)
-            self.assertEqual(ht.xsd_handler, self.stats_httpd.xsd_handler)
-            self.assertEqual(ht.xsl_handler, self.stats_httpd.xsl_handler)
-            self.assertEqual(ht.log_writer, self.stats_httpd.write_log)
-            self.assertTrue(isinstance(ht._handler, stats_httpd.HttpHandler))
-            self.assertTrue(isinstance(ht.socket, fake_socket.socket))
+        self.stats_httpd = MyStatsHttpd(get_availaddr())
+        self.assertEqual(type(self.stats_httpd.httpd), list)
+        self.assertEqual(len(self.stats_httpd.httpd), 1)
+        for httpd in self.stats_httpd.httpd:
+            self.assertTrue(isinstance(httpd, stats_httpd.HttpServer))
 
 class TestStatsHttpdError(unittest.TestCase):
     """Tests for StatsHttpdError exception"""
@@ -187,132 +316,173 @@ class TestStatsHttpd(unittest.TestCase):
     """Tests for StatsHttpd class"""
 
     def setUp(self):
-        fake_socket._CLOSED = False
-        fake_socket.has_ipv6 = True
-        self.stats_httpd = stats_httpd.StatsHttpd()
+        # set the signal handler for deadlock
+        self.sig_handler = SignalHandler(self.fail)
+        self.base = BaseModules()
+        self.stats_server = ThreadingServerManager(MyStats)
+        self.stats_server.run()
+        # checking IPv6 enabled on this platform
+        self.ipv6_enabled = is_ipv6_enabled()
 
     def tearDown(self):
-        self.stats_httpd.stop()
+        if hasattr(self, "stats_httpd"):
+            self.stats_httpd.stop()
+        self.stats_server.shutdown()
+        self.base.shutdown()
+        # reset the signal handler
+        self.sig_handler.reset()
 
     def test_init(self):
-        self.assertFalse(self.stats_httpd.mccs.get_socket()._closed)
-        self.assertEqual(self.stats_httpd.mccs.get_socket().fileno(),
-                         id(self.stats_httpd.mccs.get_socket()))
-        for ht in self.stats_httpd.httpd:
-            self.assertFalse(ht.socket._closed)
-            self.assertEqual(ht.socket.fileno(), id(ht.socket))
-        fake_socket._CLOSED = True
-        self.assertRaises(isc.cc.session.SessionError,
-                          stats_httpd.StatsHttpd)
-        fake_socket._CLOSED = False
+        server_address = get_availaddr()
+        self.stats_httpd = MyStatsHttpd(server_address)
+        self.assertEqual(self.stats_httpd.running, False)
+        self.assertEqual(self.stats_httpd.poll_intval, 0.5)
+        self.assertNotEqual(len(self.stats_httpd.httpd), 0)
+        self.assertEqual(type(self.stats_httpd.mccs), isc.config.ModuleCCSession)
+        self.assertEqual(type(self.stats_httpd.cc_session), isc.cc.Session)
+        self.assertEqual(len(self.stats_httpd.config), 2)
+        self.assertTrue('listen_on' in self.stats_httpd.config)
+        self.assertEqual(len(self.stats_httpd.config['listen_on']), 1)
+        self.assertTrue('address' in self.stats_httpd.config['listen_on'][0])
+        self.assertTrue('port' in self.stats_httpd.config['listen_on'][0])
+        self.assertTrue(server_address in set(self.stats_httpd.http_addrs))
+
+    def test_openclose_mccs(self):
+        self.stats_httpd = MyStatsHttpd(get_availaddr())
+        self.stats_httpd.close_mccs()
+        self.assertEqual(self.stats_httpd.mccs, None)
+        self.stats_httpd.open_mccs()
+        self.assertIsNotNone(self.stats_httpd.mccs)
+        self.stats_httpd.mccs = None
+        self.assertEqual(self.stats_httpd.mccs, None)
+        self.assertEqual(self.stats_httpd.close_mccs(), None)
 
     def test_mccs(self):
-        self.stats_httpd.open_mccs()
+        self.stats_httpd = MyStatsHttpd(get_availaddr())
+        self.assertIsNotNone(self.stats_httpd.mccs.get_socket())
         self.assertTrue(
-            isinstance(self.stats_httpd.mccs.get_socket(), fake_socket.socket))
+            isinstance(self.stats_httpd.mccs.get_socket(), socket.socket))
         self.assertTrue(
             isinstance(self.stats_httpd.cc_session, isc.cc.session.Session))
-        self.assertTrue(
-            isinstance(self.stats_httpd.stats_module_spec, isc.config.ModuleSpec))
-        for cfg in self.stats_httpd.stats_config_spec:
-            self.assertTrue('item_name' in cfg)
-            self.assertTrue(cfg['item_name'] in DUMMY_DATA)
-        self.assertTrue(len(self.stats_httpd.stats_config_spec), len(DUMMY_DATA))
-
-    def test_load_config(self):
-        self.stats_httpd.load_config()
-        self.assertTrue(('127.0.0.1', 8000) in set(self.stats_httpd.http_addrs))
+        statistics_spec = self.stats_httpd.get_stats_spec()
+        for mod in DUMMY_DATA:
+            self.assertTrue(mod in statistics_spec)
+            for cfg in statistics_spec[mod]:
+                self.assertTrue('item_name' in cfg)
+                self.assertTrue(cfg['item_name'] in DUMMY_DATA[mod])
+            self.assertTrue(len(statistics_spec[mod]), len(DUMMY_DATA[mod]))
+        self.stats_httpd.close_mccs()
+        self.assertIsNone(self.stats_httpd.mccs)
 
     def test_httpd(self):
         # dual stack (addresses is ipv4 and ipv6)
-        fake_socket.has_ipv6 = True
-        self.assertTrue(('127.0.0.1', 8000) in set(self.stats_httpd.http_addrs))
-        self.stats_httpd.http_addrs = [ ('::1', 8000), ('127.0.0.1', 8000) ]
-        self.assertTrue(
-            stats_httpd.HttpServer.address_family in set([fake_socket.AF_INET, fake_socket.AF_INET6]))
-        self.stats_httpd.open_httpd()
-        for ht in self.stats_httpd.httpd:
-            self.assertTrue(isinstance(ht.socket, fake_socket.socket))
-        self.stats_httpd.close_httpd()
+        if self.ipv6_enabled:
+            server_addresses = (get_availaddr('::1'), get_availaddr())
+            self.stats_httpd = MyStatsHttpd(*server_addresses)
+            for ht in self.stats_httpd.httpd:
+                self.assertTrue(isinstance(ht, stats_httpd.HttpServer))
+                self.assertTrue(ht.address_family in set([socket.AF_INET, socket.AF_INET6]))
+                self.assertTrue(isinstance(ht.socket, socket.socket))
 
         # dual stack (address is ipv6)
-        fake_socket.has_ipv6 = True
-        self.stats_httpd.http_addrs = [ ('::1', 8000) ]
-        self.stats_httpd.open_httpd()
+        if self.ipv6_enabled:
+            server_addresses = get_availaddr('::1')
+            self.stats_httpd = MyStatsHttpd(server_addresses)
+            for ht in self.stats_httpd.httpd:
+                self.assertTrue(isinstance(ht, stats_httpd.HttpServer))
+                self.assertEqual(ht.address_family, socket.AF_INET6)
+                self.assertTrue(isinstance(ht.socket, socket.socket))
+
+        # dual/single stack (address is ipv4)
+        server_addresses = get_availaddr()
+        self.stats_httpd = MyStatsHttpd(server_addresses)
         for ht in self.stats_httpd.httpd:
-            self.assertTrue(isinstance(ht.socket, fake_socket.socket))
-        self.stats_httpd.close_httpd()
+            self.assertTrue(isinstance(ht, stats_httpd.HttpServer))
+            self.assertEqual(ht.address_family, socket.AF_INET)
+            self.assertTrue(isinstance(ht.socket, socket.socket))
 
-        # dual stack (address is ipv4)
-        fake_socket.has_ipv6 = True
-        self.stats_httpd.http_addrs = [ ('127.0.0.1', 8000) ]
-        self.stats_httpd.open_httpd()
+        # any address (IPv4)
+        server_addresses = get_availaddr(address='0.0.0.0')
+        self.stats_httpd = MyStatsHttpd(server_addresses)
         for ht in self.stats_httpd.httpd:
-            self.assertTrue(isinstance(ht.socket, fake_socket.socket))
-        self.stats_httpd.close_httpd()
-
-        # only-ipv4 single stack
-        fake_socket.has_ipv6 = False
-        self.stats_httpd.http_addrs = [ ('127.0.0.1', 8000) ]
-        self.stats_httpd.open_httpd()
-        for ht in self.stats_httpd.httpd:
-            self.assertTrue(isinstance(ht.socket, fake_socket.socket))
-        self.stats_httpd.close_httpd()
-
-        # only-ipv4 single stack (force set ipv6 )
-        fake_socket.has_ipv6 = False
-        self.stats_httpd.http_addrs = [ ('::1', 8000) ]
-        self.assertRaises(stats_httpd.HttpServerError,
-            self.stats_httpd.open_httpd)
-
-        # hostname
-        self.stats_httpd.http_addrs = [ ('localhost', 8000) ]
-        self.stats_httpd.open_httpd()
-        for ht in self.stats_httpd.httpd:
-            self.assertTrue(isinstance(ht.socket, fake_socket.socket))
-        self.stats_httpd.close_httpd()
-
-        self.stats_httpd.http_addrs = [ ('my.host.domain', 8000) ]
-        self.stats_httpd.open_httpd()
-        for ht in self.stats_httpd.httpd:
-            self.assertTrue(isinstance(ht.socket, fake_socket.socket))
-        self.stats_httpd.close_httpd()
+            self.assertTrue(isinstance(ht, stats_httpd.HttpServer))
+            self.assertEqual(ht.address_family,socket.AF_INET)
+            self.assertTrue(isinstance(ht.socket, socket.socket))
+
+        # any address (IPv6)
+        if self.ipv6_enabled:
+            server_addresses = get_availaddr(address='::')
+            self.stats_httpd = MyStatsHttpd(server_addresses)
+            for ht in self.stats_httpd.httpd:
+                self.assertTrue(isinstance(ht, stats_httpd.HttpServer))
+                self.assertEqual(ht.address_family,socket.AF_INET6)
+                self.assertTrue(isinstance(ht.socket, socket.socket))
+
+        # existent hostname
+        self.assertRaises(stats_httpd.HttpServerError, MyStatsHttpd,
+                          get_availaddr(address='localhost'))
+
+        # nonexistent hostname
+        self.assertRaises(stats_httpd.HttpServerError, MyStatsHttpd, ('my.host.domain', 8000))
 
         # over flow of port number
-        self.stats_httpd.http_addrs = [ ('', 80000) ]
-        self.assertRaises(stats_httpd.HttpServerError, self.stats_httpd.open_httpd)
+        self.assertRaises(stats_httpd.HttpServerError, MyStatsHttpd, ('127.0.0.1', 80000))
+
         # negative
-        self.stats_httpd.http_addrs = [ ('', -8000) ]
-        self.assertRaises(stats_httpd.HttpServerError, self.stats_httpd.open_httpd)
-        # alphabet
-        self.stats_httpd.http_addrs = [ ('', 'ABCDE') ]
-        self.assertRaises(stats_httpd.HttpServerError, self.stats_httpd.open_httpd)
-
-    def test_start(self):
-        self.stats_httpd.cc_session.group_sendmsg(
-            { 'command': [ "shutdown" ] }, "StatsHttpd")
-        self.stats_httpd.start()
-        self.stats_httpd = stats_httpd.StatsHttpd()
-        self.assertRaises(
-            fake_select.error, self.stats_httpd.start)
+        self.assertRaises(stats_httpd.HttpServerError, MyStatsHttpd, ('127.0.0.1', -8000))
 
-    def test_stop(self):
-        # success case
-        fake_socket._CLOSED = False
-        self.stats_httpd.stop()
+        # alphabet
+        self.assertRaises(stats_httpd.HttpServerError, MyStatsHttpd, ('127.0.0.1', 'ABCDE'))
+
+        # Address already in use
+        server_addresses = get_availaddr()
+        self.stats_httpd_server = ThreadingServerManager(MyStatsHttpd, server_addresses)
+        self.stats_httpd_server.run()
+        self.assertRaises(stats_httpd.HttpServerError, MyStatsHttpd, server_addresses)
+        send_shutdown("StatsHttpd")
+
+    def test_running(self):
+        self.stats_httpd_server = ThreadingServerManager(MyStatsHttpd, get_availaddr())
+        self.stats_httpd = self.stats_httpd_server.server
         self.assertFalse(self.stats_httpd.running)
-        self.assertIsNone(self.stats_httpd.mccs)
-        for ht in self.stats_httpd.httpd:
-            self.assertTrue(ht.socket._closed)
-        self.assertTrue(self.stats_httpd.cc_session._socket._closed)
+        self.stats_httpd_server.run()
+        self.assertEqual(send_command("status", "StatsHttpd"),
+                         (0, "Stats Httpd is up. (PID " + str(os.getpid()) + ")"))
+        self.assertTrue(self.stats_httpd.running)
+        self.assertEqual(send_shutdown("StatsHttpd"), (0, None))
+        self.assertFalse(self.stats_httpd.running)
+        self.stats_httpd_server.shutdown()
+
         # failure case
-        self.stats_httpd.cc_session._socket._closed = False
-        self.stats_httpd.open_mccs()
-        self.stats_httpd.cc_session._socket._closed = True
-        self.stats_httpd.stop() # No excetion raises
-        self.stats_httpd.cc_session._socket._closed = False
+        self.stats_httpd = MyStatsHttpd(get_availaddr())
+        self.stats_httpd.cc_session.close()
+        self.assertRaises(ValueError, self.stats_httpd.start)
+
+    def test_failure_with_a_select_error (self):
+        """checks select.error is raised if the exception except
+        errno.EINTR is raised while it's selecting"""
+        def raise_select_except(*args):
+            raise select.error('dummy error')
+        orig_select = stats_httpd.select.select
+        stats_httpd.select.select = raise_select_except
+        self.stats_httpd = MyStatsHttpd(get_availaddr())
+        self.assertRaises(select.error, self.stats_httpd.start)
+        stats_httpd.select.select = orig_select
+
+    def test_nofailure_with_errno_EINTR(self):
+        """checks no exception is raised if errno.EINTR is raised
+        while it's selecting"""
+        def raise_select_except(*args):
+            raise select.error(errno.EINTR)
+        orig_select = stats_httpd.select.select
+        stats_httpd.select.select = raise_select_except
+        self.stats_httpd_server = ThreadingServerManager(MyStatsHttpd, get_availaddr())
+        self.stats_httpd_server.run()
+        self.stats_httpd_server.shutdown()
+        stats_httpd.select.select = orig_select
 
     def test_open_template(self):
+        self.stats_httpd = MyStatsHttpd(get_availaddr())
         # successful conditions
         tmpl = self.stats_httpd.open_template(stats_httpd.XML_TEMPLATE_LOCATION)
         self.assertTrue(isinstance(tmpl, string.Template))
@@ -346,13 +516,13 @@ class TestStatsHttpd(unittest.TestCase):
             self.stats_httpd.open_template, '/path/to/foo/bar')
 
     def test_commands(self):
+        self.stats_httpd = MyStatsHttpd(get_availaddr())
         self.assertEqual(self.stats_httpd.command_handler("status", None),
                          isc.config.ccsession.create_answer(
                 0, "Stats Httpd is up. (PID " + str(os.getpid()) + ")"))
         self.stats_httpd.running = True
         self.assertEqual(self.stats_httpd.command_handler("shutdown", None),
-                         isc.config.ccsession.create_answer(
-                0, "Stats Httpd is shutting down."))
+                         isc.config.ccsession.create_answer(0))
         self.assertFalse(self.stats_httpd.running)
         self.assertEqual(
             self.stats_httpd.command_handler("__UNKNOWN_COMMAND__", None),
@@ -360,42 +530,48 @@ class TestStatsHttpd(unittest.TestCase):
                 1, "Unknown command: __UNKNOWN_COMMAND__"))
 
     def test_config(self):
+        self.stats_httpd = MyStatsHttpd(get_availaddr())
         self.assertEqual(
             self.stats_httpd.config_handler(dict(_UNKNOWN_KEY_=None)),
             isc.config.ccsession.create_answer(
-                    1, "Unknown known config: _UNKNOWN_KEY_"))
-        self.assertEqual(
-            self.stats_httpd.config_handler(
-                        dict(listen_on=[dict(address="::2",port=8000)])),
-            isc.config.ccsession.create_answer(0))
-        self.assertTrue("listen_on" in self.stats_httpd.config)
-        for addr in self.stats_httpd.config["listen_on"]:
-            self.assertTrue("address" in addr)
-            self.assertTrue("port" in addr)
-            self.assertTrue(addr["address"] == "::2")
-            self.assertTrue(addr["port"] == 8000)
+                1, "unknown item _UNKNOWN_KEY_"))
 
+        addresses = get_availaddr()
         self.assertEqual(
             self.stats_httpd.config_handler(
-                        dict(listen_on=[dict(address="::1",port=80)])),
+                dict(listen_on=[dict(address=addresses[0],port=addresses[1])])),
             isc.config.ccsession.create_answer(0))
         self.assertTrue("listen_on" in self.stats_httpd.config)
         for addr in self.stats_httpd.config["listen_on"]:
             self.assertTrue("address" in addr)
             self.assertTrue("port" in addr)
-            self.assertTrue(addr["address"] == "::1")
-            self.assertTrue(addr["port"] == 80)
-
+            self.assertTrue(addr["address"] == addresses[0])
+            self.assertTrue(addr["port"] == addresses[1])
+
+        if self.ipv6_enabled:
+            addresses = get_availaddr("::1")
+            self.assertEqual(
+                self.stats_httpd.config_handler(
+                dict(listen_on=[dict(address=addresses[0],port=addresses[1])])),
+                isc.config.ccsession.create_answer(0))
+            self.assertTrue("listen_on" in self.stats_httpd.config)
+            for addr in self.stats_httpd.config["listen_on"]:
+                self.assertTrue("address" in addr)
+                self.assertTrue("port" in addr)
+                self.assertTrue(addr["address"] == addresses[0])
+                self.assertTrue(addr["port"] == addresses[1])
+
+        addresses = get_availaddr()
         self.assertEqual(
             self.stats_httpd.config_handler(
-                        dict(listen_on=[dict(address="1.2.3.4",port=54321)])),
+                dict(listen_on=[dict(address=addresses[0],port=addresses[1])])),
             isc.config.ccsession.create_answer(0))
         self.assertTrue("listen_on" in self.stats_httpd.config)
         for addr in self.stats_httpd.config["listen_on"]:
             self.assertTrue("address" in addr)
             self.assertTrue("port" in addr)
-            self.assertTrue(addr["address"] == "1.2.3.4")
-            self.assertTrue(addr["port"] == 54321)
+            self.assertTrue(addr["address"] == addresses[0])
+            self.assertTrue(addr["port"] == addresses[1])
         (ret, arg) = isc.config.ccsession.parse_answer(
             self.stats_httpd.config_handler(
                 dict(listen_on=[dict(address="1.2.3.4",port=543210)]))
@@ -403,93 +579,103 @@ class TestStatsHttpd(unittest.TestCase):
         self.assertEqual(ret, 1)
 
     def test_xml_handler(self):
-        orig_get_stats_data = stats_httpd.StatsHttpd.get_stats_data
-        stats_httpd.StatsHttpd.get_stats_data = lambda x: {'foo':'bar'}
-        xml_body1 = stats_httpd.StatsHttpd().open_template(
+        self.stats_httpd = MyStatsHttpd(get_availaddr())
+        self.stats_httpd.get_stats_data = lambda: \
+            { 'Dummy' : { 'foo':'bar' } }
+        xml_body1 = self.stats_httpd.open_template(
             stats_httpd.XML_TEMPLATE_LOCATION).substitute(
-            xml_string='<foo>bar</foo>',
+            xml_string='<Dummy><foo>bar</foo></Dummy>',
             xsd_namespace=stats_httpd.XSD_NAMESPACE,
             xsd_url_path=stats_httpd.XSD_URL_PATH,
             xsl_url_path=stats_httpd.XSL_URL_PATH)
-        xml_body2 = stats_httpd.StatsHttpd().xml_handler()
+        xml_body2 = self.stats_httpd.xml_handler()
         self.assertEqual(type(xml_body1), str)
         self.assertEqual(type(xml_body2), str)
         self.assertEqual(xml_body1, xml_body2)
-        stats_httpd.StatsHttpd.get_stats_data = lambda x: {'bar':'foo'}
-        xml_body2 = stats_httpd.StatsHttpd().xml_handler()
+        self.stats_httpd.get_stats_data = lambda: \
+            { 'Dummy' : {'bar':'foo'} }
+        xml_body2 = self.stats_httpd.xml_handler()
         self.assertNotEqual(xml_body1, xml_body2)
-        stats_httpd.StatsHttpd.get_stats_data = orig_get_stats_data
 
     def test_xsd_handler(self):
-        orig_get_stats_spec = stats_httpd.StatsHttpd.get_stats_spec
-        stats_httpd.StatsHttpd.get_stats_spec = lambda x: \
-            [{
-                "item_name": "foo",
-                "item_type": "string",
-                "item_optional": False,
-                "item_default": "bar",
-                "item_description": "foo is bar",
-                "item_title": "Foo"
-               }]
-        xsd_body1 = stats_httpd.StatsHttpd().open_template(
+        self.stats_httpd = MyStatsHttpd(get_availaddr())
+        self.stats_httpd.get_stats_spec = lambda: \
+            { "Dummy" :
+                  [{
+                        "item_name": "foo",
+                        "item_type": "string",
+                        "item_optional": False,
+                        "item_default": "bar",
+                        "item_description": "foo is bar",
+                        "item_title": "Foo"
+                        }]
+              }
+        xsd_body1 = self.stats_httpd.open_template(
             stats_httpd.XSD_TEMPLATE_LOCATION).substitute(
-            xsd_string='<all>' \
+            xsd_string=\
+                '<all><element name="Dummy"><complexType><all>' \
                 + '<element maxOccurs="1" minOccurs="1" name="foo" type="string">' \
                 + '<annotation><appinfo>Foo</appinfo>' \
                 + '<documentation>foo is bar</documentation>' \
-                + '</annotation></element></all>',
+                + '</annotation></element></all>' \
+                + '</complexType></element></all>',
             xsd_namespace=stats_httpd.XSD_NAMESPACE)
-        xsd_body2 = stats_httpd.StatsHttpd().xsd_handler()
+        xsd_body2 = self.stats_httpd.xsd_handler()
         self.assertEqual(type(xsd_body1), str)
         self.assertEqual(type(xsd_body2), str)
         self.assertEqual(xsd_body1, xsd_body2)
-        stats_httpd.StatsHttpd.get_stats_spec = lambda x: \
-            [{
-                "item_name": "bar",
-                "item_type": "string",
-                "item_optional": False,
-                "item_default": "foo",
-                "item_description": "bar is foo",
-                "item_title": "bar"
-               }]
-        xsd_body2 = stats_httpd.StatsHttpd().xsd_handler()
+        self.stats_httpd.get_stats_spec = lambda: \
+            { "Dummy" :
+                  [{
+                        "item_name": "bar",
+                        "item_type": "string",
+                        "item_optional": False,
+                        "item_default": "foo",
+                        "item_description": "bar is foo",
+                        "item_title": "bar"
+                        }]
+              }
+        xsd_body2 = self.stats_httpd.xsd_handler()
         self.assertNotEqual(xsd_body1, xsd_body2)
-        stats_httpd.StatsHttpd.get_stats_spec = orig_get_stats_spec
 
     def test_xsl_handler(self):
-        orig_get_stats_spec = stats_httpd.StatsHttpd.get_stats_spec
-        stats_httpd.StatsHttpd.get_stats_spec = lambda x: \
-            [{
-                "item_name": "foo",
-                "item_type": "string",
-                "item_optional": False,
-                "item_default": "bar",
-                "item_description": "foo is bar",
-                "item_title": "Foo"
-               }]
-        xsl_body1 = stats_httpd.StatsHttpd().open_template(
+        self.stats_httpd = MyStatsHttpd(get_availaddr())
+        self.stats_httpd.get_stats_spec = lambda: \
+            { "Dummy" :
+                  [{
+                        "item_name": "foo",
+                        "item_type": "string",
+                        "item_optional": False,
+                        "item_default": "bar",
+                        "item_description": "foo is bar",
+                        "item_title": "Foo"
+                        }]
+              }
+        xsl_body1 = self.stats_httpd.open_template(
             stats_httpd.XSL_TEMPLATE_LOCATION).substitute(
             xsl_string='<xsl:template match="*"><tr>' \
+                + '<td>Dummy</td>' \
                 + '<td class="title" title="foo is bar">Foo</td>' \
-                + '<td><xsl:value-of select="foo" /></td>' \
+                + '<td><xsl:value-of select="Dummy/foo" /></td>' \
                 + '</tr></xsl:template>',
             xsd_namespace=stats_httpd.XSD_NAMESPACE)
-        xsl_body2 = stats_httpd.StatsHttpd().xsl_handler()
+        xsl_body2 = self.stats_httpd.xsl_handler()
         self.assertEqual(type(xsl_body1), str)
         self.assertEqual(type(xsl_body2), str)
         self.assertEqual(xsl_body1, xsl_body2)
-        stats_httpd.StatsHttpd.get_stats_spec = lambda x: \
-            [{
-                "item_name": "bar",
-                "item_type": "string",
-                "item_optional": False,
-                "item_default": "foo",
-                "item_description": "bar is foo",
-                "item_title": "bar"
-               }]
-        xsl_body2 = stats_httpd.StatsHttpd().xsl_handler()
+        self.stats_httpd.get_stats_spec = lambda: \
+            { "Dummy" :
+                  [{
+                        "item_name": "bar",
+                        "item_type": "string",
+                        "item_optional": False,
+                        "item_default": "foo",
+                        "item_description": "bar is foo",
+                        "item_title": "bar"
+                        }]
+              }
+        xsl_body2 = self.stats_httpd.xsl_handler()
         self.assertNotEqual(xsl_body1, xsl_body2)
-        stats_httpd.StatsHttpd.get_stats_spec = orig_get_stats_spec
 
     def test_for_without_B10_FROM_SOURCE(self):
         # just lets it go through the code without B10_FROM_SOURCE env
@@ -500,8 +686,6 @@ class TestStatsHttpd(unittest.TestCase):
             imp.reload(stats_httpd)
             os.environ["B10_FROM_SOURCE"] = tmppath
             imp.reload(stats_httpd)
-            stats_httpd.socket = fake_socket
-            stats_httpd.select = fake_select
 
 if __name__ == "__main__":
     unittest.main()

src/bin/stats/tests/fake_select.py

@@ -1,43 +0,0 @@
-# Copyright (C) 2011  Internet Systems Consortium.
-#
-# Permission to use, copy, modify, and distribute this software for any
-# purpose with or without fee is hereby granted, provided that the above
-# copyright notice and this permission notice appear in all copies.
-#
-# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
-# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
-# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
-# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
-# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
-# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
-# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-
-"""
-A mock-up module of select
-
-*** NOTE ***
-It is only for testing stats_httpd module and not reusable for
-external module.
-"""
-
-import fake_socket
-import errno
-
-class error(Exception):
-    pass
-
-def select(rlst, wlst, xlst, timeout):
-    if type(timeout) != int and type(timeout) != float:
-            raise TypeError("Error: %s must be integer or float"
-                            % timeout.__class__.__name__)
-    for s in rlst + wlst + xlst:
-        if type(s) != fake_socket.socket:
-            raise TypeError("Error: %s must be a dummy socket"
-                            % s.__class__.__name__)
-        s._called = s._called + 1
-        if s._called > 3:
-            raise error("Something is happened!")
-        elif s._called > 2:
-            raise error(errno.EINTR)
-    return (rlst, wlst, xlst)

src/bin/stats/tests/fake_socket.py

@@ -1,70 +0,0 @@
-# Copyright (C) 2011  Internet Systems Consortium.
-#
-# Permission to use, copy, modify, and distribute this software for any
-# purpose with or without fee is hereby granted, provided that the above
-# copyright notice and this permission notice appear in all copies.
-#
-# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
-# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
-# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
-# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
-# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
-# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
-# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-
-"""
-A mock-up module of socket
-
-*** NOTE ***
-It is only for testing stats_httpd module and not reusable for
-external module.
-"""
-
-import re
-
-AF_INET = 'AF_INET'
-AF_INET6 = 'AF_INET6'
-_ADDRFAMILY = AF_INET
-has_ipv6 = True
-_CLOSED = False
-
-class gaierror(Exception):
-    pass
-
-class error(Exception):
-    pass
-
-class socket:
-
-    def __init__(self, family=None):
-        if family is None:
-            self.address_family = _ADDRFAMILY
-        else:
-            self.address_family = family
-        self._closed = _CLOSED
-        if self._closed:
-            raise error('socket is already closed!')
-        self._called = 0
-
-    def close(self):
-        self._closed = True
-
-    def fileno(self):
-        return id(self)
-
-    def bind(self, server_class):
-        (self.server_address, self.server_port) = server_class
-        if self.address_family not in set([AF_INET, AF_INET6]):
-            raise error("Address family not supported by protocol: %s" % self.address_family)
-        if self.address_family == AF_INET6 and not has_ipv6:
-            raise error("Address family not supported in this machine: %s has_ipv6: %s"
-                        % (self.address_family, str(has_ipv6)))
-        if self.address_family == AF_INET and re.search(':', self.server_address) is not None:
-            raise gaierror("Address family for hostname not supported : %s %s" % (self.server_address, self.address_family))
-        if self.address_family == AF_INET6 and re.search(':', self.server_address) is None:
-            raise error("Cannot assign requested address : %s" % str(self.server_address))
-        if type(self.server_port) is not int:
-            raise TypeError("an integer is required: %s" % str(self.server_port))
-        if self.server_port < 0 or self.server_port > 65535:
-            raise OverflowError("port number must be 0-65535.: %s" % str(self.server_port))

src/bin/stats/tests/fake_time.py

@@ -1,47 +0,0 @@
-# Copyright (C) 2010  Internet Systems Consortium.
-#
-# Permission to use, copy, modify, and distribute this software for any
-# purpose with or without fee is hereby granted, provided that the above
-# copyright notice and this permission notice appear in all copies.
-#
-# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
-# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
-# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
-# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
-# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
-# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
-# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-
-__version__ = "$Revision$"
-
-# This is a dummy time class against a Python standard time class.
-# It is just testing use only.
-# Other methods which time class has is not implemented.
-# (This class isn't orderloaded for time class.)
-
-# These variables are constant. These are example.
-_TEST_TIME_SECS = 1283364938.229088
-_TEST_TIME_STRF = '2010-09-01T18:15:38Z'
-
-def time():
-    """
-    This is a dummy time() method against time.time()
-    """
-    # return float constant value
-    return _TEST_TIME_SECS
-
-def gmtime():
-    """
-    This is a dummy gmtime() method against time.gmtime()
-    """
-    # always return nothing
-    return None
-
-def strftime(*arg):
-    """
-    This is a dummy gmtime() method against time.gmtime()
-    """
-    return _TEST_TIME_STRF
-
-

src/bin/stats/tests/http/Makefile.am

@@ -1,6 +0,0 @@
-EXTRA_DIST = __init__.py server.py
-CLEANFILES = __init__.pyc server.pyc
-CLEANDIRS = __pycache__
-
-clean-local:
-	rm -rf $(CLEANDIRS)

src/bin/stats/tests/http/server.py

@@ -1,96 +0,0 @@
-# Copyright (C) 2011  Internet Systems Consortium.
-#
-# Permission to use, copy, modify, and distribute this software for any
-# purpose with or without fee is hereby granted, provided that the above
-# copyright notice and this permission notice appear in all copies.
-#
-# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
-# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
-# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
-# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
-# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
-# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
-# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-
-"""
-A mock-up module of http.server
-
-*** NOTE ***
-It is only for testing stats_httpd module and not reusable for
-external module.
-"""
-
-import fake_socket
-
-class DummyHttpResponse:
-    def __init__(self, path):
-        self.path = path
-        self.headers={}
-        self.log = ""
-
-    def _write_log(self, msg):
-        self.log = self.log + msg
-
-class HTTPServer:
-    """
-    A mock-up class of http.server.HTTPServer
-    """
-    address_family = fake_socket.AF_INET
-    def __init__(self, server_class, handler_class):
-        self.socket = fake_socket.socket(self.address_family)
-        self.server_class = server_class
-        self.socket.bind(self.server_class)
-        self._handler = handler_class(None, None, self)
-
-    def handle_request(self):
-        pass
-
-    def server_close(self):
-        self.socket.close()
-
-class BaseHTTPRequestHandler:
-    """
-    A mock-up class of http.server.BaseHTTPRequestHandler
-    """
-
-    def __init__(self, request, client_address, server):
-        self.path = "/path/to"
-        self.headers = {}
-        self.server = server
-        self.response = DummyHttpResponse(path=self.path)
-        self.response.write = self._write
-        self.wfile = self.response
-
-    def send_response(self, code=0):
-        if self.path != self.response.path:
-            self.response = DummyHttpResponse(path=self.path)
-        self.response.code = code
-
-    def send_header(self, key, value):
-        if self.path != self.response.path:
-            self.response = DummyHttpResponse(path=self.path)
-        self.response.headers[key] = value
-
-    def end_headers(self):
-        if self.path != self.response.path:
-            self.response = DummyHttpResponse(path=self.path)
-        self.response.wrote_headers = True
-
-    def send_error(self, code, message=None):
-        if self.path != self.response.path:
-            self.response = DummyHttpResponse(path=self.path)
-        self.response.code = code
-        self.response.body = message
-
-    def address_string(self):
-        return 'dummyhost'
-
-    def log_date_time_string(self):
-        return '[DD/MM/YYYY HH:MI:SS]'
-
-    def _write(self, obj):
-        if self.path != self.response.path:
-            self.response = DummyHttpResponse(path=self.path)
-        self.response.body = obj.decode()
-

src/bin/stats/tests/isc/Makefile.am

@@ -1,8 +0,0 @@
-SUBDIRS = cc config util log log_messages
-EXTRA_DIST = __init__.py
-CLEANFILES = __init__.pyc
-
-CLEANDIRS = __pycache__
-
-clean-local:
-	rm -rf $(CLEANDIRS)

src/bin/stats/tests/isc/cc/Makefile.am

@@ -1,7 +0,0 @@
-EXTRA_DIST = __init__.py session.py
-CLEANFILES = __init__.pyc session.pyc
-
-CLEANDIRS = __pycache__
-
-clean-local:
-	rm -rf $(CLEANDIRS)

src/bin/stats/tests/isc/cc/__init__.py

@@ -1 +0,0 @@
-from isc.cc.session import *

src/bin/stats/tests/isc/cc/session.py

@@ -1,156 +0,0 @@
-# Copyright (C) 2010,2011  Internet Systems Consortium.
-#
-# Permission to use, copy, modify, and distribute this software for any
-# purpose with or without fee is hereby granted, provided that the above
-# copyright notice and this permission notice appear in all copies.
-#
-# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
-# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
-# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
-# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
-# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
-# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
-# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-
-"""
-A mock-up module of isc.cc.session
-
-*** NOTE ***
-It is only for testing stats_httpd module and not reusable for
-external module.
-"""
-
-import sys
-import fake_socket
-
-# set a dummy lname
-_TEST_LNAME = '123abc@xxxx'
-
-class Queue():
-    def __init__(self, msg=None, env={}):
-        self.msg = msg
-        self.env = env
-
-    def dump(self):
-        return { 'msg': self.msg, 'env': self.env }
-               
-class SessionError(Exception):
-    pass
-
-class SessionTimeout(Exception):
-    pass
-
-class Session:
-    def __init__(self, socket_file=None, verbose=False):
-        self._lname = _TEST_LNAME
-        self.message_queue = []
-        self.old_message_queue = []
-        try:
-            self._socket = fake_socket.socket()
-        except fake_socket.error as se:
-            raise SessionError(se)
-        self.verbose = verbose
-
-    @property
-    def lname(self):
-        return self._lname
-
-    def close(self):
-        self._socket.close()
-
-    def _clear_queues(self):
-        while len(self.message_queue) > 0:
-            self.dequeue()
-
-    def _next_sequence(self, que=None):
-        return len(self.message_queue)
-
-    def enqueue(self, msg=None, env={}):
-        if self._socket._closed:
-            raise SessionError("Session has been closed.")
-        seq = self._next_sequence()
-        env.update({"seq": 0}) # fixed here
-        que = Queue(msg=msg, env=env)
-        self.message_queue.append(que)
-        if self.verbose:
-            sys.stdout.write("[Session] enqueue: " + str(que.dump()) + "\n")
-        return seq
-
-    def dequeue(self):
-        if self._socket._closed:
-            raise SessionError("Session has been closed.")
-        que = None
-        try:
-            que = self.message_queue.pop(0) # always pop at index 0
-            self.old_message_queue.append(que)
-        except IndexError:
-            que = Queue()
-        if self.verbose:
-            sys.stdout.write("[Session] dequeue: " + str(que.dump()) + "\n")
-        return que
-
-    def get_queue(self, seq=None):
-        if self._socket._closed:
-            raise SessionError("Session has been closed.")
-        if seq is None:
-            seq = len(self.message_queue) - 1
-        que = None
-        try:
-            que = self.message_queue[seq]
-        except IndexError:
-            raise IndexError
-            que = Queue()
-        if self.verbose:
-            sys.stdout.write("[Session] get_queue: " + str(que.dump()) + "\n")
-        return que
-
-    def group_sendmsg(self, msg, group, instance="*", to="*"):
-        return self.enqueue(msg=msg, env={
-                "type": "send",
-                "from": self._lname,
-                "to": to,
-                "group": group,
-                "instance": instance })
-
-    def group_recvmsg(self, nonblock=True, seq=0):
-        que = self.dequeue()
-        if que.msg != None:
-            cmd = que.msg.get("command")
-            if cmd and cmd[0] == 'getstats':
-                # Create answer for command 'getstats'
-                retdata =  { "stats_data": {
-                            'bind10.boot_time' : "1970-01-01T00:00:00Z"
-                           }}
-                return {'result': [0, retdata]}, que.env
-        return que.msg, que.env
-
-    def group_reply(self, routing, msg):
-        return self.enqueue(msg=msg, env={
-                "type": "send",
-                "from": self._lname,
-                "to": routing["from"],
-                "group": routing["group"],
-                "instance": routing["instance"],
-                "reply": routing["seq"] })
-
-    def get_message(self, group, to='*'):
-        if self._socket._closed:
-            raise SessionError("Session has been closed.")
-        que = Queue()
-        for q in self.message_queue:
-            if q.env['group'] == group:
-                self.message_queue.remove(q)
-                self.old_message_queue.append(q)
-                que = q
-        if self.verbose:
-            sys.stdout.write("[Session] get_message: " + str(que.dump()) + "\n")
-        return q.msg
-
-    def group_subscribe(self, group, instance = "*"):
-        if self._socket._closed:
-            raise SessionError("Session has been closed.")
-
-    def group_unsubscribe(self, group, instance = "*"):
-        if self._socket._closed:
-            raise SessionError("Session has been closed.")

src/bin/stats/tests/isc/config/Makefile.am

@@ -1,7 +0,0 @@
-EXTRA_DIST = __init__.py ccsession.py
-CLEANFILES = __init__.pyc ccsession.pyc
-
-CLEANDIRS = __pycache__
-
-clean-local:
-	rm -rf $(CLEANDIRS)

src/bin/stats/tests/isc/config/__init__.py

@@ -1 +0,0 @@
-from isc.config.ccsession import *

src/bin/stats/tests/isc/config/ccsession.py

@@ -1,249 +0,0 @@
-# Copyright (C) 2010,2011  Internet Systems Consortium.
-#
-# Permission to use, copy, modify, and distribute this software for any
-# purpose with or without fee is hereby granted, provided that the above
-# copyright notice and this permission notice appear in all copies.
-#
-# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
-# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
-# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
-# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
-# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
-# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
-# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-
-"""
-A mock-up module of isc.cc.session
-
-*** NOTE ***
-It is only for testing stats_httpd module and not reusable for
-external module.
-"""
-
-import json
-import os
-import time
-from isc.cc.session import Session
-
-COMMAND_CONFIG_UPDATE = "config_update"
-
-def parse_answer(msg):
-    assert 'result' in msg
-    try:
-        return msg['result'][0], msg['result'][1]
-    except IndexError:
-        return msg['result'][0], None
-
-def create_answer(rcode, arg = None):
-    if arg is None:
-        return { 'result': [ rcode ] }
-    else:
-        return { 'result': [ rcode, arg ] }
-
-def parse_command(msg):
-    assert 'command' in msg
-    try:
-        return msg['command'][0], msg['command'][1]
-    except IndexError:
-        return msg['command'][0], None
-
-def create_command(command_name, params = None):
-    if params is None:
-        return {"command": [command_name]}
-    else:
-        return {"command": [command_name, params]}
-
-def module_spec_from_file(spec_file, check = True):
-    try:
-        file = open(spec_file)
-        json_str = file.read()
-        module_spec = json.loads(json_str)
-        file.close()
-        return ModuleSpec(module_spec['module_spec'], check)
-    except IOError as ioe:
-        raise ModuleSpecError("JSON read error: " + str(ioe))
-    except ValueError as ve:
-        raise ModuleSpecError("JSON parse error: " + str(ve))
-    except KeyError as err:
-        raise ModuleSpecError("Data definition has no module_spec element")
-
-class ModuleSpecError(Exception):
-    pass
-
-class ModuleSpec:
-    def __init__(self, module_spec, check = True):
-        # check only confi_data for testing
-        if check and "config_data" in module_spec:
-            _check_config_spec(module_spec["config_data"])
-        self._module_spec = module_spec
-
-    def get_config_spec(self):
-        return self._module_spec['config_data']
-
-    def get_commands_spec(self):
-        return self._module_spec['commands']
-
-    def get_module_name(self):
-        return self._module_spec['module_name']
-
-def _check_config_spec(config_data):
-    # config data is a list of items represented by dicts that contain
-    # things like "item_name", depending on the type they can have
-    # specific subitems
-    """Checks a list that contains the configuration part of the
-       specification. Raises a ModuleSpecError if there is a
-       problem."""
-    if type(config_data) != list:
-        raise ModuleSpecError("config_data is of type " + str(type(config_data)) + ", not a list of items")
-    for config_item in config_data:
-        _check_item_spec(config_item)
-
-def _check_item_spec(config_item):
-    """Checks the dict that defines one config item
-       (i.e. containing "item_name", "item_type", etc.
-       Raises a ModuleSpecError if there is an error"""
-    if type(config_item) != dict:
-        raise ModuleSpecError("item spec not a dict")
-    if "item_name" not in config_item:
-        raise ModuleSpecError("no item_name in config item")
-    if type(config_item["item_name"]) != str:
-        raise ModuleSpecError("item_name is not a string: " + str(config_item["item_name"]))
-    item_name = config_item["item_name"]
-    if "item_type" not in config_item:
-        raise ModuleSpecError("no item_type in config item")
-    item_type = config_item["item_type"]
-    if type(item_type) != str:
-        raise ModuleSpecError("item_type in " + item_name + " is not a string: " + str(type(item_type)))
-    if item_type not in ["integer", "real", "boolean", "string", "list", "map", "any"]:
-        raise ModuleSpecError("unknown item_type in " + item_name + ": " + item_type)
-    if "item_optional" in config_item:
-        if type(config_item["item_optional"]) != bool:
-            raise ModuleSpecError("item_default in " + item_name + " is not a boolean")
-        if not config_item["item_optional"] and "item_default" not in config_item:
-            raise ModuleSpecError("no default value for non-optional item " + item_name)
-    else:
-        raise ModuleSpecError("item_optional not in item " + item_name)
-    if "item_default" in config_item:
-        item_default = config_item["item_default"]
-        if (item_type == "integer" and type(item_default) != int) or \
-           (item_type == "real" and type(item_default) != float) or \
-           (item_type == "boolean" and type(item_default) != bool) or \
-           (item_type == "string" and type(item_default) != str) or \
-           (item_type == "list" and type(item_default) != list) or \
-           (item_type == "map" and type(item_default) != dict):
-            raise ModuleSpecError("Wrong type for item_default in " + item_name)
-    # TODO: once we have check_type, run the item default through that with the list|map_item_spec
-    if item_type == "list":
-        if "list_item_spec" not in config_item:
-            raise ModuleSpecError("no list_item_spec in list item " + item_name)
-        if type(config_item["list_item_spec"]) != dict:
-            raise ModuleSpecError("list_item_spec in " + item_name + " is not a dict")
-        _check_item_spec(config_item["list_item_spec"])
-    if item_type == "map":
-        if "map_item_spec" not in config_item:
-            raise ModuleSpecError("no map_item_sepc in map item " + item_name)
-        if type(config_item["map_item_spec"]) != list:
-            raise ModuleSpecError("map_item_spec in " + item_name + " is not a list")
-        for map_item in config_item["map_item_spec"]:
-            if type(map_item) != dict:
-                raise ModuleSpecError("map_item_spec element is not a dict")
-            _check_item_spec(map_item)
-    if 'item_format' in config_item and 'item_default' in config_item:
-        item_format = config_item["item_format"]
-        item_default = config_item["item_default"]
-        if not _check_format(item_default, item_format):
-            raise ModuleSpecError(
-                "Wrong format for " + str(item_default) + " in " + str(item_name))
-
-def _check_format(value, format_name):
-    """Check if specified value and format are correct. Return True if
-       is is correct."""
-    # TODO: should be added other format types if necessary
-    time_formats = { 'date-time' : "%Y-%m-%dT%H:%M:%SZ",
-                     'date'      : "%Y-%m-%d",
-                     'time'      : "%H:%M:%S" }
-    for fmt in time_formats:
-        if format_name == fmt:
-            try:
-                time.strptime(value, time_formats[fmt])
-                return True
-            except (ValueError, TypeError):
-                break
-    return False
-
-class ModuleCCSessionError(Exception):
-    pass
-
-class DataNotFoundError(Exception):
-    pass
-
-class ConfigData:
-    def __init__(self, specification):
-        self.specification = specification
-
-    def get_value(self, identifier):
-        """Returns a tuple where the first item is the value at the
-           given identifier, and the second item is absolutely False
-           even if the value is an unset default or not. Raises an
-           DataNotFoundError if the identifier is not found in the
-           specification file.
-           *** NOTE ***
-           There are some differences from the original method. This
-           method never handles local settings like the original
-           method. But these different behaviors aren't so big issues
-           for a mock-up method of stats_httpd because stats_httpd
-           calls this method at only first."""
-        for config_map in self.get_module_spec().get_config_spec():
-            if config_map['item_name'] == identifier:
-                if 'item_default' in config_map:
-                    return config_map['item_default'], False
-        raise DataNotFoundError("item_name %s is not found in the specfile" % identifier)
-
-    def get_module_spec(self):
-        return self.specification
-
-class ModuleCCSession(ConfigData):
-    def __init__(self, spec_file_name, config_handler, command_handler, cc_session = None):
-        module_spec = module_spec_from_file(spec_file_name)
-        ConfigData.__init__(self, module_spec)
-        self._module_name = module_spec.get_module_name()
-        self.set_config_handler(config_handler)
-        self.set_command_handler(command_handler)
-        if not cc_session:
-            self._session = Session(verbose=True)
-        else:
-            self._session = cc_session
-
-    def start(self):
-        pass
-
-    def close(self):
-        self._session.close()
-
-    def check_command(self, nonblock=True):
-        msg, env = self._session.group_recvmsg(nonblock)
-        if not msg or 'result' in msg:
-            return
-        cmd, arg = parse_command(msg)
-        answer = None
-        if cmd == COMMAND_CONFIG_UPDATE and self._config_handler:
-            answer = self._config_handler(arg)
-        elif env['group'] == self._module_name and self._command_handler:
-            answer = self._command_handler(cmd, arg)
-        if answer:
-            self._session.group_reply(env, answer)
-
-    def set_config_handler(self, config_handler):
-        self._config_handler = config_handler
-        # should we run this right now since we've changed the handler?
-
-    def set_command_handler(self, command_handler):
-        self._command_handler = command_handler
-
-    def get_module_spec(self):
-        return self.specification
-
-    def get_socket(self):
-        return self._session._socket
-

src/bin/stats/tests/isc/log/Makefile.am

@@ -1,7 +0,0 @@
-EXTRA_DIST = __init__.py
-CLEANFILES = __init__.pyc
-
-CLEANDIRS = __pycache__
-
-clean-local:
-	rm -rf $(CLEANDIRS)

src/bin/stats/tests/isc/log/__init__.py

@@ -1,33 +0,0 @@
-# Copyright (C) 2011  Internet Systems Consortium.
-#
-# Permission to use, copy, modify, and distribute this software for any
-# purpose with or without fee is hereby granted, provided that the above
-# copyright notice and this permission notice appear in all copies.
-#
-# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
-# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
-# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
-# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
-# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
-# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
-# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-
-# This file is not installed. The log.so is installed into the right place.
-# It is only to find it in the .libs directory when we run as a test or
-# from the build directory.
-# But as nobody gives us the builddir explicitly (and we can't use generation
-# from .in file, as it would put us into the builddir and we wouldn't be found)
-# we guess from current directory. Any idea for something better? This should
-# be enough for the tests, but would it work for B10_FROM_SOURCE as well?
-# Should we look there? Or define something in bind10_config?
-
-import os
-import sys
-
-for base in sys.path[:]:
-    loglibdir = os.path.join(base, 'isc/log/.libs')
-    if os.path.exists(loglibdir):
-        sys.path.insert(0, loglibdir)
-
-from log import *

src/bin/stats/tests/isc/util/Makefile.am

@@ -1,7 +0,0 @@
-EXTRA_DIST = __init__.py process.py
-CLEANFILES = __init__.pyc process.pyc
-
-CLEANDIRS = __pycache__
-
-clean-local:
-	rm -rf $(CLEANDIRS)

src/bin/stats/tests/isc/util/process.py

@@ -1,21 +0,0 @@
-# Copyright (C) 2010  Internet Systems Consortium.
-#
-# Permission to use, copy, modify, and distribute this software for any
-# purpose with or without fee is hereby granted, provided that the above
-# copyright notice and this permission notice appear in all copies.
-#
-# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
-# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
-# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
-# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
-# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
-# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
-# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-
-"""
-A dummy function of isc.util.process.rename()
-"""
-
-def rename(name=None):
-    pass

src/bin/stats/tests/test_utils.py

@@ -0,0 +1,364 @@
+"""
+Utilities and mock modules for unittests of statistics modules
+
+"""
+import os
+import io
+import time
+import sys
+import threading
+import tempfile
+import json
+import signal
+
+import msgq
+import isc.config.cfgmgr
+import stats
+import stats_httpd
+
+# Change value of BIND10_MSGQ_SOCKET_FILE in environment variables
+if 'BIND10_MSGQ_SOCKET_FILE' not in os.environ:
+    os.environ['BIND10_MSGQ_SOCKET_FILE'] = tempfile.mktemp(prefix='msgq_socket_')
+
+class SignalHandler():
+    """A signal handler class for deadlock in unittest"""
+    def __init__(self, fail_handler, timeout=20):
+        """sets a schedule in SIGARM for invoking the handler via
+        unittest.TestCase after timeout seconds (default is 20)"""
+        self.fail_handler = fail_handler
+        self.orig_handler = signal.signal(signal.SIGALRM, self.sig_handler)
+        signal.alarm(timeout)
+
+    def reset(self):
+        """resets the schedule in SIGALRM"""
+        signal.alarm(0)
+        signal.signal(signal.SIGALRM, self.orig_handler)
+
+    def sig_handler(self, signal, frame):
+        """envokes unittest.TestCase.fail as a signal handler"""
+        self.fail_handler("A deadlock might be detected")
+
+def send_command(command_name, module_name, params=None, session=None, nonblock=False, timeout=None):
+    if session is not None:
+        cc_session = session
+    else:
+        cc_session = isc.cc.Session()
+    if timeout is not None:
+        orig_timeout = cc_session.get_timeout()
+        cc_session.set_timeout(timeout * 1000)
+    command = isc.config.ccsession.create_command(command_name, params)
+    seq = cc_session.group_sendmsg(command, module_name)
+    try:
+        (answer, env) = cc_session.group_recvmsg(nonblock, seq)
+        if answer:
+            return isc.config.ccsession.parse_answer(answer)
+    except isc.cc.SessionTimeout:
+        pass
+    finally:
+        if timeout is not None:
+            cc_session.set_timeout(orig_timeout)
+        if session is None:
+            cc_session.close()
+
+def send_shutdown(module_name, **kwargs):
+    return send_command("shutdown", module_name, **kwargs)
+
+class ThreadingServerManager:
+    def __init__(self, server, *args, **kwargs):
+        self.server = server(*args, **kwargs)
+        self.server_name = server.__name__
+        self.server._thread = threading.Thread(
+            name=self.server_name, target=self.server.run)
+        self.server._thread.daemon = True
+
+    def run(self):
+        self.server._thread.start()
+        self.server._started.wait()
+        self.server._started.clear()
+
+    def shutdown(self):
+        self.server.shutdown()
+        self.server._thread.join(0) # timeout is 0
+
+def do_nothing(*args, **kwargs): pass
+
+class dummy_sys:
+    """Dummy for sys"""
+    class dummy_io:
+        write = do_nothing
+    stdout = stderr = dummy_io()
+
+class MockMsgq:
+    def __init__(self):
+        self._started = threading.Event()
+        # suppress output to stdout and stderr
+        msgq.sys = dummy_sys()
+        msgq.print = do_nothing
+        self.msgq = msgq.MsgQ(verbose=False)
+        result = self.msgq.setup()
+        if result:
+            sys.exit("Error on Msgq startup: %s" % result)
+
+    def run(self):
+        self._started.set()
+        try:
+            self.msgq.run()
+        except Exception:
+            pass
+        finally:
+            # explicitly shut down the socket of the msgq before
+            # shutting down the msgq
+            self.msgq.listen_socket.shutdown(msgq.socket.SHUT_RDWR)
+            self.msgq.shutdown()
+
+    def shutdown(self):
+        # do nothing for avoiding shutting down the msgq twice
+        pass
+
+class MockCfgmgr:
+    def __init__(self):
+        self._started = threading.Event()
+        self.cfgmgr = isc.config.cfgmgr.ConfigManager(
+            os.environ['CONFIG_TESTDATA_PATH'], "b10-config.db")
+        self.cfgmgr.read_config()
+
+    def run(self):
+        self._started.set()
+        try:
+            self.cfgmgr.run()
+        except Exception:
+            pass
+
+    def shutdown(self):
+        self.cfgmgr.running = False
+
+class MockBoss:
+    spec_str = """\
+{
+  "module_spec": {
+    "module_name": "Boss",
+    "module_description": "Mock Master process",
+    "config_data": [],
+    "commands": [
+      {
+        "command_name": "sendstats",
+        "command_description": "Send data to a statistics module at once",
+        "command_args": []
+      }
+    ],
+    "statistics": [
+      {
+        "item_name": "boot_time",
+        "item_type": "string",
+        "item_optional": false,
+        "item_default": "1970-01-01T00:00:00Z",
+        "item_title": "Boot time",
+        "item_description": "A date time when bind10 process starts initially",
+        "item_format": "date-time"
+      }
+    ]
+  }
+}
+"""
+    _BASETIME = (2011, 6, 22, 8, 14, 8, 2, 173, 0)
+
+    def __init__(self):
+        self._started = threading.Event()
+        self.running = False
+        self.spec_file = io.StringIO(self.spec_str)
+        # create ModuleCCSession object
+        self.mccs = isc.config.ModuleCCSession(
+            self.spec_file,
+            self.config_handler,
+            self.command_handler)
+        self.spec_file.close()
+        self.cc_session = self.mccs._session
+        self.got_command_name = ''
+
+    def run(self):
+        self.mccs.start()
+        self.running = True
+        self._started.set()
+        try:
+            while self.running:
+                self.mccs.check_command(False)
+        except Exception:
+            pass
+
+    def shutdown(self):
+        self.running = False
+
+    def config_handler(self, new_config):
+        return isc.config.create_answer(0)
+
+    def command_handler(self, command, *args, **kwargs):
+        self._started.set()
+        self.got_command_name = command
+        params = { "owner": "Boss",
+                   "data": {
+                'boot_time': time.strftime('%Y-%m-%dT%H:%M:%SZ', self._BASETIME)
+                }
+                   }
+        if command == 'sendstats':
+            send_command("set", "Stats", params=params, session=self.cc_session)
+            return isc.config.create_answer(0)
+        elif command == 'getstats':
+            return isc.config.create_answer(0, params)
+        return isc.config.create_answer(1, "Unknown Command")
+
+class MockAuth:
+    spec_str = """\
+{
+  "module_spec": {
+    "module_name": "Auth",
+    "module_description": "Mock Authoritative service",
+    "config_data": [],
+    "commands": [
+      {
+        "command_name": "sendstats",
+        "command_description": "Send data to a statistics module at once",
+        "command_args": []
+      }
+    ],
+    "statistics": [
+      {
+        "item_name": "queries.tcp",
+        "item_type": "integer",
+        "item_optional": false,
+        "item_default": 0,
+        "item_title": "Queries TCP",
+        "item_description": "A number of total query counts which all auth servers receive over TCP since they started initially"
+      },
+      {
+        "item_name": "queries.udp",
+        "item_type": "integer",
+        "item_optional": false,
+        "item_default": 0,
+        "item_title": "Queries UDP",
+        "item_description": "A number of total query counts which all auth servers receive over UDP since they started initially"
+      }
+    ]
+  }
+}
+"""
+    def __init__(self):
+        self._started = threading.Event()
+        self.running = False
+        self.spec_file = io.StringIO(self.spec_str)
+        # create ModuleCCSession object
+        self.mccs = isc.config.ModuleCCSession(
+            self.spec_file,
+            self.config_handler,
+            self.command_handler)
+        self.spec_file.close()
+        self.cc_session = self.mccs._session
+        self.got_command_name = ''
+        self.queries_tcp = 3
+        self.queries_udp = 2
+
+    def run(self):
+        self.mccs.start()
+        self.running = True
+        self._started.set()
+        try:
+            while self.running:
+                self.mccs.check_command(False)
+        except Exception:
+            pass
+
+    def shutdown(self):
+        self.running = False
+
+    def config_handler(self, new_config):
+        return isc.config.create_answer(0)
+
+    def command_handler(self, command, *args, **kwargs):
+        self.got_command_name = command
+        if command == 'sendstats':
+            params = { "owner": "Auth",
+                       "data": { 'queries.tcp': self.queries_tcp,
+                                 'queries.udp': self.queries_udp } }
+            return send_command("set", "Stats", params=params, session=self.cc_session)
+        return isc.config.create_answer(1, "Unknown Command")
+
+class MyStats(stats.Stats):
+    def __init__(self):
+        self._started = threading.Event()
+        stats.Stats.__init__(self)
+
+    def run(self):
+        self._started.set()
+        try:
+            self.start()
+        except Exception:
+            pass
+
+    def shutdown(self):
+        self.command_shutdown()
+
+class MyStatsHttpd(stats_httpd.StatsHttpd):
+    ORIG_SPECFILE_LOCATION = stats_httpd.SPECFILE_LOCATION
+    def __init__(self, *server_address):
+        self._started = threading.Event()
+        if server_address:
+            stats_httpd.SPECFILE_LOCATION = self.create_specfile(*server_address)
+            try:
+                stats_httpd.StatsHttpd.__init__(self)
+            finally:
+                if hasattr(stats_httpd.SPECFILE_LOCATION, "close"):
+                    stats_httpd.SPECFILE_LOCATION.close()
+                stats_httpd.SPECFILE_LOCATION = self.ORIG_SPECFILE_LOCATION
+        else:
+            stats_httpd.StatsHttpd.__init__(self)
+
+    def create_specfile(self, *server_address):
+        spec_io = open(self.ORIG_SPECFILE_LOCATION)
+        try:
+            spec = json.load(spec_io)
+            spec_io.close()
+            config = spec['module_spec']['config_data']
+            for i in range(len(config)):
+                if config[i]['item_name'] == 'listen_on':
+                    config[i]['item_default'] = \
+                        [ dict(address=a[0], port=a[1]) for a in server_address ]
+                    break
+            return io.StringIO(json.dumps(spec))
+        finally:
+            spec_io.close()
+
+    def run(self):
+        self._started.set()
+        try:
+            self.start()
+        except Exception:
+            pass
+
+    def shutdown(self):
+        self.command_handler('shutdown', None)
+
+class BaseModules:
+    def __init__(self):
+        # MockMsgq
+        self.msgq = ThreadingServerManager(MockMsgq)
+        self.msgq.run()
+        # Check whether msgq is ready. A SessionTimeout is raised here if not.
+        isc.cc.session.Session().close()
+        # MockCfgmgr
+        self.cfgmgr = ThreadingServerManager(MockCfgmgr)
+        self.cfgmgr.run()
+        # MockBoss
+        self.boss = ThreadingServerManager(MockBoss)
+        self.boss.run()
+        # MockAuth
+        self.auth = ThreadingServerManager(MockAuth)
+        self.auth.run()
+
+    def shutdown(self):
+        # MockAuth
+        self.auth.shutdown()
+        # MockBoss
+        self.boss.shutdown()
+        # MockCfgmgr
+        self.cfgmgr.shutdown()
+        # MockMsgq
+        self.msgq.shutdown()

src/bin/stats/tests/testdata/Makefile.am

@@ -1 +0,0 @@
-EXTRA_DIST = stats_test.spec

src/bin/stats/tests/testdata/stats_test.spec

@@ -1,19 +0,0 @@
-{
-  "module_spec": {
-    "module_name": "Stats",
-    "module_description": "Stats daemon",
-    "config_data": [],
-    "commands": [
-      {
-        "command_name": "status",
-        "command_description": "identify whether stats module is alive or not",
-        "command_args": []
-      },
-      {
-        "command_name": "the_dummy",
-        "command_description": "this is for testing",
-        "command_args": []
-      }
-    ]
-  }
-}

src/bin/tests/Makefile.am

@@ -14,7 +14,7 @@ endif
 # test using command-line arguments, so use check-local target instead of TESTS
 check-local:
 if ENABLE_PYTHON_COVERAGE
-	touch $(abs_top_srcdir)/.coverage 
+	touch $(abs_top_srcdir)/.coverage
 	rm -f .coverage
 	${LN_S} $(abs_top_srcdir)/.coverage .coverage
 endif

src/lib/dns/message.cc

@@ -124,10 +124,12 @@ public:
     void setOpcode(const Opcode& opcode);
     void setRcode(const Rcode& rcode);
     int parseQuestion(InputBuffer& buffer);
-    int parseSection(const Message::Section section, InputBuffer& buffer);
+    int parseSection(const Message::Section section, InputBuffer& buffer,
+                     Message::ParseOptions options);
     void addRR(Message::Section section, const Name& name,
                const RRClass& rrclass, const RRType& rrtype,
-               const RRTTL& ttl, ConstRdataPtr rdata);
+               const RRTTL& ttl, ConstRdataPtr rdata,
+               Message::ParseOptions options);
     void addEDNS(Message::Section section, const Name& name,
                  const RRClass& rrclass, const RRType& rrtype,
                  const RRTTL& ttl, const Rdata& rdata);
@@ -614,7 +616,7 @@ Message::parseHeader(InputBuffer& buffer) {
 }
 
 void
-Message::fromWire(InputBuffer& buffer) {
+Message::fromWire(InputBuffer& buffer, ParseOptions options) {
     if (impl_->mode_ != Message::PARSE) {
         isc_throw(InvalidMessageOperation,
                   "Message parse attempted in non parse mode");
@@ -626,11 +628,11 @@ Message::fromWire(InputBuffer& buffer) {
 
     impl_->counts_[SECTION_QUESTION] = impl_->parseQuestion(buffer);
     impl_->counts_[SECTION_ANSWER] =
-        impl_->parseSection(SECTION_ANSWER, buffer);
+        impl_->parseSection(SECTION_ANSWER, buffer, options);
     impl_->counts_[SECTION_AUTHORITY] =
-        impl_->parseSection(SECTION_AUTHORITY, buffer);
+        impl_->parseSection(SECTION_AUTHORITY, buffer, options);
     impl_->counts_[SECTION_ADDITIONAL] =
-        impl_->parseSection(SECTION_ADDITIONAL, buffer);
+        impl_->parseSection(SECTION_ADDITIONAL, buffer, options);
 }
 
 int
@@ -706,7 +708,7 @@ struct MatchRR : public unary_function<RRsetPtr, bool> {
 // is hardcoded here.
 int
 MessageImpl::parseSection(const Message::Section section,
-                          InputBuffer& buffer)
+                          InputBuffer& buffer, Message::ParseOptions options)
 {
     assert(section < MessageImpl::NUM_SECTIONS);
 
@@ -738,7 +740,7 @@ MessageImpl::parseSection(const Message::Section section,
             addTSIG(section, count, buffer, start_position, name, rrclass, ttl,
                     *rdata);
         } else {
-            addRR(section, name, rrclass, rrtype, ttl, rdata);
+            addRR(section, name, rrclass, rrtype, ttl, rdata, options);
             ++added;
         }
     }
@@ -749,19 +751,22 @@ MessageImpl::parseSection(const Message::Section section,
 void
 MessageImpl::addRR(Message::Section section, const Name& name,
                    const RRClass& rrclass, const RRType& rrtype,
-                   const RRTTL& ttl, ConstRdataPtr rdata)
+                   const RRTTL& ttl, ConstRdataPtr rdata,
+                   Message::ParseOptions options)
 {
-    vector<RRsetPtr>::iterator it =
-        find_if(rrsets_[section].begin(), rrsets_[section].end(),
-                MatchRR(name, rrtype, rrclass));
-    if (it != rrsets_[section].end()) {
-        (*it)->setTTL(min((*it)->getTTL(), ttl));
-        (*it)->addRdata(rdata);
-    } else {
-        RRsetPtr rrset(new RRset(name, rrclass, rrtype, ttl));
-        rrset->addRdata(rdata);
-        rrsets_[section].push_back(rrset);
+    if ((options & Message::PRESERVE_ORDER) == 0) {
+        vector<RRsetPtr>::iterator it =
+            find_if(rrsets_[section].begin(), rrsets_[section].end(),
+                    MatchRR(name, rrtype, rrclass));
+        if (it != rrsets_[section].end()) {
+            (*it)->setTTL(min((*it)->getTTL(), ttl));
+            (*it)->addRdata(rdata);
+            return;
+        }
     }
+    RRsetPtr rrset(new RRset(name, rrclass, rrtype, ttl));
+    rrset->addRdata(rdata);
+    rrsets_[section].push_back(rrset);
 }
 
 void

src/lib/dns/message.h

@@ -581,11 +581,58 @@ public:
     /// message
     void toWire(AbstractMessageRenderer& renderer, TSIGContext& tsig_ctx);
 
+    /// Parse options.
+    ///
+    /// describe PRESERVE_ORDER: note doesn't affect EDNS or TSIG.
+    ///
+    /// The option values are used as a parameter for \c fromWire().
+    /// These are values of a bitmask type.  Bitwise operations can be
+    /// performed on these values to express compound options.
+    enum ParseOptions {
+        PARSE_DEFAULT = 0,       ///< The default options
+        PRESERVE_ORDER = 1       ///< Preserve RR order and don't combine them
+    };
+
     /// \brief Parse the header section of the \c Message.
     void parseHeader(isc::util::InputBuffer& buffer);
 
-    /// \brief Parse the \c Message.
-    void fromWire(isc::util::InputBuffer& buffer);
+    /// \brief (Re)build a \c Message object from wire-format data.
+    ///
+    /// This method parses the given wire format data to build a
+    /// complete Message object.  On success, the values of the header section
+    /// fields can be accessible via corresponding get methods, and the
+    /// question and following sections can be accessible via the
+    /// corresponding iterators.  If the message contains an EDNS or TSIG,
+    /// they can be accessible via \c getEDNS() and \c getTSIGRecord(),
+    /// respectively.
+    ///
+    /// This \c Message must be in the \c PARSE mode.
+    ///
+    /// This method performs strict validation on the given message based
+    /// on the DNS protocol specifications.  If the given message data is
+    /// invalid, this method throws an exception (see the exception list).
+    ///
+    /// By default, this method combines RRs of the same name, RR type and
+    /// RR class in a section into a single RRset, even if they are interleaved
+    /// with a different type of RR (though it would be a rare case in
+    /// practice).  If the \c PRESERVE_ORDER option is specified, it handles
+    /// each RR separately, in the appearing order, and converts it to a
+    /// separate RRset (so this RRset should contain exactly one Rdata).
+    /// This mode will be necessary when the higher level protocol is
+    /// ordering conscious.  For example, in AXFR and IXFR, the position of
+    /// the SOA RRs are crucial.
+    ///
+    /// \exception InvalidMessageOperation \c Message is in the RENDER mode
+    /// \exception DNSMessageFORMERR The given message data is syntactically
+    /// \exception MessageTooShort The given data is shorter than a valid
+    /// header section
+    /// \exception std::bad_alloc Memory allocation failure
+    /// \exception Others \c Name, \c Rdata, and \c EDNS classes can also throw
+    ///
+    /// \param buffer A input buffer object that stores the wire data
+    /// \param options Parse options
+    void fromWire(isc::util::InputBuffer& buffer, ParseOptions options
+        = PARSE_DEFAULT);
 
     ///
     /// \name Protocol constants
@@ -629,6 +676,6 @@ std::ostream& operator<<(std::ostream& os, const Message& message);
 }
 #endif  // __MESSAGE_H
 
-// Local Variables: 
+// Local Variables:
 // mode: c++
-// End: 
+// End:

src/lib/dns/python/Makefile.am

@@ -39,6 +39,7 @@ pydnspp_la_CXXFLAGS = $(AM_CXXFLAGS) $(PYTHON_CXXFLAGS)
 pydnspp_la_LDFLAGS = $(PYTHON_LDFLAGS)
 
 EXTRA_DIST = tsigerror_python_inc.cc
+EXTRA_DIST += message_python_inc.cc
 
 # Python prefers .so, while some OSes (specifically MacOS) use a different
 # suffix for dynamic objects.  -module is necessary to work this around.

src/lib/dns/python/message_python.cc

@@ -39,6 +39,9 @@ using namespace isc::dns;
 using namespace isc::dns::python;
 using namespace isc::util;
 
+// Import pydoc text
+#include "message_python_inc.cc"
+
 namespace {
 class s_Message : public PyObject {
 public:
@@ -75,7 +78,7 @@ PyObject* Message_makeResponse(s_Message* self);
 PyObject* Message_toText(s_Message* self);
 PyObject* Message_str(PyObject* self);
 PyObject* Message_toWire(s_Message* self, PyObject* args);
-PyObject* Message_fromWire(s_Message* self, PyObject* args);
+PyObject* Message_fromWire(PyObject* const pyself, PyObject* args);
 
 // This list contains the actual set of functions we have in
 // python. Each entry has
@@ -157,14 +160,7 @@ PyMethodDef Message_methods[] = {
       "If the given message is not in RENDER mode, an "
       "InvalidMessageOperation is raised.\n"
        },
-    { "from_wire", reinterpret_cast<PyCFunction>(Message_fromWire), METH_VARARGS,
-      "Parses the given wire format to a Message object.\n"
-      "The first argument is a Message to parse the data into.\n"
-      "The second argument must implement the buffer interface.\n"
-      "If the given message is not in PARSE mode, an "
-      "InvalidMessageOperation is raised.\n"
-      "Raises MessageTooShort, DNSMessageFORMERR or DNSMessageBADVERS "
-      " if there is a problem parsing the message." },
+    { "from_wire", Message_fromWire, METH_VARARGS, Message_fromWire_doc },
     { NULL, NULL, 0, NULL }
 };
 
@@ -646,30 +642,54 @@ Message_toWire(s_Message* self, PyObject* args) {
 }
 
 PyObject*
-Message_fromWire(s_Message* self, PyObject* args) {
+Message_fromWire(PyObject* const pyself, PyObject* args) {
+    s_Message* self = static_cast<s_Message*>(pyself);
     const char* b;
     Py_ssize_t len;
-    if (!PyArg_ParseTuple(args, "y#", &b, &len)) {
-        return (NULL);
-    }
+    unsigned int options = Message::PARSE_DEFAULT;
+        
+    if (PyArg_ParseTuple(args, "y#", &b, &len) ||
+        PyArg_ParseTuple(args, "y#I", &b, &len, &options)) {
+        // We need to clear the error in case the first call to ParseTuple
+        // fails.
+        PyErr_Clear();
 
-    InputBuffer inbuf(b, len);
-    try {
-        self->cppobj->fromWire(inbuf);
-        Py_RETURN_NONE;
-    } catch (const InvalidMessageOperation& imo) {
-        PyErr_SetString(po_InvalidMessageOperation, imo.what());
-        return (NULL);
-    } catch (const DNSMessageFORMERR& dmfe) {
-        PyErr_SetString(po_DNSMessageFORMERR, dmfe.what());
-        return (NULL);
-    } catch (const DNSMessageBADVERS& dmfe) {
-        PyErr_SetString(po_DNSMessageBADVERS, dmfe.what());
-        return (NULL);
-    } catch (const MessageTooShort& mts) {
-        PyErr_SetString(po_MessageTooShort, mts.what());
-        return (NULL);
+        InputBuffer inbuf(b, len);
+        try {
+            self->cppobj->fromWire(
+                inbuf, static_cast<Message::ParseOptions>(options));
+            Py_RETURN_NONE;
+        } catch (const InvalidMessageOperation& imo) {
+            PyErr_SetString(po_InvalidMessageOperation, imo.what());
+            return (NULL);
+        } catch (const DNSMessageFORMERR& dmfe) {
+            PyErr_SetString(po_DNSMessageFORMERR, dmfe.what());
+            return (NULL);
+        } catch (const DNSMessageBADVERS& dmfe) {
+            PyErr_SetString(po_DNSMessageBADVERS, dmfe.what());
+            return (NULL);
+        } catch (const MessageTooShort& mts) {
+            PyErr_SetString(po_MessageTooShort, mts.what());
+            return (NULL);
+        } catch (const InvalidBufferPosition& ex) {
+            PyErr_SetString(po_DNSMessageFORMERR, ex.what());
+            return (NULL);
+        } catch (const exception& ex) {
+            const string ex_what =
+                "Error in Message.from_wire: " + string(ex.what());
+            PyErr_SetString(PyExc_RuntimeError, ex_what.c_str());
+            return (NULL);
+        } catch (...) {
+            PyErr_SetString(PyExc_RuntimeError,
+                            "Unexpected exception in Message.from_wire");
+            return (NULL);
+        }
     }
+
+    PyErr_SetString(PyExc_TypeError,
+                    "from_wire() arguments must be a byte object and "
+                    "(optional) parse options");
+    return (NULL);
 }
 
 } // end of unnamed namespace

src/lib/dns/python/message_python_inc.cc

@@ -0,0 +1,41 @@
+namespace {
+const char* const Message_fromWire_doc = "\
+from_wire(data, options=PARSE_DEFAULT)\n\
+\n\
+(Re)build a Message object from wire-format data.\n\
+\n\
+This method parses the given wire format data to build a complete\n\
+Message object. On success, the values of the header section fields\n\
+can be accessible via corresponding get methods, and the question and\n\
+following sections can be accessible via the corresponding iterators.\n\
+If the message contains an EDNS or TSIG, they can be accessible via\n\
+get_edns() and get_tsig_record(), respectively.\n\
+\n\
+This Message must be in the PARSE mode.\n\
+\n\
+This method performs strict validation on the given message based on\n\
+the DNS protocol specifications. If the given message data is invalid,\n\
+this method throws an exception (see the exception list).\n\
+\n\
+By default, this method combines RRs of the same name, RR type and RR\n\
+class in a section into a single RRset, even if they are interleaved\n\
+with a different type of RR (though it would be a rare case in\n\
+practice). If the PRESERVE_ORDER option is specified, it handles each\n\
+RR separately, in the appearing order, and converts it to a separate\n\
+RRset (so this RRset should contain exactly one Rdata). This mode will\n\
+be necessary when the higher level protocol is ordering conscious. For\n\
+example, in AXFR and IXFR, the position of the SOA RRs are crucial.\n\
+\n\
+Exceptions:\n\
+  InvalidMessageOperation Message is in the RENDER mode\n\
+  DNSMessageFORMERR The given message data is syntactically\n\
+  MessageTooShort The given data is shorter than a valid header\n\
+             section\n\
+  Others     Name, Rdata, and EDNS classes can also throw\n\
+\n\
+Parameters:\n\
+  data       A byte object of the wire data\n\
+  options    Parse options\n\
+\n\
+";
+} // unnamed namespace

src/lib/dns/python/pydnspp.cc

@@ -89,64 +89,91 @@ initModulePart_Message(PyObject* mod) {
     if (PyType_Ready(&message_type) < 0) {
         return (false);
     }
+    void* p = &message_type;
+    if (PyModule_AddObject(mod, "Message", static_cast<PyObject*>(p)) < 0) {
+        return (false);
+    }
     Py_INCREF(&message_type);
 
-    // Class variables
-    // These are added to the tp_dict of the type object
-    //
-    addClassVariable(message_type, "PARSE",
-                     Py_BuildValue("I", Message::PARSE));
-    addClassVariable(message_type, "RENDER",
-                     Py_BuildValue("I", Message::RENDER));
-
-    addClassVariable(message_type, "HEADERFLAG_QR",
-                     Py_BuildValue("I", Message::HEADERFLAG_QR));
-    addClassVariable(message_type, "HEADERFLAG_AA",
-                     Py_BuildValue("I", Message::HEADERFLAG_AA));
-    addClassVariable(message_type, "HEADERFLAG_TC",
-                     Py_BuildValue("I", Message::HEADERFLAG_TC));
-    addClassVariable(message_type, "HEADERFLAG_RD",
-                     Py_BuildValue("I", Message::HEADERFLAG_RD));
-    addClassVariable(message_type, "HEADERFLAG_RA",
-                     Py_BuildValue("I", Message::HEADERFLAG_RA));
-    addClassVariable(message_type, "HEADERFLAG_AD",
-                     Py_BuildValue("I", Message::HEADERFLAG_AD));
-    addClassVariable(message_type, "HEADERFLAG_CD",
-                     Py_BuildValue("I", Message::HEADERFLAG_CD));
-
-    addClassVariable(message_type, "SECTION_QUESTION",
-                     Py_BuildValue("I", Message::SECTION_QUESTION));
-    addClassVariable(message_type, "SECTION_ANSWER",
-                     Py_BuildValue("I", Message::SECTION_ANSWER));
-    addClassVariable(message_type, "SECTION_AUTHORITY",
-                     Py_BuildValue("I", Message::SECTION_AUTHORITY));
-    addClassVariable(message_type, "SECTION_ADDITIONAL",
-                     Py_BuildValue("I", Message::SECTION_ADDITIONAL));
-
-    addClassVariable(message_type, "DEFAULT_MAX_UDPSIZE",
-                     Py_BuildValue("I", Message::DEFAULT_MAX_UDPSIZE));
-
-    /* Class-specific exceptions */
-    po_MessageTooShort = PyErr_NewException("pydnspp.MessageTooShort", NULL,
-                                            NULL);
-    PyModule_AddObject(mod, "MessageTooShort", po_MessageTooShort);
-    po_InvalidMessageSection =
-        PyErr_NewException("pydnspp.InvalidMessageSection", NULL, NULL);
-    PyModule_AddObject(mod, "InvalidMessageSection", po_InvalidMessageSection);
-    po_InvalidMessageOperation =
-        PyErr_NewException("pydnspp.InvalidMessageOperation", NULL, NULL);
-    PyModule_AddObject(mod, "InvalidMessageOperation",
-                       po_InvalidMessageOperation);
-    po_InvalidMessageUDPSize =
-        PyErr_NewException("pydnspp.InvalidMessageUDPSize", NULL, NULL);
-    PyModule_AddObject(mod, "InvalidMessageUDPSize", po_InvalidMessageUDPSize);
-    po_DNSMessageBADVERS = PyErr_NewException("pydnspp.DNSMessageBADVERS",
-                                              NULL, NULL);
-    PyModule_AddObject(mod, "DNSMessageBADVERS", po_DNSMessageBADVERS);
-
-    PyModule_AddObject(mod, "Message",
-                       reinterpret_cast<PyObject*>(&message_type));
-
+    try {
+        //
+        // Constant class variables
+        //
+
+        // Parse mode
+        installClassVariable(message_type, "PARSE",
+                             Py_BuildValue("I", Message::PARSE));
+        installClassVariable(message_type, "RENDER",
+                             Py_BuildValue("I", Message::RENDER));
+
+        // Parse options
+        installClassVariable(message_type, "PARSE_DEFAULT",
+                             Py_BuildValue("I", Message::PARSE_DEFAULT));
+        installClassVariable(message_type, "PRESERVE_ORDER",
+                             Py_BuildValue("I", Message::PRESERVE_ORDER));
+
+        // Header flags
+        installClassVariable(message_type, "HEADERFLAG_QR",
+                             Py_BuildValue("I", Message::HEADERFLAG_QR));
+        installClassVariable(message_type, "HEADERFLAG_AA",
+                             Py_BuildValue("I", Message::HEADERFLAG_AA));
+        installClassVariable(message_type, "HEADERFLAG_TC",
+                             Py_BuildValue("I", Message::HEADERFLAG_TC));
+        installClassVariable(message_type, "HEADERFLAG_RD",
+                             Py_BuildValue("I", Message::HEADERFLAG_RD));
+        installClassVariable(message_type, "HEADERFLAG_RA",
+                             Py_BuildValue("I", Message::HEADERFLAG_RA));
+        installClassVariable(message_type, "HEADERFLAG_AD",
+                             Py_BuildValue("I", Message::HEADERFLAG_AD));
+        installClassVariable(message_type, "HEADERFLAG_CD",
+                             Py_BuildValue("I", Message::HEADERFLAG_CD));
+
+        // Sections
+        installClassVariable(message_type, "SECTION_QUESTION",
+                             Py_BuildValue("I", Message::SECTION_QUESTION));
+        installClassVariable(message_type, "SECTION_ANSWER",
+                             Py_BuildValue("I", Message::SECTION_ANSWER));
+        installClassVariable(message_type, "SECTION_AUTHORITY",
+                             Py_BuildValue("I", Message::SECTION_AUTHORITY));
+        installClassVariable(message_type, "SECTION_ADDITIONAL",
+                             Py_BuildValue("I", Message::SECTION_ADDITIONAL));
+
+        // Protocol constant
+        installClassVariable(message_type, "DEFAULT_MAX_UDPSIZE",
+                             Py_BuildValue("I", Message::DEFAULT_MAX_UDPSIZE));
+
+        /* Class-specific exceptions */
+        po_MessageTooShort =
+            PyErr_NewException("pydnspp.MessageTooShort", NULL, NULL);
+        PyObjectContainer(po_MessageTooShort).installToModule(
+            mod, "MessageTooShort");
+        po_InvalidMessageSection =
+            PyErr_NewException("pydnspp.InvalidMessageSection", NULL, NULL);
+        PyObjectContainer(po_InvalidMessageSection).installToModule(
+            mod, "InvalidMessageSection");
+        po_InvalidMessageOperation =
+            PyErr_NewException("pydnspp.InvalidMessageOperation", NULL, NULL);
+        PyObjectContainer(po_InvalidMessageOperation).installToModule(
+            mod, "InvalidMessageOperation");
+        po_InvalidMessageUDPSize =
+            PyErr_NewException("pydnspp.InvalidMessageUDPSize", NULL, NULL);
+        PyObjectContainer(po_InvalidMessageUDPSize).installToModule(
+            mod, "InvalidMessageUDPSize");
+        po_DNSMessageBADVERS =
+            PyErr_NewException("pydnspp.DNSMessageBADVERS", NULL, NULL);
+        PyObjectContainer(po_DNSMessageBADVERS).installToModule(
+            mod, "DNSMessageBADVERS");
+    } catch (const std::exception& ex) {
+        const std::string ex_what =
+            "Unexpected failure in Message initialization: " +
+            std::string(ex.what());
+        PyErr_SetString(po_IscException, ex_what.c_str());
+        return (false);
+    } catch (...) {
+        PyErr_SetString(PyExc_SystemError,
+                        "Unexpected failure in Message initialization");
+        return (false);
+    }
 
     return (true);
 }

src/lib/dns/python/tests/message_python_test.py

@@ -29,9 +29,9 @@ if "TESTDATA_PATH" in os.environ:
 else:
     testdata_path = "../tests/testdata"
 
-def factoryFromFile(message, file):
+def factoryFromFile(message, file, parse_options=Message.PARSE_DEFAULT):
     data = read_wire_data(file)
-    message.from_wire(data)
+    message.from_wire(data, parse_options)
     return data
 
 # we don't have direct comparison for rrsets right now (should we?
@@ -466,6 +466,54 @@ test.example.com. 3600 IN A 192.0.2.2
         self.assertEqual("192.0.2.2", rdata[1].to_text())
         self.assertEqual(2, len(rdata))
 
+    def test_from_wire_short_buffer(self):
+        data = read_wire_data("message_fromWire22.wire")
+        self.assertRaises(DNSMessageFORMERR, self.p.from_wire, data[:-1])
+
+    def test_from_wire_combind_rrs(self):
+        factoryFromFile(self.p, "message_fromWire19.wire")
+        rrset = self.p.get_section(Message.SECTION_ANSWER)[0]
+        self.assertEqual(RRType("A"), rrset.get_type())
+        self.assertEqual(2, len(rrset.get_rdata()))
+
+        rrset = self.p.get_section(Message.SECTION_ANSWER)[1]
+        self.assertEqual(RRType("AAAA"), rrset.get_type())
+        self.assertEqual(1, len(rrset.get_rdata()))
+
+    def check_preserve_rrs(self, message, section):
+        rrset = message.get_section(section)[0]
+        self.assertEqual(RRType("A"), rrset.get_type())
+        rdata = rrset.get_rdata()
+        self.assertEqual(1, len(rdata))
+        self.assertEqual('192.0.2.1', rdata[0].to_text())
+
+        rrset = message.get_section(section)[1]
+        self.assertEqual(RRType("AAAA"), rrset.get_type())
+        rdata = rrset.get_rdata()
+        self.assertEqual(1, len(rdata))
+        self.assertEqual('2001:db8::1', rdata[0].to_text())
+
+        rrset = message.get_section(section)[2]
+        self.assertEqual(RRType("A"), rrset.get_type())
+        rdata = rrset.get_rdata()
+        self.assertEqual(1, len(rdata))
+        self.assertEqual('192.0.2.2', rdata[0].to_text())
+
+    def test_from_wire_preserve_answer(self):
+        factoryFromFile(self.p, "message_fromWire19.wire",
+                        Message.PRESERVE_ORDER)
+        self.check_preserve_rrs(self.p, Message.SECTION_ANSWER)
+
+    def test_from_wire_preserve_authority(self):
+        factoryFromFile(self.p, "message_fromWire20.wire",
+                        Message.PRESERVE_ORDER)
+        self.check_preserve_rrs(self.p, Message.SECTION_AUTHORITY)
+
+    def test_from_wire_preserve_additional(self):
+        factoryFromFile(self.p, "message_fromWire21.wire",
+                        Message.PRESERVE_ORDER)
+        self.check_preserve_rrs(self.p, Message.SECTION_ADDITIONAL)
+
     def test_EDNS0ExtCode(self):
         # Extended Rcode = BADVERS
         message_parse = Message(Message.PARSE)

src/lib/dns/tests/message_unittest.cc

@@ -118,16 +118,20 @@ protected:
     vector<unsigned char> received_data;
     vector<unsigned char> expected_data;
 
-    void factoryFromFile(Message& message, const char* datafile);
+    void factoryFromFile(Message& message, const char* datafile,
+                         Message::ParseOptions options =
+                         Message::PARSE_DEFAULT);
 };
 
 void
-MessageTest::factoryFromFile(Message& message, const char* datafile) {
+MessageTest::factoryFromFile(Message& message, const char* datafile,
+                             Message::ParseOptions options)
+{
     received_data.clear();
     UnitTestUtil::readWireData(datafile, received_data);
 
     InputBuffer buffer(&received_data[0], received_data.size());
-    message.fromWire(buffer);
+    message.fromWire(buffer, options);
 }
 
 TEST_F(MessageTest, headerFlag) {
@@ -175,7 +179,6 @@ TEST_F(MessageTest, headerFlag) {
     EXPECT_THROW(message_parse.setHeaderFlag(Message::HEADERFLAG_QR),
                  InvalidMessageOperation);
 }
-
 TEST_F(MessageTest, getEDNS) {
     EXPECT_FALSE(message_parse.getEDNS()); // by default EDNS isn't set
 
@@ -532,7 +535,46 @@ TEST_F(MessageTest, appendSection) {
     
 }
 
+TEST_F(MessageTest, parseHeader) {
+    received_data.clear();
+    UnitTestUtil::readWireData("message_fromWire1", received_data);
+
+    // parseHeader() isn't allowed in the render mode.
+    InputBuffer buffer(&received_data[0], received_data.size());
+    EXPECT_THROW(message_render.parseHeader(buffer), InvalidMessageOperation);
+
+    message_parse.parseHeader(buffer);
+    EXPECT_EQ(0x1035, message_parse.getQid());
+    EXPECT_EQ(Opcode::QUERY(), message_parse.getOpcode());
+    EXPECT_EQ(Rcode::NOERROR(), message_parse.getRcode());
+    EXPECT_TRUE(message_parse.getHeaderFlag(Message::HEADERFLAG_QR));
+    EXPECT_TRUE(message_parse.getHeaderFlag(Message::HEADERFLAG_AA));
+    EXPECT_FALSE(message_parse.getHeaderFlag(Message::HEADERFLAG_TC));
+    EXPECT_TRUE(message_parse.getHeaderFlag(Message::HEADERFLAG_RD));
+    EXPECT_FALSE(message_parse.getHeaderFlag(Message::HEADERFLAG_RA));
+    EXPECT_FALSE(message_parse.getHeaderFlag(Message::HEADERFLAG_AD));
+    EXPECT_FALSE(message_parse.getHeaderFlag(Message::HEADERFLAG_CD));
+    EXPECT_EQ(1, message_parse.getRRCount(Message::SECTION_QUESTION));
+    EXPECT_EQ(2, message_parse.getRRCount(Message::SECTION_ANSWER));
+    EXPECT_EQ(0, message_parse.getRRCount(Message::SECTION_AUTHORITY));
+    EXPECT_EQ(0, message_parse.getRRCount(Message::SECTION_ADDITIONAL));
+
+    // Only the header part should have been examined.
+    EXPECT_EQ(12, buffer.getPosition()); // 12 = size of the header section
+    EXPECT_TRUE(message_parse.beginQuestion() == message_parse.endQuestion());
+    EXPECT_TRUE(message_parse.beginSection(Message::SECTION_ANSWER) ==
+                message_parse.endSection(Message::SECTION_ANSWER));
+    EXPECT_TRUE(message_parse.beginSection(Message::SECTION_AUTHORITY) ==
+                message_parse.endSection(Message::SECTION_AUTHORITY));
+    EXPECT_TRUE(message_parse.beginSection(Message::SECTION_ADDITIONAL) ==
+                message_parse.endSection(Message::SECTION_ADDITIONAL));
+}
+
 TEST_F(MessageTest, fromWire) {
+    // fromWire() isn't allowed in the render mode.
+    EXPECT_THROW(factoryFromFile(message_render, "message_fromWire1"),
+                 InvalidMessageOperation);
+
     factoryFromFile(message_parse, "message_fromWire1");
     EXPECT_EQ(0x1035, message_parse.getQid());
     EXPECT_EQ(Opcode::QUERY(), message_parse.getOpcode());
@@ -564,6 +606,87 @@ TEST_F(MessageTest, fromWire) {
     EXPECT_TRUE(it->isLast());
 }
 
+TEST_F(MessageTest, fromWireShortBuffer) {
+    // We trim a valid message (ending with an SOA RR) for one byte.
+    // fromWire() should throw an exception while parsing the trimmed RR.
+    UnitTestUtil::readWireData("message_fromWire22.wire", received_data);
+    InputBuffer buffer(&received_data[0], received_data.size() - 1);
+    EXPECT_THROW(message_parse.fromWire(buffer), InvalidBufferPosition);
+}
+
+TEST_F(MessageTest, fromWireCombineRRs) {
+    // This message contains 3 RRs in the answer section in the order of
+    // A, AAAA, A types.  fromWire() should combine the two A RRs into a
+    // single RRset by default.
+    factoryFromFile(message_parse, "message_fromWire19.wire");
+
+    RRsetIterator it = message_parse.beginSection(Message::SECTION_ANSWER);
+    RRsetIterator it_end = message_parse.endSection(Message::SECTION_ANSWER);
+    ASSERT_TRUE(it != it_end);
+    EXPECT_EQ(RRType::A(), (*it)->getType());
+    EXPECT_EQ(2, (*it)->getRdataCount());
+
+    ++it;
+    ASSERT_TRUE(it != it_end);
+    EXPECT_EQ(RRType::AAAA(), (*it)->getType());
+    EXPECT_EQ(1, (*it)->getRdataCount());
+}
+
+// A helper function for a test pattern commonly used in several tests below.
+void
+preserveRRCheck(const Message& message, Message::Section section) {
+    RRsetIterator it = message.beginSection(section);
+    RRsetIterator it_end = message.endSection(section);
+    ASSERT_TRUE(it != it_end);
+    EXPECT_EQ(RRType::A(), (*it)->getType());
+    EXPECT_EQ(1, (*it)->getRdataCount());
+    EXPECT_EQ("192.0.2.1", (*it)->getRdataIterator()->getCurrent().toText());
+
+    ++it;
+    ASSERT_TRUE(it != it_end);
+    EXPECT_EQ(RRType::AAAA(), (*it)->getType());
+    EXPECT_EQ(1, (*it)->getRdataCount());
+    EXPECT_EQ("2001:db8::1", (*it)->getRdataIterator()->getCurrent().toText());
+
+    ++it;
+    ASSERT_TRUE(it != it_end);
+    EXPECT_EQ(RRType::A(), (*it)->getType());
+    EXPECT_EQ(1, (*it)->getRdataCount());
+    EXPECT_EQ("192.0.2.2", (*it)->getRdataIterator()->getCurrent().toText());
+}
+
+TEST_F(MessageTest, fromWirePreserveAnswer) {
+    // Using the same data as the previous test, but specify the PRESERVE_ORDER
+    // option.  The received order of RRs should be preserved, and each RR
+    // should be stored in a single RRset.
+    factoryFromFile(message_parse, "message_fromWire19.wire",
+                    Message::PRESERVE_ORDER);
+    {
+        SCOPED_TRACE("preserve answer RRs");
+        preserveRRCheck(message_parse, Message::SECTION_ANSWER);
+    }
+}
+
+TEST_F(MessageTest, fromWirePreserveAuthority) {
+    // Same for the previous test, but for the authority section.
+    factoryFromFile(message_parse, "message_fromWire20.wire",
+                    Message::PRESERVE_ORDER);
+    {
+        SCOPED_TRACE("preserve authority RRs");
+        preserveRRCheck(message_parse, Message::SECTION_AUTHORITY);
+    }
+}
+
+TEST_F(MessageTest, fromWirePreserveAdditional) {
+    // Same for the previous test, but for the additional section.
+    factoryFromFile(message_parse, "message_fromWire21.wire",
+                    Message::PRESERVE_ORDER);
+    {
+        SCOPED_TRACE("preserve additional RRs");
+        preserveRRCheck(message_parse, Message::SECTION_ADDITIONAL);
+    }
+}
+
 TEST_F(MessageTest, EDNS0ExtRcode) {
     // Extended Rcode = BADVERS
     factoryFromFile(message_parse, "message_fromWire10.wire");

src/lib/dns/tests/testdata/Makefile.am

@@ -6,7 +6,9 @@ BUILT_SOURCES += message_fromWire10.wire message_fromWire11.wire
 BUILT_SOURCES += message_fromWire12.wire message_fromWire13.wire
 BUILT_SOURCES += message_fromWire14.wire message_fromWire15.wire
 BUILT_SOURCES += message_fromWire16.wire message_fromWire17.wire
-BUILT_SOURCES += message_fromWire18.wire
+BUILT_SOURCES += message_fromWire18.wire message_fromWire19.wire
+BUILT_SOURCES += message_fromWire20.wire message_fromWire21.wire
+BUILT_SOURCES += message_fromWire22.wire
 BUILT_SOURCES += message_toWire2.wire message_toWire3.wire
 BUILT_SOURCES += message_toWire4.wire message_toWire5.wire
 BUILT_SOURCES += message_toText1.wire message_toText2.wire
@@ -71,6 +73,8 @@ EXTRA_DIST += message_fromWire11.spec message_fromWire12.spec
 EXTRA_DIST += message_fromWire13.spec message_fromWire14.spec
 EXTRA_DIST += message_fromWire15.spec message_fromWire16.spec
 EXTRA_DIST += message_fromWire17.spec message_fromWire18.spec
+EXTRA_DIST += message_fromWire19.spec message_fromWire20.spec
+EXTRA_DIST += message_fromWire21.spec message_fromWire22.spec
 EXTRA_DIST += message_toWire1 message_toWire2.spec message_toWire3.spec
 EXTRA_DIST += message_toWire4.spec message_toWire5.spec
 EXTRA_DIST += message_toText1.txt message_toText1.spec

src/lib/dns/tests/testdata/message_fromWire19.spec

@@ -0,0 +1,20 @@
+#
+# A non realistic DNS response message containing mixed types of RRs in the
+# answer section in a mixed order.
+#
+
+[custom]
+sections: header:question:a/1:aaaa:a/2
+[header]
+qr: 1
+ancount: 3
+[question]
+name: www.example.com
+rrtype: A
+[a/1]
+as_rr: True
+[aaaa]
+as_rr: True
+[a/2]
+as_rr: True
+address: 192.0.2.2

src/lib/dns/tests/testdata/message_fromWire20.spec

@@ -0,0 +1,20 @@
+#
+# A non realistic DNS response message containing mixed types of RRs in the
+# authority section in a mixed order.
+#
+
+[custom]
+sections: header:question:a/1:aaaa:a/2
+[header]
+qr: 1
+nscount: 3
+[question]
+name: www.example.com
+rrtype: A
+[a/1]
+as_rr: True
+[aaaa]
+as_rr: True
+[a/2]
+as_rr: True
+address: 192.0.2.2

src/lib/dns/tests/testdata/message_fromWire21.spec

@@ -0,0 +1,20 @@
+#
+# A non realistic DNS response message containing mixed types of RRs in the
+# additional section in a mixed order.
+#
+
+[custom]
+sections: header:question:a/1:aaaa:a/2
+[header]
+qr: 1
+arcount: 3
+[question]
+name: www.example.com
+rrtype: A
+[a/1]
+as_rr: True
+[aaaa]
+as_rr: True
+[a/2]
+as_rr: True
+address: 192.0.2.2

src/lib/dns/tests/testdata/message_fromWire22.spec

@@ -0,0 +1,14 @@
+#
+# A simple DNS message containing one SOA RR in the answer section.  This is
+# intended to be trimmed to emulate a bogus message.
+#
+
+[custom]
+sections: header:question:soa
+[header]
+qr: 1
+ancount: 1
+[question]
+rrtype: SOA
+[soa]
+as_rr: True

src/lib/python/isc/log/log.cc

@@ -185,7 +185,7 @@ init(PyObject*, PyObject* args) {
     Py_RETURN_NONE;
 }
 
-// This initialization is for unit tests.  It allows message settings to be
+// This initialization is for unit tests.  It allows message settings to
 // be determined by a set of B10_xxx environment variables.  (See the
 // description of initLogger() for more details.)  The function has been named
 // resetUnitTestRootLogger() here as being more descriptive and

tests/system/bindctl/tests.sh

@@ -24,6 +24,10 @@ SYSTEMTESTTOP=..
 status=0
 n=0
 
+# TODO: consider consistency with statistics definition in auth.spec
+auth_queries_tcp="\<queries\.tcp\>"
+auth_queries_udp="\<queries\.udp\>"
+
 echo "I:Checking b10-auth is working by default ($n)"
 $DIG +norec @10.53.0.1 -p 53210 ns.example.com. A >dig.out.$n || status=1
 # perform a simple check on the output (digcomp would be too much for this)
@@ -40,8 +44,8 @@ echo 'Stats show
 	--csv-file-dir=$BINDCTL_CSV_DIR > bindctl.out.$n || status=1
 # the server should have received 1 UDP and 1 TCP queries (TCP query was
 # sent from the server startup script)
-grep "\"auth.queries.tcp\": 1," bindctl.out.$n > /dev/null || status=1
-grep "\"auth.queries.udp\": 1," bindctl.out.$n > /dev/null || status=1
+grep $auth_queries_tcp".*\<1\>" bindctl.out.$n > /dev/null || status=1
+grep $auth_queries_udp".*\<1\>" bindctl.out.$n > /dev/null || status=1
 if [ $status != 0 ]; then echo "I:failed"; fi
 n=`expr $n + 1`
 
@@ -73,8 +77,8 @@ echo 'Stats show
 ' | $RUN_BINDCTL \
 	--csv-file-dir=$BINDCTL_CSV_DIR > bindctl.out.$n || status=1
 # The statistics counters should have been reset while stop/start.
-grep "\"auth.queries.tcp\": 0," bindctl.out.$n > /dev/null || status=1
-grep "\"auth.queries.udp\": 1," bindctl.out.$n > /dev/null || status=1
+grep $auth_queries_tcp".*\<0\>" bindctl.out.$n > /dev/null || status=1
+grep $auth_queries_udp".*\<1\>" bindctl.out.$n > /dev/null || status=1
 if [ $status != 0 ]; then echo "I:failed"; fi
 n=`expr $n + 1`
 
@@ -97,8 +101,8 @@ echo 'Stats show
 ' | $RUN_BINDCTL \
 	--csv-file-dir=$BINDCTL_CSV_DIR > bindctl.out.$n || status=1
 # The statistics counters shouldn't be reset due to hot-swapping datasource.
-grep "\"auth.queries.tcp\": 0," bindctl.out.$n > /dev/null || status=1
-grep "\"auth.queries.udp\": 2," bindctl.out.$n > /dev/null || status=1
+grep $auth_queries_tcp".*\<0\>" bindctl.out.$n > /dev/null || status=1
+grep $auth_queries_udp".*\<2\>" bindctl.out.$n > /dev/null || status=1
 if [ $status != 0 ]; then echo "I:failed"; fi
 n=`expr $n + 1`