[917] implement stats_data2xml and modify definition of per-zone count

src/bin/stats/stats-httpd-xml.tpl

@@ -15,10 +15,4 @@
  - OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
  - PERFORMANCE OF THIS SOFTWARE.
 -->
-
-<stats:stats_data version="1.0"
-  xmlns:stats="$xsd_namespace"
-  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="$xsd_namespace $xsd_url_path">
-  $xml_string
-</stats:stats_data>
+$xml_string

src/bin/stats/stats_httpd.py.in

@@ -29,6 +29,7 @@ import http.server
 import socket
 import string
 import xml.etree.ElementTree
+import urllib.parse
 
 import isc.cc
 import isc.config
@@ -85,14 +86,28 @@ class HttpHandler(http.server.BaseHTTPRequestHandler):
 
     def send_head(self):
         try:
-            if self.path == XML_URL_PATH:
-                body = self.server.xml_handler()
-            elif self.path == XSD_URL_PATH:
-                body = self.server.xsd_handler()
-            elif self.path == XSL_URL_PATH:
-                body = self.server.xsl_handler()
+            req_path = self.path
+            req_path = urllib.parse.urlsplit(req_path).path
+            req_path = urllib.parse.unquote(req_path)
+            req_path = os.path.normpath(req_path)
+            path_dirs = req_path.split('/')
+            path_dirs = [ d for d in filter(None, path_dirs) ]
+            module_name = None
+            item_name = None
+            # in case of /bind10/statistics/xxx/YYY/zzz/
+            if len(path_dirs) >= 5:
+                item_name = path_dirs[4]
+            # in case of /bind10/statistics/xxx/YYY/
+            if len(path_dirs) >= 4:
+                module_name = path_dirs[3]
+            if req_path.startswith(XML_URL_PATH):
+                body = self.server.xml_handler(module_name, item_name)
+            elif req_path.startswith(XSD_URL_PATH):
+                body = self.server.xsd_handler(module_name, item_name)
+            elif req_path.startswith(XSL_URL_PATH):
+                body = self.server.xsl_handler(module_name, item_name)
             else:
-                if self.path == '/' and 'Host' in self.headers.keys():
+                if req_path == '/' and 'Host' in self.headers.keys():
                     # redirect to XML URL only when requested with '/'
                     self.send_response(302)
                     self.send_header(
@@ -334,12 +349,19 @@ class StatsHttpd:
             return isc.config.ccsession.create_answer(
                 1, "Unknown command: " + str(command))
 
-    def get_stats_data(self):
+    def get_stats_data(self, owner=None, name=None):
         """Requests statistics data to the Stats daemon and returns
-        the data which obtains from it"""
+        the data which obtains from it. args are owner and name."""
+        param = {}
+        if owner is None and name is None:
+            param = None
+        if owner is not None:
+            param['owner'] = owner
+        if name is not None:
+            param['name'] = name
         try:
             seq = self.cc_session.group_sendmsg(
-                isc.config.ccsession.create_command('show'), 'Stats')
+                isc.config.ccsession.create_command('show', param), 'Stats')
             (answer, env) = self.cc_session.group_recvmsg(False, seq)
             if answer:
                 (rcode, value) = isc.config.ccsession.parse_answer(answer)
@@ -353,9 +375,16 @@ class StatsHttpd:
             else:
                 raise StatsHttpdError("Stats module: %s" % str(value))
 
-    def get_stats_spec(self):
+    def get_stats_spec(self, owner=None, name=None):
         """Requests statistics data to the Stats daemon and returns
-        the data which obtains from it"""
+        the data which obtains from it. args are owner and name."""
+        param = {}
+        if owner is None and name is None:
+            param = None
+        if owner is not None:
+            param['owner'] = owner
+        if name is not None:
+            param['name'] = name
         try:
             seq = self.cc_session.group_sendmsg(
                 isc.config.ccsession.create_command('showschema'), 'Stats')
@@ -371,40 +400,82 @@ class StatsHttpd:
             raise StatsHttpdError("%s: %s" %
                                   (err.__class__.__name__, err))
 
-    def xml_handler(self):
+    def stats_data2xml(self, stats_spec, stats_data, xml_elem):
+        """Reads stats_data and stats_spec specified as first and
+        second arguments, and modify the xml object specified as
+        fourth argument. xml_elem must be modified and always returns
+        None."""
+        # assumed started with module_spec or started with item_spec in statistics
+        if type(stats_spec) is dict:
+            # assumed started with module_spec
+            if 'item_name' not in stats_spec \
+                    and 'item_type' not in stats_spec:
+                for module_name in stats_spec.keys():
+                    elem = xml.etree.ElementTree.Element(module_name)
+                    self.stats_data2xml(stats_spec[module_name],
+                                        stats_data[module_name], elem)
+                    xml_elem.append(elem)
+            # started with item_spec in statistics
+            else:
+                if stats_spec['item_type'] == 'map':
+                    elem = xml.etree.ElementTree.Element(stats_spec['item_name'])
+                    self.stats_data2xml(stats_spec['map_item_spec'],
+                                        stats_data,
+                                        elem)
+                    xml_elem.append(elem)
+                elif stats_spec['item_type'] == 'list':
+                    elem = xml.etree.ElementTree.Element(stats_spec['item_name'])
+                    for item in stats_data:
+                        self.stats_data2xml(stats_spec['list_item_spec'],
+                                            item,
+                                            elem)
+                    xml_elem.append(elem)
+                else:
+                    elem = xml.etree.ElementTree.Element(stats_spec['item_name'])
+                    elem.text = str(stats_data)
+                    xml_elem.append(elem)
+        # assumed started with stats_spec
+        elif type(stats_spec) is list:
+            for item_spec in stats_spec:
+                #elem = xml.etree.ElementTree.Element(item_spec['item_name'])
+                self.stats_data2xml(item_spec,
+                                    stats_data[item_spec['item_name']],
+                                    xml_elem)
+                #xml_elem.append(elem)
+        else:
+            xml_elem.text = str(stats_data)
+        return None
+
+    def xml_handler(self, module_name=None, item_name=None):
         """Handler which requests to Stats daemon to obtain statistics
         data and returns the body of XML document"""
-        xml_list=[]
-        for (mod, spec) in self.get_stats_data().items():
-            if not spec: continue
-            elem1 = xml.etree.ElementTree.Element(str(mod))
-            for (k, v) in spec.items():
-                elem2 = xml.etree.ElementTree.Element(str(k))
-                elem2.text = str(v)
-                elem1.append(elem2)
-            # The coding conversion is tricky. xml..tostring() of Python 3.2
-            # returns bytes (not string) regardless of the coding, while
-            # tostring() of Python 3.1 returns a string.  To support both
-            # cases transparently, we first make sure tostring() returns
-            # bytes by specifying utf-8 and then convert the result to a
-            # plain string (code below assume it).
-            xml_list.append(
-                str(xml.etree.ElementTree.tostring(elem1, encoding='utf-8'),
-                    encoding='us-ascii'))
-        xml_string = "".join(xml_list)
+        stats_spec = self.get_stats_spec(module_name, item_name)
+        stats_data = self.get_stats_data(module_name, item_name)
+        xml_elem = xml.etree.ElementTree.Element(
+            'bind10:statistics',
+            attrib={ 'xsi:schemaLocation' : XSD_NAMESPACE + ' ' + XSD_URL_PATH,
+                     'xmlns:bind10' : XSD_NAMESPACE,
+                     'xmlns:xsi' : "http://www.w3.org/2001/XMLSchema-instance" })
+        self.stats_data2xml(stats_spec, stats_data, xml_elem)
+        # The coding conversion is tricky. xml..tostring() of Python 3.2
+        # returns bytes (not string) regardless of the coding, while
+        # tostring() of Python 3.1 returns a string.  To support both
+        # cases transparently, we first make sure tostring() returns
+        # bytes by specifying utf-8 and then convert the result to a
+        # plain string (code below assume it).
+        xml_string = str(xml.etree.ElementTree.tostring(xml_elem, encoding='utf-8'),
+                         encoding='us-ascii')
         self.xml_body = self.open_template(XML_TEMPLATE_LOCATION).substitute(
             xml_string=xml_string,
-            xsd_namespace=XSD_NAMESPACE,
-            xsd_url_path=XSD_URL_PATH,
             xsl_url_path=XSL_URL_PATH)
         assert self.xml_body is not None
         return self.xml_body
 
-    def xsd_handler(self):
+    def xsd_handler(self, module_name=None, item_name=None):
         """Handler which just returns the body of XSD document"""
         # for XSD
         xsd_root = xml.etree.ElementTree.Element("all") # started with "all" tag
-        for (mod, spec) in self.get_stats_spec().items():
+        for (mod, spec) in self.get_stats_spec(module_name, item_name).items():
             if not spec: continue
             alltag = xml.etree.ElementTree.Element("all")
             for item in spec:
@@ -445,13 +516,13 @@ class StatsHttpd:
         assert self.xsd_body is not None
         return self.xsd_body
 
-    def xsl_handler(self):
+    def xsl_handler(self, module_name=None, item_name=None):
         """Handler which just returns the body of XSL document"""
         # for XSL
         xsd_root = xml.etree.ElementTree.Element(
             "xsl:template",
             dict(match="*")) # started with xml:template tag
-        for (mod, spec) in self.get_stats_spec().items():
+        for (mod, spec) in self.get_stats_spec(module_name, item_name).items():
             if not spec: continue
             for item in spec:
                 tr = xml.etree.ElementTree.Element("tr")

src/bin/stats/tests/Makefile.am

@@ -1,5 +1,6 @@
 PYCOVERAGE_RUN = @PYCOVERAGE_RUN@
 PYTESTS = b10-stats_test.py b10-stats-httpd_test.py
+PYTESTS = b10-stats-httpd_test.py
 EXTRA_DIST = $(PYTESTS) test_utils.py
 CLEANFILES = test_utils.pyc msgq_socket_test
 

src/bin/stats/tests/b10-stats-httpd_test.py

@@ -46,7 +46,7 @@ DUMMY_DATA = {
     'Auth' : {
         "queries.tcp": 2,
         "queries.udp": 3,
-        "queries.per-zone": [{
+        "queries.perzone": [{
                 "zonename": "test.example",
                 "queries.tcp": 2,
                 "queries.udp": 3
@@ -142,6 +142,23 @@ class TestHttpHandler(unittest.TestCase):
         self.assertTrue(int(response.getheader("Content-Length")) > 0)
         self.assertEqual(response.status, 200)
         root = xml.etree.ElementTree.parse(response).getroot()
+        self.assertTrue(root.tag.find('statistics') > 0)
+        for (k,v) in root.attrib.items():
+            if k.find('schemaLocation') > 0:
+                self.assertEqual(v, stats_httpd.XSD_NAMESPACE + ' ' + stats_httpd.XSD_URL_PATH)
+        for mod in DUMMY_DATA:
+            for (item, value) in DUMMY_DATA[mod].items():
+                self.assertIsNotNone(root.find(mod + '/' + item))
+
+        """
+        # URL is '/bind10/statistics/xml/Auth/queries.tcp/'
+        self.client.putrequest('GET', stats_httpd.XML_URL_PATH + '/Auth/queries.tcp/')
+        self.client.endheaders()
+        response = self.client.getresponse()
+        self.assertEqual(response.getheader("Content-type"), "text/xml")
+        self.assertTrue(int(response.getheader("Content-Length")) > 0)
+        self.assertEqual(response.status, 200)
+        root = xml.etree.ElementTree.parse(response).getroot()
         self.assertTrue(root.tag.find('stats_data') > 0)
         for (k,v) in root.attrib.items():
             if k.find('schemaLocation') > 0:
@@ -149,6 +166,7 @@ class TestHttpHandler(unittest.TestCase):
         for mod in DUMMY_DATA:
             for (item, value) in DUMMY_DATA[mod].items():
                 self.assertIsNotNone(root.find(mod + '/' + item))
+        """
 
         # URL is '/bind10/statitics/xsd'
         self.client.putrequest('GET', stats_httpd.XSD_URL_PATH)
@@ -169,6 +187,27 @@ class TestHttpHandler(unittest.TestCase):
         for elm in root.findall(xsdpath):
             self.assertIsNotNone(elm.attrib['name'])
             self.assertTrue(elm.attrib['name'] in DUMMY_DATA)
+        """
+        # URL is '/bind10/statitics/xsd/Auth/queries.tcp/'
+        self.client.putrequest('GET', stats_httpd.XSD_URL_PATH + '/Auth/queries.tcp/')
+        self.client.endheaders()
+        response = self.client.getresponse()
+        self.assertEqual(response.getheader("Content-type"), "text/xml")
+        self.assertTrue(int(response.getheader("Content-Length")) > 0)
+        self.assertEqual(response.status, 200)
+        root = xml.etree.ElementTree.parse(response).getroot()
+        url_xmlschema = '{http://www.w3.org/2001/XMLSchema}'
+        tags = [ url_xmlschema + t for t in [ 'element', 'complexType', 'all', 'element' ] ]
+        xsdpath = '/'.join(tags)
+        self.assertTrue(root.tag.find('schema') > 0)
+        self.assertTrue(hasattr(root, 'attrib'))
+        self.assertTrue('targetNamespace' in root.attrib)
+        self.assertEqual(root.attrib['targetNamespace'],
+                         stats_httpd.XSD_NAMESPACE)
+        for elm in root.findall(xsdpath):
+            self.assertIsNotNone(elm.attrib['name'])
+            self.assertTrue(elm.attrib['name'] in DUMMY_DATA)
+        """
 
         # URL is '/bind10/statitics/xsl'
         self.client.putrequest('GET', stats_httpd.XSL_URL_PATH)
@@ -197,6 +236,35 @@ class TestHttpHandler(unittest.TestCase):
             self.assertTrue(valueof.attrib['select'] in \
                                 [ tds[0].text+'/'+item for item in DUMMY_DATA[tds[0].text].keys() ])
 
+        """
+        # URL is '/bind10/statitics/xsl/Auth/queries.tcp/'
+        self.client.putrequest('GET', stats_httpd.XSL_URL_PATH + '/Auth/queries.tcp/')
+        self.client.endheaders()
+        response = self.client.getresponse()
+        self.assertEqual(response.getheader("Content-type"), "text/xml")
+        self.assertTrue(int(response.getheader("Content-Length")) > 0)
+        self.assertEqual(response.status, 200)
+        root = xml.etree.ElementTree.parse(response).getroot()
+        url_trans = '{http://www.w3.org/1999/XSL/Transform}'
+        url_xhtml = '{http://www.w3.org/1999/xhtml}'
+        xslpath = url_trans + 'template/' + url_xhtml + 'tr'
+        self.assertEqual(root.tag, url_trans + 'stylesheet')
+        for tr in root.findall(xslpath):
+            tds = tr.findall(url_xhtml + 'td')
+            self.assertIsNotNone(tds)
+            self.assertEqual(type(tds), list)
+            self.assertTrue(len(tds) > 2)
+            self.assertTrue(hasattr(tds[0], 'text'))
+            self.assertTrue(tds[0].text in DUMMY_DATA)
+            valueof = tds[2].find(url_trans + 'value-of')
+            self.assertIsNotNone(valueof)
+            self.assertTrue(hasattr(valueof, 'attrib'))
+            self.assertIsNotNone(valueof.attrib)
+            self.assertTrue('select' in valueof.attrib)
+            self.assertTrue(valueof.attrib['select'] in \
+                                [ tds[0].text+'/'+item for item in DUMMY_DATA[tds[0].text].keys() ])
+        """
+
         # 302 redirect
         self.client._http_vsn_str = 'HTTP/1.1'
         self.client.putrequest('GET', '/')
@@ -493,8 +561,6 @@ class TestStatsHttpd(unittest.TestCase):
         self.assertTrue(isinstance(tmpl, string.Template))
         opts = dict(
             xml_string="<dummy></dummy>",
-            xsd_namespace="http://host/path/to/",
-            xsd_url_path="/path/to/",
             xsl_url_path="/path/to/")
         lines = tmpl.substitute(opts)
         for n in opts:
@@ -585,26 +651,32 @@ class TestStatsHttpd(unittest.TestCase):
 
     def test_xml_handler(self):
         self.stats_httpd = MyStatsHttpd(get_availaddr())
-        self.stats_httpd.get_stats_data = lambda: \
+        self.stats_httpd.get_stats_spec = lambda x,y: \
+            { 'Dummy' : [ {
+                'item_name' : 'foo',
+                'item_type' : 'string' } ] }
+        self.stats_httpd.get_stats_data = lambda x,y: \
             { 'Dummy' : { 'foo':'bar' } }
         xml_body1 = self.stats_httpd.open_template(
             stats_httpd.XML_TEMPLATE_LOCATION).substitute(
-            xml_string='<Dummy><foo>bar</foo></Dummy>',
-            xsd_namespace=stats_httpd.XSD_NAMESPACE,
-            xsd_url_path=stats_httpd.XSD_URL_PATH,
+            xml_string='<bind10:statistics xmlns:bind10="http://bind10.isc.org/bind10/statistics/xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://bind10.isc.org/bind10/statistics/xsd /bind10/statistics/xsd"><Dummy><foo>bar</foo></Dummy></bind10:statistics>',
             xsl_url_path=stats_httpd.XSL_URL_PATH)
         xml_body2 = self.stats_httpd.xml_handler()
         self.assertEqual(type(xml_body1), str)
         self.assertEqual(type(xml_body2), str)
         self.assertEqual(xml_body1, xml_body2)
-        self.stats_httpd.get_stats_data = lambda: \
+        self.stats_httpd.get_stats_spec = lambda x,y: \
+            { 'Dummy' : [ {
+                'item_name' : 'bar',
+                'item_type' : 'string' } ] }
+        self.stats_httpd.get_stats_data = lambda x,y: \
             { 'Dummy' : {'bar':'foo'} }
         xml_body2 = self.stats_httpd.xml_handler()
         self.assertNotEqual(xml_body1, xml_body2)
 
     def test_xsd_handler(self):
         self.stats_httpd = MyStatsHttpd(get_availaddr())
-        self.stats_httpd.get_stats_spec = lambda: \
+        self.stats_httpd.get_stats_spec = lambda x,y: \
             { "Dummy" :
                   [{
                         "item_name": "foo",
@@ -629,7 +701,7 @@ class TestStatsHttpd(unittest.TestCase):
         self.assertEqual(type(xsd_body1), str)
         self.assertEqual(type(xsd_body2), str)
         self.assertEqual(xsd_body1, xsd_body2)
-        self.stats_httpd.get_stats_spec = lambda: \
+        self.stats_httpd.get_stats_spec = lambda x,y: \
             { "Dummy" :
                   [{
                         "item_name": "bar",
@@ -645,7 +717,7 @@ class TestStatsHttpd(unittest.TestCase):
 
     def test_xsl_handler(self):
         self.stats_httpd = MyStatsHttpd(get_availaddr())
-        self.stats_httpd.get_stats_spec = lambda: \
+        self.stats_httpd.get_stats_spec = lambda x,y: \
             { "Dummy" :
                   [{
                         "item_name": "foo",
@@ -668,7 +740,7 @@ class TestStatsHttpd(unittest.TestCase):
         self.assertEqual(type(xsl_body1), str)
         self.assertEqual(type(xsl_body2), str)
         self.assertEqual(xsl_body1, xsl_body2)
-        self.stats_httpd.get_stats_spec = lambda: \
+        self.stats_httpd.get_stats_spec = lambda x,y: \
             { "Dummy" :
                   [{
                         "item_name": "bar",

src/bin/stats/tests/test_utils.py

@@ -234,14 +234,25 @@ class MockAuth:
         "item_description": "A number of total query counts which all auth servers receive over UDP since they started initially"
       },
       {
-        "item_name": "queries.per-zone",
+        "item_name": "queries.perzone",
         "item_type": "list",
         "item_optional": false,
-        "item_default": [],
+        "item_default": [
+          {
+            "zonename" : "test1.example",
+            "queries.udp" : 1,
+            "queries.tcp" : 2
+          },
+          {
+            "zonename" : "test2.example",
+            "queries.udp" : 3,
+            "queries.tcp" : 4
+          }
+        ],
         "item_title": "Queries per zone",
         "item_description": "Queries per zone",
         "list_item_spec": {
-          "item_name": "item",
+          "item_name": "zones",
           "item_type": "map",
           "item_optional": false,
           "item_default": {},
@@ -290,7 +301,7 @@ class MockAuth:
         self.queries_tcp = 3
         self.queries_udp = 2
         self.queries_per_zone = [{
-                'zonename': 'test.example',
+                'zonename': 'test1.example',
                 'queries.tcp': 5,
                 'queries.udp': 4
                 }]