[2225] introduced new counter classes and implemented unixsocket counters

 - implemented a base class Counter and a concrete XfroutCounter as an external
   module under isc.statistics. Because it is easy to implement another
   concrete Counter class for other module in future. The caller module can
   statically import it. The new counter class provides a getter method for
   each statistics item. It is intended for making loose relationship between
   the counter class and the caller module.

 - added implementation of unixsocket counter into the existing UnixSockServer
   Class.

 - added new tests for checking counters implemented in UnixSockServer are
   working properly into xfrout_test.py.

 - implemented enabling/disabling counting in the new counter class.

configure.ac

@@ -1251,6 +1251,8 @@ AC_CONFIG_FILES([Makefile
                  src/lib/python/isc/server_common/tests/Makefile
                  src/lib/python/isc/sysinfo/Makefile
                  src/lib/python/isc/sysinfo/tests/Makefile
+                 src/lib/python/isc/statistics/Makefile
+                 src/lib/python/isc/statistics/tests/Makefile
                  src/lib/config/Makefile
                  src/lib/config/tests/Makefile
                  src/lib/config/tests/testdata/Makefile

src/lib/python/isc/Makefile.am

@@ -1,5 +1,5 @@
 SUBDIRS = datasrc cc config dns log net notify util testutils acl bind10
-SUBDIRS += xfrin log_messages server_common ddns sysinfo
+SUBDIRS += xfrin log_messages server_common ddns sysinfo statistics
 
 python_PYTHON = __init__.py
 

src/lib/python/isc/statistics/Makefile.am

@@ -0,0 +1,9 @@
+SUBDIRS = . tests
+
+python_PYTHON = __init__.py counter.py
+pythondir = $(pyexecdir)/isc/statistics
+
+CLEANDIRS = __pycache__
+
+clean-local:
+	rm -rf $(CLEANDIRS)

src/lib/python/isc/statistics/__init__.py

@@ -0,0 +1 @@
+from isc.statistics.counter import *

src/lib/python/isc/statistics/counter.py

@@ -0,0 +1,300 @@
+# Copyright (C) 2012  Internet Systems Consortium.
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
+# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
+# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+'''Statistics counter countainer for modules'''
+
+import threading
+import isc.config
+
+_COUNTER = None
+
+def init(spec_file_name):
+    """A creator method for a counter class. It creates a counter
+    object by the module name of the given spec file. An argument is a
+    specification file name."""
+    module_spec = isc.config.module_spec_from_file(spec_file_name)
+    class_name = '%sCounter' % module_spec.get_module_name()
+    global _COUNTER
+    _COUNTER = globals()[class_name](module_spec)
+
+# These method are dummies for notify_out in case XfroutCounter is not
+# loaded.
+def inc_notifyoutv4(self, arg): pass
+def inc_notifyoutv6(self, arg): pass
+
+class Counter():
+    """A basic counter class for concrete classes"""
+    _statistics_spec = {}
+    _statistics_data = {}
+    _disabled = False
+    _rlock = threading.RLock()
+
+    def __init__(self, module_spec):
+        self._statistics_spec = module_spec.get_statistics_spec()
+        global clear_counters
+        global disable
+        global enable
+        clear_counters = self.clear_counters
+        enable = self.enable
+        disable = self.disable
+
+    def clear_counters(self):
+        """clears all statistics data"""
+        with self._rlock:
+            self._statistics_data = {}
+
+    def disable(self):
+        """disables incrementing/decrementing counters"""
+        self._disabled = True
+
+    def enable(self):
+        """enables incrementing/decrementing counters"""
+        self._disabled = False
+
+class XfroutCounter(Counter):
+    """A module for holding all statistics counters of Xfrout. The
+    counter numbers can be accessed by the accesseers defined
+    according to a spec file. In this class, the structure of per-zone
+    counters is assumed to be like this:
+
+        zones/example.com./notifyoutv4
+        zones/example.com./notifyoutv6
+        zones/example.com./xfrrej
+        zones/example.com./xfrreqdone
+        ixfr_running
+        axfr_running
+        socket/unixdomain/open
+        socket/unixdomain/openfail
+        socket/unixdomain/close
+        socket/unixdomain/bindfail
+        socket/unixdomain/acceptfail
+        socket/unixdomain/accept
+        socket/unixdomain/senderr
+        socket/unixdomain/recverr
+    """
+
+    # '_SERVER_' is a special zone name representing an entire
+    # count. It doesn't mean a specific zone, but it means an
+    # entire count in the server.
+    _entire_server = '_SERVER_'
+    # zone names are contained under this dirname in the spec file.
+    _perzone_prefix = 'zones'
+    _xfrrunning_names = []
+    _unixsocket_names = []
+
+    def __init__(self, module_spec):
+        Counter.__init__(self, module_spec)
+        self._xfrrunning_names = [ \
+            n for n in \
+                isc.config.spec_name_list(self._statistics_spec) \
+                if 'xfr_running' in n ]
+        self._unixsocket_names = [ \
+            n.split('/')[-1] for n in \
+                isc.config.spec_name_list(\
+                self._statistics_spec, "", True) \
+                if n.find('socket/unixdomain/') == 0 ]
+        self._create_perzone_functors()
+        self._create_xfrrunning_functors()
+        self._create_unixsocket_functors()
+        global dump_default_statistics
+        global dump_statistics
+        dump_default_statistics = self.dump_default_statistics
+        dump_statistics = self.dump_statistics
+
+    def _create_perzone_functors(self):
+        """Creates increment method of each per-zone counter based on
+        the spec file. Incrementer can be accessed by name
+        "inc_${item_name}".Incrementers are passed to the
+        XfroutSession and NotifyOut class as counter handlers."""
+        # add a new element under the named_set item for the zone
+        zones_spec = isc.config.find_spec_part(
+            self._statistics_spec, self._perzone_prefix)
+        item_list =  isc.config.spec_name_list(\
+            zones_spec['named_set_item_spec']['map_item_spec'])
+        # can be accessed by the name 'inc_xxx'
+        for item in item_list:
+            def __incrementer(zone_name, counter_name=item, step=1):
+                """A per-zone incrementer for counter_name. Locks the
+                thread because it is considered to be invoked by a
+                multi-threading caller."""
+                if self._disabled: return
+                with self._rlock:
+                    self._add_perzone_counter(zone_name)
+                    self._statistics_data[self._perzone_prefix]\
+                        [zone_name][counter_name] += step
+            def __getter(zone_name, counter_name=item):
+                """A getter method for perzone counters"""
+                return isc.cc.data.find(
+                    self._statistics_data,
+                    '%s/%s/%s' % ( self._perzone_prefix,
+                                   zone_name,
+                                   counter_name )
+                    )
+            globals()['inc_%s' % item] = __incrementer
+            globals()['get_%s' % item] = __getter
+
+    def _create_xfrrunning_functors(self):
+        """Creates increment/decrement method of (a|i)xfr_running
+        based on the spec file. Incrementer can be accessed by name
+        "inc_${item_name}". Decrementer can be accessed by name
+        "dec_${item_name}". Both of them are passed to the
+        XfroutSession as counter handlers."""
+        # can be accessed by the name 'inc_xxx' or 'dec_xxx'
+        for item in self._xfrrunning_names:
+            def __incrementer(counter_name=item, step=1):
+                """A incrementer for axfr or ixfr running. Locks the
+                thread because it is considered to be invoked by a
+                multi-threading caller."""
+                if self._disabled: return
+                with self._rlock:
+                    self._add_xfrrunning_counter(counter_name)
+                    self._statistics_data[counter_name] += step
+            def __decrementer(counter_name=item, step=-1):
+                """A decrementer for axfr or ixfr running. Locks the
+                thread because it is considered to be invoked by a
+                multi-threading caller."""
+                if self._disabled: return
+                with self._rlock:
+                    self._statistics_data[counter_name] += step
+            def __getter(counter_name=item):
+                """A getter method for xfr_running counters"""
+                return isc.cc.data.find(
+                        self._statistics_data, counter_name )
+            globals()['inc_%s' % item] = __incrementer
+            globals()['dec_%s' % item] = __decrementer
+            globals()['get_%s' % item] = __getter
+
+    def _create_unixsocket_functors(self):
+        """Creates increment/decrement method of (a|i)xfr_running
+        based on the spec file. Incrementer can be accessed by name
+        "inc_${item_name}". Decrementer can be accessed by name
+        "dec_${item_name}". Both of them are passed to the
+        XfroutSession as counter handlers."""
+        # can be accessed by the name 'inc_xxx' or 'dec_xxx'
+        for item in self._unixsocket_names:
+            def __incrementer(counter_name=item, step=1):
+                """A incrementer for axfr or ixfr running. Locks the
+                thread because it is considered to be invoked by a
+                multi-threading caller."""
+                if self._disabled: return
+                with self._rlock:
+                    self._add_unixsocket_counter(counter_name)
+                    self._statistics_data['socket']['unixdomain']\
+                       [counter_name] += step
+            def __getter(counter_name=item):
+                """A getter method for unixsockets counters"""
+                return isc.cc.data.find(
+                    self._statistics_data,
+                    'socket/unixdomain/%s' % counter_name )
+            globals()['inc_unixsocket_%s' % item] = __incrementer
+            globals()['get_unixsocket_%s' % item] = __getter
+
+    def _add_perzone_counter(self, zone):
+        """Adds a named_set-type counter for each zone name."""
+        try:
+            self._statistics_data[self._perzone_prefix][zone]
+        except KeyError:
+            # add a new element under the named_set item for the zone
+            map_spec = isc.config.find_spec_part(
+                self._statistics_spec, '%s/%s' % \
+                    (self._perzone_prefix, zone))['map_item_spec']
+            id_list =  isc.config.spec_name_list(map_spec)
+            for id_ in id_list:
+                spec = isc.config.find_spec_part(map_spec, id_)
+                isc.cc.data.set(self._statistics_data,
+                                '%s/%s/%s' % \
+                                    (self._perzone_prefix, zone, id_),
+                                spec['item_default'])
+
+    def _add_xfrrunning_counter(self, counter_name):
+        """Adds a counter for counting (a|i)xfr_running"""
+        try:
+            self._statistics_data[counter_name]
+        except KeyError:
+            # examines the names of xfer running
+            for n in self._xfrrunning_names:
+                spec = isc.config.find_spec_part\
+                    (self._statistics_spec, n)
+                isc.cc.data.set(self._statistics_data, n, \
+                                spec['item_default'])
+
+    def _add_unixsocket_counter(self, counter_name):
+        """Adds a counter for counting unix sockets"""
+        try:
+            self._statistics_data['socket']['unixdomain'][counter_name]
+        except KeyError:
+            # examines the name of unixsocket
+            name = 'socket/unixdomain/%s' % counter_name
+            spec = isc.config.find_spec_part\
+                (self._statistics_spec, name)
+            isc.cc.data.set(self._statistics_data, name, \
+                                spec['item_default'])
+
+    def dump_default_statistics(self):
+        """Returns default statistics data from the spec file"""
+        statistics_data = {}
+        for id_ in isc.config.spec_name_list(self._statistics_spec):
+            spec = isc.config.find_spec_part(\
+                self._statistics_spec, id_)
+            if 'item_default' in spec:
+                statistics_data.update({id_: spec['item_default']})
+        return statistics_data
+
+    def dump_statistics(self):
+        """Calculates an entire server counts, and returns statistics
+        data format to send out the stats module including each
+        counter. If there is no counts, then it returns an empty
+        dictionary."""
+        # If self.statistics_data contains nothing of zone name, it
+        # returns an empty dict.
+        if len(self._statistics_data) == 0: return {}
+        # for per-zone counter
+        zones = self._statistics_data[self._perzone_prefix]
+        # Start calculation for '_SERVER_' counts
+        attrs = self.dump_default_statistics()\
+            [self._perzone_prefix][self._entire_server]
+        statistics_data = {self._perzone_prefix: {}}
+        for attr in attrs:
+            sum_ = 0
+            for name in zones:
+                if name == self._entire_server: continue
+                if attr in zones[name]:
+                    if  name not in \
+                            statistics_data[self._perzone_prefix]:
+                        statistics_data[self._perzone_prefix][name]\
+                            = {}
+                    statistics_data[self._perzone_prefix][name].\
+                        update({attr: zones[name][attr]})
+                    sum_ += zones[name][attr]
+            if  sum_ > 0:
+                if self._entire_server not in \
+                        statistics_data[self._perzone_prefix]:
+                    statistics_data[self._perzone_prefix][self._entire_server]\
+                        = {}
+                statistics_data[self._perzone_prefix][self._entire_server]\
+                    .update({attr:sum_})
+
+        # for xfrrunning incrementer/decrementer
+        for name in self._xfrrunning_names:
+            if name in self._statistics_data:
+                statistics_data[name] = self._statistics_data[name]
+
+        # for unixsocket incrementer/decrementer
+        if 'socket' in self._statistics_data:
+            statistics_data['socket'] = \
+                self._statistics_data['socket']
+
+        return statistics_data
+

src/lib/python/isc/statistics/tests/Makefile.am

@@ -0,0 +1,29 @@
+PYCOVERAGE_RUN=@PYCOVERAGE_RUN@
+PYTESTS = counter_test.py
+EXTRA_DIST = $(PYTESTS)
+
+# If necessary (rare cases), explicitly specify paths to dynamic libraries
+# required by loadable python modules.
+LIBRARY_PATH_PLACEHOLDER =
+if SET_ENV_LIBRARY_PATH
+LIBRARY_PATH_PLACEHOLDER += $(ENV_LIBRARY_PATH)=$(abs_top_builddir)/src/lib/cryptolink/.libs:$(abs_top_builddir)/src/lib/dns/.libs:$(abs_top_builddir)/src/lib/dns/python/.libs:$(abs_top_builddir)/src/lib/cc/.libs:$(abs_top_builddir)/src/lib/config/.libs:$(abs_top_builddir)/src/lib/log/.libs:$(abs_top_builddir)/src/lib/util/.libs:$(abs_top_builddir)/src/lib/exceptions/.libs:$(abs_top_builddir)/src/lib/datasrc/.libs:$$$(ENV_LIBRARY_PATH)
+else
+# Some systems need the ds path even if not all paths are necessary
+LIBRARY_PATH_PLACEHOLDER += $(ENV_LIBRARY_PATH)=$(abs_top_builddir)/src/lib/datasrc/.libs
+endif
+
+# test using command-line arguments, so use check-local target instead of TESTS
+check-local:
+if ENABLE_PYTHON_COVERAGE
+	touch $(abs_top_srcdir)/.coverage
+	rm -f .coverage
+	${LN_S} $(abs_top_srcdir)/.coverage .coverage
+endif
+	for pytest in $(PYTESTS) ; do \
+	echo Running test: $$pytest ; \
+	PYTHONPATH=$(COMMON_PYTHON_PATH):$(abs_top_builddir)/src/bin/xfrout:$(abs_top_builddir)/src/lib/dns/python/.libs \
+	$(LIBRARY_PATH_PLACEHOLDER) \
+	TESTDATASRCDIR=$(abs_top_srcdir)/src/lib/python/isc/statistics/tests/testdata/ \
+	B10_FROM_BUILD=$(abs_top_builddir) \
+	$(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \
+	done

src/lib/python/isc/statistics/tests/counter_test.py

@@ -0,0 +1,171 @@
+# Copyright (C) 2012  Internet Systems Consortium.
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
+# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
+# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+'''Tests for isc.statistics.counter'''
+
+import unittest
+import threading
+import isc.config
+import xfrout
+
+TEST_ZONE_NAME_STR = "example.com."
+
+from isc.statistics import counter
+
+class TestCounter(unittest.TestCase):
+
+    def setUp(self):
+        module_spec = isc.config.module_spec_from_file(\
+            xfrout.SPECFILE_LOCATION)
+        self.counter = counter.Counter(module_spec)
+
+    def test_clear_counters(self):
+        self.counter._statistics_data = {'counter': 1}
+        self.counter.clear_counters()
+        self.assertEqual(self.counter._statistics_data,
+                         {})
+
+    def test_enablediable(self):
+        self.assertFalse(self.counter._disabled)
+        self.counter.disable()
+        self.assertTrue(self.counter._disabled)
+        self.counter.enable()
+        self.assertFalse(self.counter._disabled)
+
+class TestXfroutCounter(unittest.TestCase):
+    _number = 3 # number of the threads
+    _cycle = 10000 # number of counting per thread
+
+    def setUp(self):
+        self._module_spec = isc.config.module_spec_from_file(\
+            xfrout.SPECFILE_LOCATION)
+        self._statistics_spec = \
+            self._module_spec.get_statistics_spec()
+        counter.init(xfrout.SPECFILE_LOCATION)
+        self.xfrout_counter = counter._COUNTER
+        self._entire_server    = self.xfrout_counter._entire_server
+        self._perzone_prefix   = self.xfrout_counter._perzone_prefix
+        self._xfrrunning_names = self.xfrout_counter._xfrrunning_names
+        self._unixsocket_names = self.xfrout_counter._unixsocket_names
+        self._started = threading.Event()
+
+    def test_dump_default_statistics(self):
+        self.assertTrue(\
+            self._module_spec.validate_statistics(\
+                True,
+                counter.dump_default_statistics(),
+                )
+            )
+
+    def setup_functor(self, incrementer, *args):
+        self._started.wait()
+        for i in range(self._cycle): incrementer(*args)
+
+    def start_functor(self, incrementer, *args):
+        threads = []
+        for i in range(self._number):
+            threads.append(threading.Thread(\
+                    target=self.setup_functor, \
+                        args=(incrementer,) + args \
+                        ))
+        for th in threads: th.start()
+        self._started.set()
+        for th in threads: th.join()
+
+    def get_count(self, zone_name, counter_name):
+        return isc.cc.data.find(\
+            counter.dump_statistics(),\
+                '%s/%s/%s' % (self._perzone_prefix,\
+                                  zone_name, counter_name))
+
+    def test_functors(self):
+        # for per-zone counters
+        result = { self._entire_server: {},
+                   TEST_ZONE_NAME_STR: {} }
+        self._perzone_counters = isc.config.spec_name_list(\
+            isc.config.find_spec_part(\
+                self._statistics_spec, self._perzone_prefix)\
+                ['named_set_item_spec']['map_item_spec'])
+        for counter_name in self._perzone_counters:
+            incrementer = getattr(counter,'inc_%s' % counter_name)
+            self.start_functor(incrementer, TEST_ZONE_NAME_STR)
+            getter = getattr(counter,'get_%s' % counter_name)
+            self.assertEqual(getter(TEST_ZONE_NAME_STR),
+                             self._number * self._cycle)
+            self.assertEqual(self.get_count(self._entire_server,
+                        counter_name), self._number * self._cycle)
+            # checks disable/enable
+            counter.disable()
+            incrementer(TEST_ZONE_NAME_STR)
+            self.assertEqual(getter(TEST_ZONE_NAME_STR),
+                             self._number * self._cycle)
+            counter.enable()
+            incrementer(TEST_ZONE_NAME_STR)
+            self.assertEqual(getter(TEST_ZONE_NAME_STR),
+                             self._number * self._cycle + 1)
+            result[self._entire_server][counter_name] = \
+                result[TEST_ZONE_NAME_STR][counter_name] = \
+                self._number * self._cycle + 1
+
+        statistics_data = {self._perzone_prefix: result}
+
+        # for {a|i}xfrrunning counters
+        for counter_name in self._xfrrunning_names:
+            incrementer = getattr(counter,'inc_%s' % counter_name)
+            self.start_functor(incrementer)
+            getter = getattr(counter,'get_%s' % counter_name)
+            self.assertEqual(getter(), self._number * self._cycle)
+            decrementer = getattr(counter,'dec_%s' % counter_name)
+            self.start_functor(decrementer)
+            self.assertEqual(getter(), 0)
+            # checks disable/enable
+            counter.disable()
+            incrementer()
+            self.assertEqual(getter(), 0)
+            counter.enable()
+            incrementer()
+            self.assertGreater(getter(), 0)
+            counter.disable()
+            decrementer()
+            self.assertGreater(getter(), 0)
+            counter.enable()
+            decrementer()
+            self.assertEqual(getter(), 0)
+            statistics_data[counter_name] = 0
+
+        # for unixsocket counters
+        statistics_data.update({'socket': {'unixdomain': {}}})
+        for counter_name in self._unixsocket_names:
+            incrementer = getattr(counter, 'inc_unixsocket_%s' % counter_name)
+            self.start_functor(incrementer)
+            getter = getattr(counter, 'get_unixsocket_%s' % counter_name)
+            self.assertEqual(getter(), self._number * self._cycle)
+            # checks disable/enable
+            counter.disable()
+            incrementer()
+            self.assertEqual(getter(), self._number * self._cycle)
+            counter.enable()
+            incrementer()
+            self.assertEqual(getter(), self._number * self._cycle + 1)
+            statistics_data['socket']['unixdomain'][counter_name] = \
+                self._number * self._cycle + 1
+
+        # totally chacking
+        self.assertEqual(counter.dump_statistics(), statistics_data)
+        self.assertTrue(self._module_spec.validate_statistics(\
+                True, statistics_data))
+
+if __name__== "__main__":
+    unittest.main()