[trac930] modify Stats

 - remove unneeded subject and listener classes

 - add StatsError for handling errors in Stats

 - add some new methods (update_modules, update_statistics_data and
   get_statistics_data)

 - modify implementations of existent commands(show and set) according changes
   stats.spec

 - remove reset and remove command because stats module couldn't manage other
   modules' statistics data schema

 - add implementation of strict validation of each statistics data
   (If the validation is failed, it puts out the error.)

 - stats module shows its PID when status command invoked

 - add new command showschema invokable via bindctl

 - set command requires arguments of owner module name and statistics item name

 - show and showschema commands accepts arguments of owner module name and
   statistics item name

 - exits at exit code 1 if got runtime errors

 - has boot time in _BASETIME

src/bin/stats/stats.py.in

@@ -15,16 +15,17 @@
 # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
 # WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
+"""
+Statistics daemon in BIND 10
+
+"""
 import sys; sys.path.append ('@@PYTHONPATH@@')
 import os
-import signal
-import select
 from time import time, strftime, gmtime
 from optparse import OptionParser, OptionValueError
-from collections import defaultdict
-from isc.config.ccsession import ModuleCCSession, create_answer
-from isc.cc import Session, SessionError
 
+import isc
+import isc.util.process
 import isc.log
 from isc.log_messages.stats_messages import *
 
@@ -35,183 +36,111 @@ logger = isc.log.Logger("stats")
 # have #1074
 DBG_STATS_MESSAGING = 30
 
+# This is for boot_time of Stats
+_BASETIME = gmtime()
+
 # for setproctitle
-import isc.util.process
 isc.util.process.rename()
 
 # If B10_FROM_SOURCE is set in the environment, we use data files
 # from a directory relative to that, otherwise we use the ones
 # installed on the system
 if "B10_FROM_SOURCE" in os.environ:
-    BASE_LOCATION = os.environ["B10_FROM_SOURCE"] + os.sep + \
-        "src" + os.sep + "bin" + os.sep + "stats"
+    SPECFILE_LOCATION = os.environ["B10_FROM_SOURCE"] + os.sep + \
+        "src" + os.sep + "bin" + os.sep + "stats" + os.sep + "stats.spec"
 else:
     PREFIX = "@prefix@"
     DATAROOTDIR = "@datarootdir@"
-    BASE_LOCATION = "@datadir@" + os.sep + "@PACKAGE@"
-    BASE_LOCATION = BASE_LOCATION.replace("${datarootdir}", DATAROOTDIR).replace("${prefix}", PREFIX)
-SPECFILE_LOCATION = BASE_LOCATION + os.sep + "stats.spec"
-SCHEMA_SPECFILE_LOCATION = BASE_LOCATION + os.sep + "stats-schema.spec"
+    SPECFILE_LOCATION = "@datadir@" + os.sep + "@PACKAGE@" + os.sep + "stats.spec"
+    SPECFILE_LOCATION = SPECFILE_LOCATION.replace("${datarootdir}", DATAROOTDIR)\
+        .replace("${prefix}", PREFIX)
 
-class Singleton(type):
+def get_timestamp():
     """
-    A abstract class of singleton pattern
+    get current timestamp
     """
-    # Because of singleton pattern: 
-    #   At the beginning of coding, one UNIX domain socket is needed
-    #  for config manager, another socket is needed for stats module,
-    #  then stats module might need two sockets. So I adopted the
-    #  singleton pattern because I avoid creating multiple sockets in
-    #  one stats module. But in the initial version stats module
-    #  reports only via bindctl, so just one socket is needed. To use
-    #  the singleton pattern is not important now. :(
-
-    def __init__(self, *args, **kwargs):
-        type.__init__(self, *args, **kwargs)
-        self._instances = {}
-
-    def __call__(self, *args, **kwargs):
-        if args not in self._instances:
-            self._instances[args]={}
-        kw = tuple(kwargs.items())
-        if  kw not in self._instances[args]:
-            self._instances[args][kw] = type.__call__(self, *args, **kwargs)
-        return self._instances[args][kw]
+    return time()
 
-class Callback():
+def get_datetime(gmt=None):
     """
-    A Callback handler class
+    get current datetime
     """
-    def __init__(self, name=None, callback=None, args=(), kwargs={}):
-        self.name = name
-        self.callback = callback
-        self.args = args
-        self.kwargs = kwargs
+    if not gmt: gmt = gmtime()
+    return strftime("%Y-%m-%dT%H:%M:%SZ", gmt)
 
-    def __call__(self, *args, **kwargs):
-        if not args:
-            args = self.args
-        if not kwargs:
-            kwargs = self.kwargs
-        if self.callback:
-            return self.callback(*args, **kwargs)
-
-class Subject():
+def parse_spec(spec):
     """
-    A abstract subject class of observer pattern
+    parse spec type data
     """
-    # Because of observer pattern:
-    #   In the initial release, I'm also sure that observer pattern
-    #  isn't definitely needed because the interface between gathering
-    #  and reporting statistics data is single.  However in the future
-    #  release, the interfaces may be multiple, that is, multiple
-    #  listeners may be needed. For example, one interface, which
-    #  stats module has, is for between ''config manager'' and stats
-    #  module, another interface is for between ''HTTP server'' and
-    #  stats module, and one more interface is for between ''SNMP
-    #  server'' and stats module. So by considering that stats module
-    #  needs multiple interfaces in the future release, I adopted the
-    #  observer pattern in stats module. But I don't have concrete
-    #  ideas in case of multiple listener currently.
-
-    def __init__(self):
-        self._listeners = []
-
-    def attach(self, listener):
-        if not listener in self._listeners:
-            self._listeners.append(listener)
-
-    def detach(self, listener):
-        try:
-            self._listeners.remove(listener)
-        except ValueError:
-            pass
-
-    def notify(self, event, modifier=None):
-        for listener in self._listeners:
-            if modifier != listener:
-                listener.update(event)
+    def _parse_spec(spec):
+        item_type = spec['item_type']
+        if item_type == "integer":
+            return int(spec.get('item_default', 0))
+        elif item_type == "real":
+            return float(spec.get('item_default', 0.0))
+        elif item_type == "boolean":
+            return bool(spec.get('item_default', False))
+        elif item_type == "string":
+            return str(spec.get('item_default', ""))
+        elif item_type == "list":
+            return spec.get(
+                    "item_default",
+                    [ _parse_spec(s) for s in spec["list_item_spec"] ])
+        elif item_type == "map":
+            return spec.get(
+                    "item_default",
+                    dict([ (s["item_name"], _parse_spec(s)) for s in spec["map_item_spec"] ]) )
+        else:
+            return spec.get("item_default", None)
+    return dict([ (s['item_name'], _parse_spec(s)) for s in spec ])
 
-class Listener():
+class Callback():
     """
-    A abstract listener class of observer pattern
+    A Callback handler class
     """
-    def __init__(self, subject):
-        self.subject = subject
-        self.subject.attach(self)
-        self.events = {}
+    def __init__(self, command=None, args=(), kwargs={}):
+        self.command = command
+        self.args = args
+        self.kwargs = kwargs
 
-    def update(self, name):
-        if name in self.events:
-            callback = self.events[name]
-            return callback()
+    def __call__(self, *args, **kwargs):
+        if not args: args = self.args
+        if not kwargs: kwargs = self.kwargs
+        if self.command: return self.command(*args, **kwargs)
 
-    def add_event(self, event):
-        self.events[event.name]=event
+class StatsError(Exception):
+    """Exception class for Stats class"""
+    pass
 
-class SessionSubject(Subject, metaclass=Singleton):
+class Stats:
     """
-    A concrete subject class which creates CC session object
+    Main class of stats module
     """
-    def __init__(self, session=None):
-        Subject.__init__(self)
-        self.session=session
-        self.running = False
-
-    def start(self):
-        self.running = True
-        self.notify('start')
-
-    def stop(self):
+    def __init__(self):
         self.running = False
-        self.notify('stop')
-
-    def check(self):
-        self.notify('check')
-
-class CCSessionListener(Listener):
-    """
-    A concrete listener class which creates SessionSubject object and
-    ModuleCCSession object
-    """
-    def __init__(self, subject):
-        Listener.__init__(self, subject)
-        self.session = subject.session
-        self.boot_time = get_datetime()
-
         # create ModuleCCSession object
-        self.cc_session = ModuleCCSession(SPECFILE_LOCATION,
-                                          self.config_handler,
-                                          self.command_handler,
-                                          self.session)
-
-        self.session = self.subject.session = self.cc_session._session
-
-        # initialize internal data
-        self.stats_spec = isc.config.module_spec_from_file(SCHEMA_SPECFILE_LOCATION).get_config_spec()
-        self.stats_data = self.initialize_data(self.stats_spec)
-
-        # add event handler invoked via SessionSubject object
-        self.add_event(Callback('start', self.start))
-        self.add_event(Callback('stop', self.stop))
-        self.add_event(Callback('check', self.check))
-        # don't add 'command_' suffix to the special commands in
-        # order to prevent executing internal command via bindctl
-
+        self.mccs = isc.config.ModuleCCSession(SPECFILE_LOCATION,
+                                               self.config_handler,
+                                               self.command_handler)
+        self.cc_session = self.mccs._session
+        # get module spec
+        self.module_name = self.mccs.get_module_spec().get_module_name()
+        self.modules = {}
+        self.statistics_data = {}
         # get commands spec
-        self.commands_spec = self.cc_session.get_module_spec().get_commands_spec()
-
+        self.commands_spec = self.mccs.get_module_spec().get_commands_spec()
         # add event handler related command_handler of ModuleCCSession
-        # invoked via bindctl
+        self.callbacks = {}
         for cmd in self.commands_spec:
+            # add prefix "command_"
+            name = "command_" + cmd["command_name"]
             try:
-                # add prefix "command_"
-                name = "command_" + cmd["command_name"]
                 callback = getattr(self, name)
-                kwargs = self.initialize_data(cmd["command_args"])
-                self.add_event(Callback(name=name, callback=callback, args=(), kwargs=kwargs))
-            except AttributeError as ae:
-                logger.error(STATS_UNKNOWN_COMMAND_IN_SPEC, cmd["command_name"])
+                kwargs = parse_spec(cmd["command_args"])
+                self.callbacks[name] = Callback(command=callback, kwargs=kwargs)
+            except AttributeError:
+                raise StatsError(STATS_UNKNOWN_COMMAND_IN_SPEC, cmd["command_name"])
+        self.mccs.start()
 
     def _update_stats_data(self, args):
         # 'args' must be dictionary type
@@ -223,38 +152,30 @@ class CCSessionListener(Listener):
 
     def start(self):
         """
-        start the cc chanel
+        Start stats module
         """
-        # set initial value
-        self.stats_data['stats.boot_time'] = self.boot_time
-        self.stats_data['stats.start_time'] = get_datetime()
-        self.stats_data['stats.last_update_time'] = get_datetime()
-        self.stats_data['stats.lname'] = self.session.lname
-        self.cc_session.start()
+        self.running = True
+        # TODO: should be added into new logging interface
+        # if self.verbose:
+        #     sys.stdout.write("[b10-stats] starting\n")
+
         # request Bob to send statistics data
         logger.debug(DBG_STATS_MESSAGING, STATS_SEND_REQUEST_BOSS)
-        cmd = isc.config.ccsession.create_command("getstats", None)
-        seq = self.session.group_sendmsg(cmd, 'Boss')
-        try:
-            answer, env = self.session.group_recvmsg(False, seq)
-            if answer:
-                rcode, arg = isc.config.ccsession.parse_answer(answer)
-                if rcode == 0:
-                    self._update_stats_data(arg)
-        except isc.cc.session.SessionTimeout:
-            pass
-
-    def stop(self):
-        """
-        stop the cc chanel
-        """
-        return self.cc_session.close()
-
-    def check(self):
-        """
-        check the cc chanel
-        """
-        return self.cc_session.check_command(False)
+        cmd = isc.config.ccsession.create_command("sendstats", None)
+        seq = self.cc_session.group_sendmsg(cmd, 'Boss')
+        self.cc_session.group_recvmsg(True, seq)
+
+        # initialized Statistics data
+        errors = self.update_statistics_data(
+            self.module_name,
+            lname=self.cc_session.lname,
+            boot_time=get_datetime(_BASETIME)
+            )
+        if errors:
+            raise StatsError("stats spec file is incorrect")
+
+        while self.running:
+            self.mccs.check_command(False)
 
     def config_handler(self, new_config):
         """
@@ -262,169 +183,200 @@ class CCSessionListener(Listener):
         """
         logger.debug(DBG_STATS_MESSAGING, STATS_RECEIVED_NEW_CONFIG,
                      new_config)
-
         # do nothing currently
-        return create_answer(0)
+        return isc.config.create_answer(0)
 
-    def command_handler(self, command, *args, **kwargs):
+    def command_handler(self, command, kwargs):
         """
         handle commands from the cc channel
         """
-        # add 'command_' suffix in order to executing command via bindctl
         name = 'command_' + command
-        
-        if name in self.events:
-            event = self.events[name]
-            return event(*args, **kwargs)
+        if name in self.callbacks:
+            callback = self.callbacks[name]
+            if kwargs:
+                return callback(**kwargs)
+            else:
+                return callback()
         else:
-            return self.command_unknown(command, args)
+            logger.error(STATS_RECEIVED_UNKNOWN_COMMAND, command)
+            return isc.config.create_answer(1, "Unknown command: '"+str(command)+"'")
 
-    def command_shutdown(self, args):
+    def update_modules(self):
         """
-        handle shutdown command
+        update information of each module
         """
-        logger.info(STATS_RECEIVED_SHUTDOWN_COMMAND)
-        self.subject.running = False
-        return create_answer(0)
-
-    def command_set(self, args, stats_data={}):
+        modules = {}
+        seq = self.cc_session.group_sendmsg(
+            isc.config.ccsession.create_command(
+                isc.config.ccsession.COMMAND_GET_STATISTICS_SPEC),
+            'ConfigManager')
+        (answer, env) = self.cc_session.group_recvmsg(False, seq)
+        if answer:
+            (rcode, value) = isc.config.ccsession.parse_answer(answer)
+            if rcode == 0:
+                for mod in value:
+                    spec = { "module_name" : mod,
+                             "statistics"  : [] }
+                    if value[mod] and type(value[mod]) is list:
+                        spec["statistics"] = value[mod]
+                    modules[mod] = isc.config.module_spec.ModuleSpec(spec)
+        modules[self.module_name] = self.mccs.get_module_spec()
+        self.modules = modules
+
+    def get_statistics_data(self, owner=None, name=None):
         """
-        handle set command
+        return statistics data which stats module has of each module
         """
-        self._update_stats_data(args)
-        return create_answer(0)
+        self.update_statistics_data()
+        if owner and name:
+            try:
+                return self.statistics_data[owner][name]
+            except KeyError:
+                pass
+        elif owner:
+            try:
+                return self.statistics_data[owner]
+            except KeyError:
+                pass
+        elif name:
+            pass
+        else:
+            return self.statistics_data
 
-    def command_remove(self, args, stats_item_name=''):
+    def update_statistics_data(self, owner=None, **data):
         """
-        handle remove command
+        change statistics date of specified module into specified data
         """
-
-        # 'args' must be dictionary type
-        if args and args['stats_item_name'] in self.stats_data:
-            stats_item_name = args['stats_item_name']
-
-        logger.debug(DBG_STATS_MESSAGING, STATS_RECEIVED_REMOVE_COMMAND,
-                     stats_item_name)
-
-        # just remove one item
-        self.stats_data.pop(stats_item_name)
-
-        return create_answer(0)
-
-    def command_show(self, args, stats_item_name=''):
+        self.update_modules()
+        statistics_data = {}
+        for (name, module) in self.modules.items():
+            value = parse_spec(module.get_statistics_spec())
+            if module.validate_statistics(True, value):
+                statistics_data[name] = value
+        for (name, value) in self.statistics_data.items():
+            if name in statistics_data:
+                statistics_data[name].update(value)
+            else:
+                statistics_data[name] = value
+        self.statistics_data = statistics_data
+        if owner and data:
+            errors = []
+            try:
+                if self.modules[owner].validate_statistics(False, data, errors):
+                    self.statistics_data[owner].update(data)
+                    return
+            except KeyError:
+                errors.append('unknown module name')
+            return errors
+
+    def command_status(self):
         """
-        handle show command
+        handle status command
         """
+        logger.debug(DBG_STATS_MESSAGING, STATS_RECEIVED_STATUS_COMMAND)
+        return isc.config.create_answer(
+            0, "Stats is up. (PID " + str(os.getpid()) + ")")
 
-        # always overwrite 'report_time' and 'stats.timestamp'
-        # if "show" command invoked
-        self.stats_data['report_time'] = get_datetime()
-        self.stats_data['stats.timestamp'] = get_timestamp()
-
-        # if with args
-        if args and args['stats_item_name'] in self.stats_data:
-            stats_item_name = args['stats_item_name']
-            logger.debug(DBG_STATS_MESSAGING,
-                         STATS_RECEIVED_SHOW_NAME_COMMAND,
-                         stats_item_name)
-            return create_answer(0, {stats_item_name: self.stats_data[stats_item_name]})
-
-        logger.debug(DBG_STATS_MESSAGING,
-                     STATS_RECEIVED_SHOW_ALL_COMMAND)
-        return create_answer(0, self.stats_data)
-
-    def command_reset(self, args):
+    def command_shutdown(self):
         """
-        handle reset command
+        handle shutdown command
         """
-        logger.debug(DBG_STATS_MESSAGING,
-                     STATS_RECEIVED_RESET_COMMAND)
-
-        # re-initialize internal variables
-        self.stats_data = self.initialize_data(self.stats_spec)
-
-        # reset initial value
-        self.stats_data['stats.boot_time'] = self.boot_time
-        self.stats_data['stats.start_time'] = get_datetime()
-        self.stats_data['stats.last_update_time'] = get_datetime()
-        self.stats_data['stats.lname'] = self.session.lname
-
-        return create_answer(0)
+        logger.info(STATS_RECEIVED_SHUTDOWN_COMMAND)
+        self.running = False
+        return isc.config.create_answer(0)
 
-    def command_status(self, args):
+    def command_show(self, owner=None, name=None):
         """
-        handle status command
+        handle show command
         """
-        logger.debug(DBG_STATS_MESSAGING, STATS_RECEIVED_STATUS_COMMAND)
-        # just return "I'm alive."
-        return create_answer(0, "I'm alive.")
+        if (owner or name):
+            logger.debug(DBG_STATS_MESSAGING,
+                         STATS_RECEIVED_SHOW_NAME_COMMAND,
+                         str(owner)+", "+str(name))
+        else:
+            logger.debug(DBG_STATS_MESSAGING,
+                         STATS_RECEIVED_SHOW_ALL_COMMAND)
+        if owner and not name:
+            return isc.config.create_answer(1, "item name is not specified")
+        errors = self.update_statistics_data(
+            self.module_name,
+            timestamp=get_timestamp(),
+            report_time=get_datetime()
+            )
+        if errors: raise StatsError("stats spec file is incorrect")
+        ret = self.get_statistics_data(owner, name)
+        if ret:
+            return isc.config.create_answer(0, ret)
+        else:
+            return isc.config.create_answer(
+                1, "specified module name and/or item name are incorrect")
 
-    def command_unknown(self, command, args):
+    def command_showschema(self, owner=None, name=None):
         """
-        handle an unknown command
+        handle show command
         """
-        logger.error(STATS_RECEIVED_UNKNOWN_COMMAND, command)
-        return create_answer(1, "Unknown command: '"+str(command)+"'")
-
+        # TODO: should be added into new logging interface
+        # if self.verbose:
+        #     sys.stdout.write("[b10-stats] 'showschema' command received\n")
+        self.update_modules()
+        schema = {}
+        schema_byname = {}
+        for mod in self.modules:
+            spec = self.modules[mod].get_statistics_spec()
+            schema_byname[mod] = {}
+            if spec:
+                schema[mod] = spec
+                for item in spec:
+                    schema_byname[mod][item['item_name']] = item
+        if owner:
+            try:
+                if name:
+                    return isc.config.create_answer(0, schema_byname[owner][name])
+                else:
+                    return isc.config.create_answer(0, schema[owner])
+            except KeyError:
+                pass
+        else:
+            if name:
+                return isc.config.create_answer(1, "module name is not specified")
+            else:
+                return isc.config.create_answer(0, schema)
+        return isc.config.create_answer(
+                1, "specified module name and/or item name are incorrect")
 
-    def initialize_data(self, spec):
+    def command_set(self, owner, data):
         """
-        initialize stats data
+        handle set command
         """
-        def __get_init_val(spec):
-            if spec['item_type'] == 'null':
-                return None
-            elif spec['item_type'] == 'boolean':
-                return bool(spec.get('item_default', False))
-            elif spec['item_type'] == 'string':
-                return str(spec.get('item_default', ''))
-            elif spec['item_type'] in set(['number', 'integer']):
-                return int(spec.get('item_default', 0))
-            elif spec['item_type'] in set(['float', 'double', 'real']):
-                return float(spec.get('item_default', 0.0))
-            elif spec['item_type'] in set(['list', 'array']):
-                return spec.get('item_default',
-                                [ __get_init_val(s) for s in spec['list_item_spec'] ])
-            elif spec['item_type'] in set(['map', 'object']):
-                return spec.get('item_default',
-                                dict([ (s['item_name'], __get_init_val(s)) for s in spec['map_item_spec'] ]) )
-            else:
-                return spec.get('item_default')
-        return dict([ (s['item_name'], __get_init_val(s)) for s in spec ])
+        errors = self.update_statistics_data(owner, **data)
+        if errors:
+            return isc.config.create_answer(
+                1,
+                "specified module name and/or statistics data are incorrect: "
+                + ", ".join(errors))
+        errors = self.update_statistics_data(
+            self.module_name, last_update_time=get_datetime() )
+        if errors:
+            raise StatsError("stats spec file is incorrect")
+        return isc.config.create_answer(0)
 
-def get_timestamp():
-    """
-    get current timestamp
-    """
-    return time()
-
-def get_datetime():
-    """
-    get current datetime
-    """
-    return strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())
-
-def main(session=None):
+if __name__ == "__main__":
     try:
         parser = OptionParser()
-        parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
-                      help="display more about what is going on")
+        parser.add_option(
+            "-v", "--verbose", dest="verbose", action="store_true",
+            help="display more about what is going on")
         (options, args) = parser.parse_args()
         if options.verbose:
             isc.log.init("b10-stats", "DEBUG", 99)
-        subject = SessionSubject(session=session)
-        listener = CCSessionListener(subject)
-        subject.start()
-        while subject.running:
-            subject.check()
-        subject.stop()
-
+        stats = Stats()
+        stats.start()
     except OptionValueError as ove:
         logger.fatal(STATS_BAD_OPTION_VALUE, ove)
     except SessionError as se:
         logger.fatal(STATS_CC_SESSION_ERROR, se)
+    # TODO: should be added into new logging interface
+    except StatsError as se:
+        sys.exit("[b10-stats] %s" % se)
     except KeyboardInterrupt as kie:
         logger.info(STATS_STOPPED_BY_KEYBOARD)
-
-if __name__ == "__main__":
-    main()