# Copyright (C) 2012-2013 Internet Systems Consortium. # # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. # # THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM # DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL # INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT, # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING # FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION # WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """BIND 10 statistics counters module This module handles the statistics counters for BIND 10 modules. For using the module `counter.py`, first a counters object should be created in each module (like b10-xfrin or b10-xfrout) after importing this module. A spec file can be specified as an argument when creating the counters object: from isc.statistics import Counters self.counters = Counters("/path/to/foo.spec") The first argument of Counters() can be specified, which is the location of the specification file (like src/bin/xfrout/xfrout.spec). If Counters is constructed this way, statistics counters can be accessed from each module. For example, in case that the item `xfrreqdone` is defined in statistics_spec in xfrout.spec, the following methods are callable. Since these methods require the string of the zone name in the first argument, if we have the following code in b10-xfrout: self.counters.inc('zones', zone_name, 'xfrreqdone') then the counter for xfrreqdone corresponding to zone_name is incremented. For getting the current number of this counter, we can use the following code: number = self.counters.get('zones', zone_name, 'xfrreqdone') then the current count is obtained and set in the variable `number`. Such a getter method would be mainly used for unit-testing. As other example, for the item `axfr_running`, the decrementer method is also callable. This method is used for decrementing a counter. For the item `axfr_running`, an argument like zone name is not required: self.counters.dec('axfr_running') These methods are effective in other modules. For example, in case that this module `counter.py` is once imported in a main module such as b10-xfrout, then for the item `notifyoutv4`, the `inc()` method can be invoked in another module such as notify_out.py, which is firstly imported in the main module. self.counters.inc('zones', zone_name, 'notifyoutv4') In this example this is for incrementing the counter of the item `notifyoutv4`. Thus, such statement can be also written in another library like isc.notify.notify_out. If this module `counter.py` isn't imported in the main module but imported in such a library module as isc.notify.notify_out, in this example, empty methods would be invoked, which is directly defined in `counter.py`. """ import threading import isc.config from datetime import datetime # static internal functions def _add_counter(element, spec, identifier): """Returns value of the identifier if the identifier is in the element. Otherwise, sets a default value from the spec and returns it. If the top-level type of the identifier is named_set and the second-level type is map, it sets a set of default values under the level and then returns the default value of the identifier. This method raises DataNotFoundError if the element is invalid for spec.""" try: return isc.cc.data.find(element, identifier) except isc.cc.data.DataNotFoundError: pass # Note: If there is a named_set type item in the statistics spec # and if there are map type items under it, all of items under the # map type item need to be added. For example, we're assuming that # this method is now adding a counter whose identifier is like # dir1/dir2/dir3/counter1. If both of dir1 and dir2 are named_set # types, and if dir3 is a map type, and if counter1, counter2, and # counter3 are defined as items under dir3 by the statistics spec, # this method would add other two counters: # # dir1/dir2/dir3/counter2 # dir1/dir2/dir3/counter3 # # Otherwise this method just adds the only counter # dir1/dir2/dir3/counter1. # examine spec from the top-level item and know whether # has_named_set, and check whether spec and identifier are correct pidr = '' has_named_set = False for idr in identifier.split('/'): if len(pidr) > 0: idr = pidr + '/' + idr spec_ = isc.config.find_spec_part(spec, idr) if isc.config.spec_part_is_named_set(spec_): has_named_set = True break pidr = idr # add all elements in map type if has_named_set has_map = False if has_named_set: p_idr = identifier.rsplit('/', 1)[0] p_spec = isc.config.find_spec_part(spec, p_idr) if isc.config.spec_part_is_map(p_spec): has_map = True for name in isc.config.spec_name_list(p_spec['map_item_spec']): idr_ = p_idr + '/' + name spc_ = isc.config.find_spec_part(spec, idr_) isc.cc.data.set(element, idr_, spc_['item_default']) # otherwise add a specific element if not has_map: spec_ = isc.config.find_spec_part(spec, identifier) isc.cc.data.set(element, identifier, spec_['item_default']) return isc.cc.data.find(element, identifier) def _set_counter(element, spec, identifier, value): """Invokes _add_counter() for checking whether the identifier is in the element. If not, it creates a new identifier in the element and set the default value from the spec. After that, it sets the value specified in the arguments.""" _add_counter(element, spec, identifier) isc.cc.data.set(element, identifier, value) def _get_counter(element, identifier): """Returns the value of the identifier in the element""" return isc.cc.data.find(element, identifier) def _inc_counter(element, spec, identifier, step=1): """Increments the value of the identifier in the element to the step from the current value. If the identifier isn't in the element, it creates a new identifier in the element.""" isc.cc.data.set(element, identifier, _add_counter(element, spec, identifier) + step) def _start_timer(): """Returns the current datetime as a datetime object.""" return datetime.now() def _stop_timer(start_time, element, spec, identifier): """Sets duration time in seconds as a value of the identifier in the element, which is in seconds between start_time and the current time and is float-type.""" delta = datetime.now() - start_time # FIXME: The following statement can be replaced by: # sec = delta.total_seconds() # but total_seconds() is not available in Python 3.1. Please update # this code when we depend on Python 3.2. sec = round(delta.days * 86400 + delta.seconds + \ delta.microseconds * 1E-6, 6) _set_counter(element, spec, identifier, sec) def _concat(*args, sep='/'): """A helper function that is used to generate an identifier for statistics item names. It concatenates words in args with a separator('/') """ return sep.join(args) class _Statistics(): """Statistics data set""" # default statistics data _data = {} # default statistics spec used in case the specfile is omitted when # constructing a Counters() object _spec = [ { "item_name": "zones", "item_type": "named_set", "item_optional": False, "item_default": { "_SERVER_" : { "notifyoutv4" : 0, "notifyoutv6" : 0 } }, "item_title": "Zone names", "item_description": "Zone names", "named_set_item_spec": { "item_name": "classname", "item_type": "named_set", "item_optional": False, "item_default": {}, "item_title": "RR class name", "item_description": "RR class name", "named_set_item_spec": { "item_name": "zonename", "item_type": "map", "item_optional": False, "item_default": {}, "item_title": "Zone name", "item_description": "Zone name", "map_item_spec": [ { "item_name": "notifyoutv4", "item_type": "integer", "item_optional": False, "item_default": 0, "item_title": "IPv4 notifies", "item_description": "Number of IPv4 notifies per zone name sent out" }, { "item_name": "notifyoutv6", "item_type": "integer", "item_optional": False, "item_default": 0, "item_title": "IPv6 notifies", "item_description": "Number of IPv6 notifies per zone name sent out" } ] } } } ] class Counters(): """A class for holding and manipulating all statistics counters for a module. A Counters object may be created by specifying a spec file of the module in argument. According to statistics specification in the spec file, a counter value can be incremented, decremented or obtained. Methods such as inc(), dec() and get() are useful for this. Counters objects also have timer functionality. The timer can be started and stopped, and the duration between start and stop can be obtained. Methods such as start_timer(), stop_timer() and get() are useful for this. Saved counters can be cleared by the method clear_all(). Manipulating counters and timers can be temporarily disabled. If disabled, counter values are not changed even if methods to update them are invoked. Including per-zone counters, a list of counters which can be handled in the class are like the following: zones/IN/example.com./notifyoutv4 zones/IN/example.com./notifyoutv6 zones/IN/example.com./xfrrej zones/IN/example.com./xfrreqdone zones/IN/example.com./soaoutv4 zones/IN/example.com./soaoutv6 zones/IN/example.com./axfrreqv4 zones/IN/example.com./axfrreqv6 zones/IN/example.com./ixfrreqv4 zones/IN/example.com./ixfrreqv6 zones/IN/example.com./xfrsuccess zones/IN/example.com./xfrfail zones/IN/example.com./last_ixfr_duration zones/IN/example.com./last_axfr_duration ixfr_running axfr_running socket/unixdomain/open socket/unixdomain/openfail socket/unixdomain/close socket/unixdomain/bindfail socket/unixdomain/acceptfail socket/unixdomain/accept socket/unixdomain/senderr socket/unixdomain/recverr socket/ipv4/tcp/open socket/ipv4/tcp/openfail socket/ipv4/tcp/close socket/ipv4/tcp/connfail socket/ipv4/tcp/conn socket/ipv4/tcp/senderr socket/ipv4/tcp/recverr socket/ipv6/tcp/open socket/ipv6/tcp/openfail socket/ipv6/tcp/close socket/ipv6/tcp/connfail socket/ipv6/tcp/conn socket/ipv6/tcp/senderr socket/ipv6/tcp/recverr """ # '_SERVER_' is a special zone name representing an entire # count. It doesn't mean a specific zone, but it means an # entire count in the server. _entire_server = '_SERVER_' # zone names are contained under this dirname in the spec file. _perzone_prefix = 'zones' # default statistics data set _statistics = _Statistics() def __init__(self, spec_file_name=None): """A constructor for the Counters class. A path to the spec file can be specified in spec_file_name. Statistics data based on statistics spec can be accumulated if spec_file_name is specified. If omitted, a default statistics spec is used. The default statistics spec is defined in a hidden class named _Statistics(). """ self._zones_item_list = [] self._start_time = {} self._disabled = False self._rlock = threading.RLock() if not spec_file_name: return # change the default statistics spec self._statistics._spec = \ isc.config.module_spec_from_file(spec_file_name).\ get_statistics_spec() if self._perzone_prefix in \ isc.config.spec_name_list(self._statistics._spec): self._zones_item_list = isc.config.spec_name_list( isc.config.find_spec_part( self._statistics._spec, '%s/%s/%s' % (self._perzone_prefix, '_CLASS_', self._entire_server))) def clear_all(self): """clears all statistics data""" with self._rlock: self._statistics._data = {} def disable(self): """disables incrementing/decrementing counters""" with self._rlock: self._disabled = True def enable(self): """enables incrementing/decrementing counters""" with self._rlock: self._disabled = False def _incdec(self, *args, step=1): """A common helper function for incrementing or decrementing a counter. It acquires a lock to support multi-threaded use. isc.cc.data.DataNotFoundError is raised when incrementing the counter of the item undefined in the spec file.""" identifier = _concat(*args) with self._rlock: if self._disabled: return _inc_counter(self._statistics._data, self._statistics._spec, identifier, step) def inc(self, *args): """An incrementer for a counter. It acquires a lock to support multi-threaded use. isc.cc.data.DataNotFoundError is raised when incrementing the counter of the item undefined in the spec file.""" return self._incdec(*args) def dec(self, *args): """A decrementer for a counter. It acquires a lock to support multi-threaded use. isc.cc.data.DataNotFoundError is raised when decrementing the counter of the item undefined in the spec file.""" return self._incdec(*args, step=-1) def get(self, *args): """A getter method for counters. It returns the current number of the specified counter. isc.cc.data.DataNotFoundError is raised when the counter doesn't have a number yet.""" identifier = _concat(*args) return _get_counter(self._statistics._data, identifier) def start_timer(self, *args): """Starts a timer which is identified by args and keeps it running until stop_timer() is called. It acquires a lock to support multi-threaded use. If the specified timer is already started but not yet stopped, the last start time is overwritten.""" identifier = _concat(*args) with self._rlock: if self._disabled: return isc.cc.data.set(self._start_time, identifier, _start_timer()) def stop_timer(self, *args): """Stops a timer which is identified by args. It acquires a lock to support multi-threaded use. If the timer isn't started by start_timer() yet, it raises no exception. However if args aren't defined in the spec file, it raises DataNotFoundError. """ identifier = _concat(*args) with self._rlock: if self._disabled: return try: start_time = isc.cc.data.find(self._start_time, identifier) except isc.cc.data.DataNotFoundError: # do not set the end time if the timer isn't started return # set the end time _stop_timer( start_time, self._statistics._data, self._statistics._spec, identifier) # A datetime value of once used timer should be deleted # for a future use. # Here, names of branch and leaf are obtained from a # string of identifier. The branch name is equivalent to # the position of datetime to be deleted and the leaf name # is equivalent to the value of datetime to be deleted. (branch, leaf) = identifier.rsplit('/', 1) # Then map of branch is obtained from self._start_time by # using isc.cc.data.find(). branch_map = isc.cc.data.find(self._start_time, branch) # Finally a value of the leaf name is deleted from the # map. del branch_map[leaf] def get_statistics(self): """Calculates an entire server's counts, and returns statistics data in a format to send out to the stats module, including each counter. If nothing is counted yet, then it returns an empty dictionary.""" # entire copy statistics_data = self._statistics._data.copy() # If there is no 'zones' found in statistics_data, # i.e. statistics_data contains no per-zone counter, it just # returns statistics_data because calculating total counts # across the zone names isn't necessary. if self._perzone_prefix not in statistics_data: return statistics_data zones = statistics_data[self._perzone_prefix] # Start calculation for '_SERVER_' counts zones_spec = isc.config.find_spec_part(self._statistics._spec, self._perzone_prefix) zones_data = {} for cls in zones.keys(): for zone in zones[cls].keys(): for (attr, val) in zones[cls][zone].items(): id_str1 = '%s/%s/%s' % (cls, zone, attr) id_str2 = '%s/%s/%s' % (cls, self._entire_server, attr) _set_counter(zones_data, zones_spec, id_str1, val) _inc_counter(zones_data, zones_spec, id_str2, val) # insert entire-server counts statistics_data[self._perzone_prefix] = dict( statistics_data[self._perzone_prefix], **zones_data) return statistics_data