[2964] added helper python module DataSrcClientsMgr.

src/lib/python/isc/server_common/Makefile.am

@@ -1,6 +1,7 @@
 SUBDIRS = tests
 
 python_PYTHON = __init__.py tsig_keyring.py auth_command.py dns_tcp.py
+python_PYTHON += datasrc_clients_mgr.py
 python_PYTHON += logger.py
 
 pythondir = $(pyexecdir)/isc/server_common

src/lib/python/isc/server_common/datasrc_clients_mgr.py

@@ -0,0 +1,127 @@
+# Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
+# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
+# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+import isc.dns
+import isc.datasrc
+import threading
+import json
+
+class ConfigError(Exception):
+    """Exception class raised for data source configuration errors."""
+    pass
+
+class DataSrcClientsMgr:
+    """A container of data source client lists.
+
+    This class represents a set of isc.datasrc.ConfigurableClientList
+    objects (currently per RR class), and provides APIs to configure
+    the lists and access to a specific list in a thread safe manner.
+
+    It is intended to be used by applications that refer to the global
+    'data_sources' module.  The reconfigure() method can be called from
+    a configuration callback for the module of the application.  The
+    get_client_list() is a simple search method to get the configured
+    ConfigurableClientList object for a specified RR class (if any),
+    while still allowing a separate thread to reconfigure the entire lists.
+
+    """
+    def __init__(self, use_cache=False):
+        """Constructor.
+
+        In the initial implementation, user applications of this class are
+        generally expected to NOT use in-memory cache; use_cache would be
+        set to True only for tests.  In future, some applications such as
+        outbound zone transfer may want to set it to True.
+
+        Parameter:
+          use_cache (bool): If set to True, enable in-memory cache on
+                            (re)configuration.
+
+        """
+        self.__use_cache = use_cache
+
+        # Map from RRClass to ConfigurableClientList.  Resetting this map
+        # is protected by __map_lock.  Note that this lock doesn't protect
+        # "updates" of the map content.  __clients_map shouldn't be accessed
+        # by class users directly.
+        self.__clients_map = {}
+        self.__map_lock = threading.Lock()
+
+    def get_client_list(self, rrclass):
+        """Return the configured ConfigurableClientList for the RR class.
+
+        If no client list is configured for the specified RR class, it
+        returns None.
+
+        This method should not raise an exception as long as the parameter
+        is of valid type.
+
+        This method can be safely called from a thread even if a different
+        thread is calling reconfigure().  Also, it's safe for the caller
+        to use the returned list even if reconfigure() is called while or
+        after the call to this thread.
+
+        Note that this class does not protect furtther access to the returned
+        list from multiple threads; it's the caller's responsbility to make
+        such access thread safe.  In general, the find() method on the list
+        and the use of ZoneFinder created by a DataSourceClient in the list
+        cannot be done by multiple threads without explicit synchronization.
+        On the other hand, multiple threads can create and use ZoneUpdater
+        or ZoneIterator on a DataSourceClient in parallel.
+
+        Parameter:
+          rrclass (isc.dns.RRClass): the RR class of the ConfigurableClientList
+            to be returned.
+        """
+        with self.__map_lock:
+            client_list = self.__clients_map.get(rrclass)
+        return client_list
+
+    def reconfigure(self, config):
+        """(Re)configure the set of client lists.
+
+        This method takes a new set of data source configuration, builds
+        a new set of ConfigurableClientList objects corresponding to the
+        configuration, and replaces the internal set with the newly built
+        one.  Its parameter is expected to be the "new configuration"
+        parameter of a configuration update callback for the global
+        "data_sources" module.  It should match the configuration data
+        of the module spec (see the datasrc.spec file).
+
+        Any error in reconfiguration is converted to a ConfigError
+        exception and is raised from the method.  This method guarantees
+        strong exception safety: unless building a new set for the new
+        configuration is fully completed, the old set is intact.
+
+        See the description of get_client_list() for thread considerations.
+
+        Parameter:
+          config (dict): configuration data for the data_sources module.
+
+        """
+        try:
+            new_map = {}
+            for rrclass_cfg, class_cfg in config.get('classes').items():
+                rrclass = isc.dns.RRClass(rrclass_cfg)
+                new_client_list = isc.datasrc.ConfigurableClientList(rrclass)
+                new_client_list.configure(json.dumps(class_cfg),
+                                          self.__use_cache)
+                new_map[rrclass] = new_client_list
+            with self.__map_lock:
+                self.__clients_map = new_map
+        except Exception as ex:
+            # Catch all types of exceptions as a whole: there won't be much
+            # granularity for exceptions raised from the C++ module anyway.
+            raise ConfigError(ex)

src/lib/python/isc/server_common/tests/Makefile.am

@@ -1,5 +1,5 @@
 PYCOVERAGE_RUN = @PYCOVERAGE_RUN@
-PYTESTS = tsig_keyring_test.py dns_tcp_test.py
+PYTESTS = tsig_keyring_test.py dns_tcp_test.py datasrc_clients_mgr_test.py
 EXTRA_DIST = $(PYTESTS)
 
 # If necessary (rare cases), explicitly specify paths to dynamic libraries
@@ -21,5 +21,6 @@ endif
 	$(LIBRARY_PATH_PLACEHOLDER) \
 	PYTHONPATH=$(COMMON_PYTHON_PATH):$(abs_top_builddir)/src/lib/dns/python/.libs \
 	B10_LOCKFILE_DIR_FROM_BUILD=$(abs_top_builddir) \
+	B10_FROM_BUILD=$(abs_top_builddir) \
 	$(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \
 	done

src/lib/python/isc/server_common/tests/datasrc_clients_mgr_test.py

@@ -0,0 +1,109 @@
+# Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
+# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
+# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+import isc.log
+from isc.server_common.datasrc_clients_mgr import *
+from isc.dns import *
+import unittest
+import isc.config
+import os
+
+DATASRC_SPECFILE = os.environ["B10_FROM_BUILD"] + \
+    "/src/bin/cfgmgr/plugins/datasrc.spec"
+DEFAULT_CONFIG = \
+    isc.config.ConfigData(isc.config.module_spec_from_file(DATASRC_SPECFILE)).\
+    get_full_config()
+
+class DataSrcClientsMgrTest(unittest.TestCase):
+    def setUp(self):
+        # We construct the manager with enabling in-memory cache for easier
+        # tests.  There should be no risk of inter-thread issues in the tests.
+        self.__mgr = DataSrcClientsMgr(use_cache=True)
+
+    def test_init(self):
+        """Check some initial state.
+
+        Initially there's no client list available, but get_client_list
+        doesn't cause disruption.
+        """
+        self.assertIsNone(self.__mgr.get_client_list(RRClass.IN))
+        self.assertIsNone(self.__mgr.get_client_list(RRClass.CH))
+
+    def test_reconfigure(self):
+        """Check configuration behavior.
+
+        First try the default configuration, and replace it with something
+        else.
+        """
+
+        # There should be at least in-memory only data for the static
+        # bind/CH zone. (We don't assume the existence of SQLite3 datasrc,
+        # so it'll still work if and when we make the default DB-independent).
+        self.__mgr.reconfigure(DEFAULT_CONFIG)
+        clist = self.__mgr.get_client_list(RRClass.CH)
+        self.assertIsNotNone(clist)
+        self.assertTrue(clist.find(Name('bind'), True, False)[2])
+
+        # Reconfigure it with a simple new config: the list for CH will be
+        # gone, and and an empty list for IN will be installed.
+        self.__mgr.reconfigure({"classes": {"IN": []}})
+        self.assertIsNone(self.__mgr.get_client_list(RRClass.CH))
+        self.assertIsNotNone(self.__mgr.get_client_list(RRClass.IN))
+
+    def test_reconfigure_error(self):
+        """Check reconfigure failure preserves the old config."""
+        # Configure it with the default
+        self.__mgr.reconfigure(DEFAULT_CONFIG)
+        self.assertIsNotNone(self.__mgr.get_client_list(RRClass.CH))
+
+        # Then try invalid configuration
+        self.assertRaises(ConfigError, self.__mgr.reconfigure, 42)
+        self.assertIsNotNone(self.__mgr.get_client_list(RRClass.CH))
+
+        # Another type of invalid configuration: exception would come from
+        # the C++ wrapper.
+        self.assertRaises(ConfigError,
+                          self.__mgr.reconfigure, {"classes": {"IN": 42}})
+        self.assertIsNotNone(self.__mgr.get_client_list(RRClass.CH))
+
+    def test_reconfig_while_using_old(self):
+        """Check datasrc client and finder can work even after list is gone."""
+        self.__mgr.reconfigure(DEFAULT_CONFIG)
+        clist = self.__mgr.get_client_list(RRClass.CH)
+        self.__mgr.reconfigure({"classes": {"IN": []}})
+
+        datasrc_client, finder, exact = clist.find(Name('bind'))
+        self.assertTrue(exact)
+
+        # Reset the client list
+        clist = None
+
+        # Both finder and datasrc client should still work without causing
+        # disruption.  We shouldn't have to inspect too much details of the
+        # returned values.
+        result, rrset, _ = finder.find(Name('bind'), RRType.SOA)
+        self.assertEqual(Name('bind'), rrset.get_name())
+        self.assertEqual(RRType.SOA, rrset.get_type())
+        self.assertEqual(RRClass.CH, rrset.get_class())
+        self.assertEqual(RRTTL(0), rrset.get_ttl())
+
+        # iterator should produce some non empty set of RRsets
+        rrsets = datasrc_client.get_iterator(Name('bind'))
+        self.assertNotEqual(0, len(list(rrsets)))
+
+if __name__ == "__main__":
+    isc.log.init("bind10")
+    isc.log.resetUnitTestRootLogger()
+    unittest.main()