[master] Merge branch 'trac2854'

configure.ac

@@ -1192,6 +1192,8 @@ AC_CONFIG_FILES([Makefile
                  src/bin/loadzone/Makefile
                  src/bin/loadzone/tests/Makefile
                  src/bin/loadzone/tests/correct/Makefile
+                 src/bin/memmgr/Makefile
+                 src/bin/memmgr/tests/Makefile
                  src/bin/msgq/Makefile
                  src/bin/msgq/tests/Makefile
                  src/bin/auth/Makefile
@@ -1264,6 +1266,8 @@ AC_CONFIG_FILES([Makefile
                  src/lib/python/isc/bind10/tests/Makefile
                  src/lib/python/isc/ddns/Makefile
                  src/lib/python/isc/ddns/tests/Makefile
+                 src/lib/python/isc/memmgr/Makefile
+                 src/lib/python/isc/memmgr/tests/Makefile
                  src/lib/python/isc/xfrin/Makefile
                  src/lib/python/isc/xfrin/tests/Makefile
                  src/lib/python/isc/server_common/Makefile
@@ -1377,6 +1381,8 @@ AC_OUTPUT([doc/version.ent
            src/bin/loadzone/loadzone.py
            src/bin/usermgr/run_b10-cmdctl-usermgr.sh
            src/bin/usermgr/b10-cmdctl-usermgr.py
+           src/bin/memmgr/memmgr.py
+           src/bin/memmgr/memmgr.spec.pre
            src/bin/msgq/msgq.py
            src/bin/msgq/run_msgq.sh
            src/bin/auth/auth.spec.pre
@@ -1395,6 +1401,7 @@ AC_OUTPUT([doc/version.ent
            src/lib/python/isc/notify/tests/notify_out_test
            src/lib/python/isc/log/tests/log_console.py
            src/lib/python/isc/log_messages/work/__init__.py
+           src/lib/python/isc/server_common/bind10_server.py
            src/lib/dns/gen-rdatacode.py
            src/lib/python/bind10_config.py
            src/lib/cc/session_config.h.pre

src/bin/Makefile.am

@@ -1,5 +1,5 @@
 SUBDIRS = bind10 bindctl cfgmgr ddns loadzone msgq cmdctl auth xfrin \
 	xfrout usermgr zonemgr stats tests resolver sockcreator dhcp4 dhcp6 d2\
-	dbutil sysinfo
+	dbutil sysinfo memmgr
 
 check-recursive: all-recursive

src/bin/bind10/run_bind10.sh.in

@@ -20,7 +20,7 @@ export PYTHON_EXEC
 
 BIND10_PATH=@abs_top_builddir@/src/bin/bind10
 
-PATH=@abs_top_builddir@/src/bin/msgq:@abs_top_builddir@/src/bin/auth:@abs_top_builddir@/src/bin/resolver:@abs_top_builddir@/src/bin/cfgmgr:@abs_top_builddir@/src/bin/cmdctl:@abs_top_builddir@/src/bin/stats:@abs_top_builddir@/src/bin/xfrin:@abs_top_builddir@/src/bin/xfrout:@abs_top_builddir@/src/bin/zonemgr:@abs_top_builddir@/src/bin/ddns:@abs_top_builddir@/src/bin/dhcp6:@abs_top_builddir@/src/bin/sockcreator:$PATH
+PATH=@abs_top_builddir@/src/bin/msgq:@abs_top_builddir@/src/bin/auth:@abs_top_builddir@/src/bin/resolver:@abs_top_builddir@/src/bin/cfgmgr:@abs_top_builddir@/src/bin/cmdctl:@abs_top_builddir@/src/bin/stats:@abs_top_builddir@/src/bin/xfrin:@abs_top_builddir@/src/bin/xfrout:@abs_top_builddir@/src/bin/zonemgr:@abs_top_builddir@/src/bin/ddns:@abs_top_builddir@/src/bin/dhcp6:@abs_top_builddir@/src/bin/sockcreator:@abs_top_builddir@/src/bin/memmgr:$PATH
 export PATH
 
 PYTHONPATH=@abs_top_builddir@/src/lib/python/isc/log_messages:@abs_top_builddir@/src/lib/python/isc/cc:@abs_top_builddir@/src/lib/python:@abs_top_builddir@/src/lib/dns/python/.libs:@abs_top_builddir@/src/lib/xfr/.libs:@abs_top_builddir@/src/lib/log/.libs:@abs_top_builddir@/src/lib/util/io/.libs:@abs_top_builddir@/src/lib/python/isc/config:@abs_top_builddir@/src/lib/python/isc/acl/.libs:@abs_top_builddir@/src/lib/python/isc/datasrc/.libs

src/bin/memmgr/.gitignore

@@ -0,0 +1,4 @@
+/b10-memmgr
+/memmgr.py
+/memmgr.spec
+/b10-memmgr.8

src/bin/memmgr/Makefile.am

@@ -0,0 +1,62 @@
+SUBDIRS = . tests
+
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+
+pkglibexec_SCRIPTS = b10-memmgr
+
+b10_memmgrdir = $(pkgdatadir)
+b10_memmgr_DATA = memmgr.spec
+
+nodist_pylogmessage_PYTHON = $(PYTHON_LOGMSGPKG_DIR)/work/memmgr_messages.py
+pylogmessagedir = $(pyexecdir)/isc/log_messages/
+
+CLEANFILES = b10-memmgr memmgr.pyc
+CLEANFILES += $(PYTHON_LOGMSGPKG_DIR)/work/memmgr_messages.py
+CLEANFILES += $(PYTHON_LOGMSGPKG_DIR)/work/memmgr_messages.pyc
+CLEANFILES += memmgr.spec
+
+EXTRA_DIST =  memmgr_messages.mes
+
+man_MANS = b10-memmgr.8
+DISTCLEANFILES = $(man_MANS)
+EXTRA_DIST += $(man_MANS) b10-memmgr.xml
+
+if GENERATE_DOCS
+
+b10-memmgr.8: b10-memmgr.xml
+	@XSLTPROC@ --novalid --xinclude --nonet -o $@ http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl $(srcdir)/b10-memmgr.xml
+
+else
+
+$(man_MANS):
+	@echo Man generation disabled.  Creating dummy $@.  Configure with --enable-generate-docs to enable it.
+	@echo Man generation disabled.  Remove this file, configure with --enable-generate-docs, and rebuild BIND 10 > $@
+
+endif
+
+# Define rule to build logging source files from message file
+$(PYTHON_LOGMSGPKG_DIR)/work/memmgr_messages.py : memmgr_messages.mes
+	$(top_builddir)/src/lib/log/compiler/message \
+	-d $(PYTHON_LOGMSGPKG_DIR)/work -p $(srcdir)/memmgr_messages.mes
+
+memmgr.spec: memmgr.spec.pre
+	$(SED) -e "s|@@LOCALSTATEDIR@@|$(localstatedir)|" memmgr.spec.pre > $@
+
+# this is done here since configure.ac AC_OUTPUT doesn't expand exec_prefix
+b10-memmgr: memmgr.py $(PYTHON_LOGMSGPKG_DIR)/work/memmgr_messages.py
+	$(SED) -e "s|@@PYTHONPATH@@|@pyexecdir@|" memmgr.py >$@
+	chmod a+x $@
+
+CLEANDIRS = __pycache__
+
+# install the default directory for memory-mapped files.  Note that the
+# path must be identical to the default value in memmgr.spec.  We'll make
+# it readable only for the owner to minimize the risk of accidents.
+install-data-local:
+	$(mkinstalldirs) $(DESTDIR)@localstatedir@/@PACKAGE@/mapped_files
+
+install-data-hook:
+	-chmod 700 $(DESTDIR)@localstatedir@/@PACKAGE@/mapped_files
+
+clean-local:
+	rm -rf $(CLEANDIRS)

src/bin/memmgr/b10-memmgr.xml

@@ -0,0 +1,109 @@
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
+               "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
+	       [<!ENTITY mdash "&#8212;">]>
+<!--
+ - Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+ -
+ - Permission to use, copy, modify, and/or distribute this software for any
+ - purpose with or without fee is hereby granted, provided that the above
+ - copyright notice and this permission notice appear in all copies.
+ -
+ - THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+ - REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ - AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+ - INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ - LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ - OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ - PERFORMANCE OF THIS SOFTWARE.
+-->
+
+<refentry>
+
+  <refentryinfo>
+    <date>June 11, 2013</date>
+  </refentryinfo>
+
+  <refmeta>
+    <refentrytitle>b10-memmgr</refentrytitle>
+    <manvolnum>8</manvolnum>
+    <refmiscinfo>BIND10</refmiscinfo>
+  </refmeta>
+
+  <refnamediv>
+    <refname>b10-memmgr</refname>
+    <refpurpose>BIND 10 memory manager daemon</refpurpose>
+  </refnamediv>
+
+  <docinfo>
+    <copyright>
+      <year>2013</year>
+      <holder>Internet Systems Consortium, Inc. ("ISC")</holder>
+    </copyright>
+  </docinfo>
+
+  <refsynopsisdiv>
+    <cmdsynopsis>
+      <command>b10-memmgr</command>
+    </cmdsynopsis>
+  </refsynopsisdiv>
+
+  <refsect1>
+    <title>DESCRIPTION</title>
+    <para>The <command>b10-memmgr</command> daemon manages shared
+      memory segments storing in-memory DNS zone data, and
+      communicates with other BIND 10 modules about the segment
+      information so the entire system will use these segments
+      in a consistent manner.
+    </para>
+  </refsect1>
+
+  <refsect1>
+    <title>ARGUMENTS</title>
+
+    <para>The <command>b10-memmgr</command> daemon does not take
+      any command line arguments.
+    </para>
+  </refsect1>
+
+  <refsect1>
+    <title>CONFIGURATION AND COMMANDS</title>
+    <para>
+      The configurable settings are:
+    </para>
+    <para>
+      <varname>mapped_file_dir</varname>
+      A path to store files to be mapped to memory.  This must be
+      writable to the <command>b10-memmgr</command> daemon.
+    </para>
+
+    <para>
+      The module commands are:
+    </para>
+    <para>
+      <command>shutdown</command> exits <command>b10-memmgr</command>.
+    </para>
+  </refsect1>
+
+
+  <refsect1>
+    <title>SEE ALSO</title>
+    <para>
+      <citerefentry>
+        <refentrytitle>bind10</refentrytitle><manvolnum>8</manvolnum>
+      </citerefentry>,
+      <citetitle>BIND 10 Guide</citetitle>.
+    </para>
+  </refsect1>
+
+  <refsect1>
+    <title>HISTORY</title>
+    <para>
+      The <command>b10-memmgr</command> daemon was first implemented
+      in 2013 for the ISC BIND 10 project.
+    </para>
+  </refsect1>
+</refentry><!--
+ - Local variables:
+ - mode: sgml
+ - End:
+-->

src/bin/memmgr/memmgr.py.in

@@ -0,0 +1,154 @@
+#!@PYTHON@
+
+# Copyright (C) 2013  Internet Systems Consortium.
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
+# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
+# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+import copy
+import os
+import sys
+import signal
+
+sys.path.append('@@PYTHONPATH@@')
+import isc.log
+from isc.config import ModuleSpecError, ModuleCCSessionError
+from isc.log_messages.memmgr_messages import *
+from isc.server_common.bind10_server import BIND10Server, BIND10ServerFatal
+from isc.server_common.datasrc_clients_mgr \
+    import DataSrcClientsMgr, ConfigError
+from isc.memmgr.datasrc_info import DataSrcInfo
+import isc.util.process
+
+MODULE_NAME = 'memmgr'
+
+isc.log.init('b10-memmgr', buffer=True)
+logger = isc.log.Logger(MODULE_NAME)
+
+isc.util.process.rename()
+
+class ConfigError(Exception):
+    """An exception class raised for configuration errors of Memmgr."""
+    pass
+
+class Memmgr(BIND10Server):
+    def __init__(self):
+        # Running configurable parameters: on initial configuration this will
+        # be a dict: str=>config_value.
+        # This is defined as "protected" so tests can inspect it; others
+        # shouldn't use it directly.
+        self._config_params = None
+
+        # The manager to keep track of data source configuration.  Allow
+        # tests to inspect/tweak it.
+        self._datasrc_clients_mgr = DataSrcClientsMgr(use_cache=True)
+
+        # List of DataSrcInfo.  Used as a queue to maintain info for all
+        # active configuration generations.  Allow tests to inspec it.
+        self._datasrc_info_list = []
+
+    def _config_handler(self, new_config):
+        """Configuration handler, called via BIND10Server.
+
+        This method must be exception free.  We assume minimum validity
+        about the parameter, though: it should be a valid dict, and conform
+        to the type specification of the spec file.
+
+        """
+        logger.debug(logger.DBGLVL_TRACE_BASIC, MEMMGR_CONFIG_UPDATE)
+
+        # Default answer:
+        answer = isc.config.create_answer(0)
+
+        # If this is the first time, initialize the local attributes with the
+        # latest full config data, which consist of the defaults with
+        # possibly overridden by user config.  Otherwise, just apply the latest
+        # diff.
+        if self._config_params is None:
+            new_config = self.mod_ccsession.get_full_config()
+        try:
+            self.__update_config(new_config)
+        except Exception as ex:
+            logger.error(MEMMGR_CONFIG_FAIL, ex)
+            answer = isc.config.create_answer(
+                1, 'Memmgr failed to apply configuration updates: ' + str(ex))
+
+        return answer
+
+    def __update_config(self, new_config):
+        """Apply config changes to local attributes.
+
+        This is a subroutine of _config_handler.  It's supposed to provide
+        strong exception guarantee: either all changes successfully apply
+        or, if any error is found, none applies.  In the latter case the
+        entire original configuration should be kept.
+
+        Errors are to be reported as an exception.
+
+        """
+        # If this is the first time, build everything from the scratch.
+        # Otherwise, make a full local copy and update it.
+        if self._config_params is None:
+            new_config_params = {}
+        else:
+            new_config_params = copy.deepcopy(self._config_params)
+
+        new_mapped_file_dir = new_config.get('mapped_file_dir')
+        if new_mapped_file_dir is not None:
+            if not os.path.isdir(new_mapped_file_dir):
+                raise ConfigError('mapped_file_dir is not a directory: ' +
+                                  new_mapped_file_dir)
+            if not os.access(new_mapped_file_dir, os.W_OK):
+                raise ConfigError('mapped_file_dir is not writable: ' +
+                                  new_mapped_file_dir)
+            new_config_params['mapped_file_dir'] = new_mapped_file_dir
+
+        # All copy, switch to the new configuration.
+        self._config_params = new_config_params
+
+    def _setup_module(self):
+        """Module specific initialization for BIND10Server."""
+        try:
+            # memmgr isn't usable if data source is not configured, and
+            # as long as cfgmgr is ready there's no timing issue.  So we
+            # immediately shut it down if it's missing.  See ddns.py.in
+            # about exceptions to catch.
+            self.mod_ccsession.add_remote_config_by_name(
+                'data_sources', self._datasrc_config_handler)
+        except (ModuleSpecError, ModuleCCSessionError) as ex:
+            logger.error(MEMMGR_NO_DATASRC_CONF, ex)
+            raise BIND10ServerFatal('failed to setup memmgr module')
+
+    def _datasrc_config_handler(self, new_config, config_data):
+        """Callback of data_sources configuration update.
+
+        This method must be exception free, so we catch all expected
+        exceptions internally; unexpected ones should mean a programming
+        error and will terminate the program.
+
+        """
+        try:
+            self._datasrc_clients_mgr.reconfigure(new_config, config_data)
+            genid, clients_map = self._datasrc_clients_mgr.get_clients_map()
+            datasrc_info = DataSrcInfo(genid, clients_map, self._config_params)
+            self._datasrc_info_list.append(datasrc_info)
+
+            # Full datasrc reconfig will be rare, so would be worth logging
+            # at the info level.
+            logger.info(MEMMGR_DATASRC_RECONFIGURED, genid)
+        except isc.server_common.datasrc_clients_mgr.ConfigError as ex:
+            logger.error(MEMMGR_DATASRC_CONFIG_ERROR, ex)
+
+if '__main__' == __name__:
+    mgr = Memmgr()
+    sys.exit(mgr.run(MODULE_NAME))

src/bin/memmgr/memmgr.spec.pre.in

@@ -0,0 +1,25 @@
+{
+  "module_spec": {
+    "module_name": "Memmgr",
+    "config_data": [
+      { "item_name": "mapped_file_dir",
+        "item_type": "string",
+        "item_optional": true,
+        "item_default": "@@LOCALSTATEDIR@@/@PACKAGE@/mapped_files"
+      }
+    ],
+    "commands": [
+      {
+        "command_name": "shutdown",
+        "command_description": "Shut down Memmgr",
+        "command_args": [
+          {
+            "item_name": "pid",
+            "item_type": "integer",
+            "item_optional": true
+          }
+        ]
+      }
+    ]
+  }
+}

src/bin/memmgr/memmgr_messages.mes

@@ -0,0 +1,51 @@
+# Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+# PERFORMANCE OF THIS SOFTWARE.
+
+# When you add a message to this file, it is a good idea to run
+# <topsrcdir>/tools/reorder_message_file.py to make sure the
+# messages are in the correct order.
+
+% MEMMGR_CONFIG_FAIL failed to apply configuration updates: %1
+The memmgr daemon tried to apply configuration updates but found an error.
+The cause of the error is included in the message.  None of the received
+updates applied, and the daemon keeps running with the previous configuration.
+
+% MEMMGR_CONFIG_UPDATE received new configuration
+A debug message.  The memmgr daemon receives configuratiopn updates
+and is now applying them to its running configurations.
+
+% MEMMGR_DATASRC_CONFIG_ERROR failed to update data source configuration: %1
+Configuration for the global data sources is updated, but the update
+cannot be applied to memmgr.  The memmgr module will still keep running
+with the previous configuration, but the cause of the failure and
+other log messages must be carefully examined because if only memmgr
+rejects the new configuration then the entire BIND 10 system will have
+inconsistent state among different modules.  If other modules accept
+the update but memmgr produces this error, it's quite likely that the
+system isn't working as expected, and is probably better to be shut down
+to figure out and fix the cause.
+
+% MEMMGR_DATASRC_RECONFIGURED data source configuration has been updated, generation ID %1
+The memmgr daemon received a new version of data source configuration,
+and has successfully applied it to the local state.  Loading of new zone
+data into memory will possibly take place.
+
+% MEMMGR_NO_DATASRC_CONF failed to add data source configuration: %1
+The memmgr daemon tried to incorporate data source configuration on
+its startup but failed to do so.  Due to internal implementation
+details this shouldn't happen as long as the BIND 10 system has been
+installed correctly.  So, if this error message is logged, you should
+probably reinstall the entire system, preferably from the scratch, and
+see if it still happens.  The memmgr daemon cannot do any meaningful
+work without data sources, so it immediately terminates itself.

src/bin/memmgr/tests/memmgr_test.py

@@ -0,0 +1,211 @@
+# Copyright (C) 2013  Internet Systems Consortium.
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
+# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
+# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+import unittest
+import os
+import re
+
+import isc.log
+from isc.dns import RRClass
+import isc.config
+from isc.config import parse_answer
+import memmgr
+from isc.testutils.ccsession_mock import MockModuleCCSession
+
+class MyCCSession(MockModuleCCSession, isc.config.ConfigData):
+    def __init__(self, specfile, config_handler, command_handler):
+        super().__init__()
+        specfile = os.environ['B10_FROM_BUILD'] + '/src/bin/memmgr/memmgr.spec'
+        module_spec = isc.config.module_spec_from_file(specfile)
+        isc.config.ConfigData.__init__(self, module_spec)
+        self.add_remote_params = [] # for inspection
+        self.add_remote_exception = None # to raise exception from the method
+
+    def start(self):
+        pass
+
+    def add_remote_config_by_name(self, mod_name, handler):
+        if self.add_remote_exception is not None:
+            raise self.add_remote_exception
+        self.add_remote_params.append((mod_name, handler))
+
+class MockMemmgr(memmgr.Memmgr):
+    def _setup_ccsession(self):
+        orig_cls = isc.config.ModuleCCSession
+        isc.config.ModuleCCSession = MyCCSession
+        try:
+            super()._setup_ccsession()
+        finally:
+            isc.config.ModuleCCSession = orig_cls
+
+# Defined for easier tests with DataSrcClientsMgr.reconfigure(), which
+# only needs get_value() method
+class MockConfigData:
+    def __init__(self, data):
+        self.__data = data
+
+    def get_value(self, identifier):
+        return self.__data[identifier], False
+
+class TestMemmgr(unittest.TestCase):
+    def setUp(self):
+        # Some tests use this directory.  Make sure it doesn't pre-exist.
+        self.__test_mapped_file_dir = \
+            os.environ['B10_FROM_BUILD'] + \
+            '/src/bin/memmgr/tests/test_mapped_files'
+        if os.path.isdir(self.__test_mapped_file_dir):
+            os.rmdir(self.__test_mapped_file_dir)
+
+        self.__mgr = MockMemmgr()
+        # Fake some 'os' module functions for easier tests
+        self.__orig_os_access = os.access
+        self.__orig_isdir = os.path.isdir
+
+    def tearDown(self):
+        # Restore faked values
+        os.access = self.__orig_os_access
+        os.path.isdir = self.__orig_isdir
+
+        # If at test created a mapped-files directory, delete it.
+        if os.path.isdir(self.__test_mapped_file_dir):
+            os.rmdir(self.__test_mapped_file_dir)
+
+    def test_init(self):
+        """Check some initial conditions"""
+        self.assertIsNone(self.__mgr._config_params)
+        self.assertEqual([], self.__mgr._datasrc_info_list)
+
+        # Try to configure a data source clients with the manager.  This
+        # should confirm the manager object is instantiated enabling in-memory
+        # cache.
+        cfg_data = MockConfigData(
+            {"classes": {"IN": [{"type": "MasterFiles",
+                                 "cache-enable": True, "params": {}}]}})
+        self.__mgr._datasrc_clients_mgr.reconfigure({}, cfg_data)
+        clist = \
+            self.__mgr._datasrc_clients_mgr.get_client_list(RRClass.IN)
+        self.assertEqual(1, len(clist.get_status()))
+
+    def test_configure(self):
+        self.__mgr._setup_ccsession()
+
+        # Pretend specified directories exist and writable
+        os.path.isdir = lambda x: True
+        os.access = lambda x, y: True
+
+        # At the initial configuration, if mapped_file_dir isn't specified,
+        # the default value will be set.
+        self.assertEqual((0, None),
+                         parse_answer(self.__mgr._config_handler({})))
+        self.assertEqual('mapped_files',
+                         self.__mgr._config_params['mapped_file_dir'].
+                         split('/')[-1])
+
+        # Update the configuration.
+        user_cfg = {'mapped_file_dir': '/some/path/dir'}
+        self.assertEqual((0, None),
+                         parse_answer(self.__mgr._config_handler(user_cfg)))
+        self.assertEqual('/some/path/dir',
+                         self.__mgr._config_params['mapped_file_dir'])
+
+        # Bad update: diretory doesn't exist (we assume it really doesn't
+        # exist in the tested environment).  Update won't be made.
+        os.path.isdir = self.__orig_isdir # use real library
+        user_cfg = {'mapped_file_dir': '/another/path/dir'}
+        answer = parse_answer(self.__mgr._config_handler(user_cfg))
+        self.assertEqual(1, answer[0])
+        self.assertIsNotNone(re.search('not a directory', answer[1]))
+
+        # Bad update: directory exists but is not readable.
+        os.mkdir(self.__test_mapped_file_dir, 0o500) # drop writable bit
+        os.access = self.__orig_os_access
+        user_cfg = {'mapped_file_dir': self.__test_mapped_file_dir}
+        answer = parse_answer(self.__mgr._config_handler(user_cfg))
+        self.assertEqual(1, answer[0])
+        self.assertIsNotNone(re.search('not writable', answer[1]))
+
+    def test_setup_module(self):
+        # _setup_module should add data_sources remote module with
+        # expected parameters.
+        self.__mgr._setup_ccsession()
+        self.assertEqual([], self.__mgr.mod_ccsession.add_remote_params)
+        self.__mgr._setup_module()
+        self.assertEqual([('data_sources',
+                           self.__mgr._datasrc_config_handler)],
+                         self.__mgr.mod_ccsession.add_remote_params)
+
+        # If data source isn't configured it's considered fatal (checking the
+        # same scenario with two possible exception types)
+        self.__mgr.mod_ccsession.add_remote_exception = \
+            isc.config.ModuleCCSessionError('faked exception')
+        self.assertRaises(isc.server_common.bind10_server.BIND10ServerFatal,
+                          self.__mgr._setup_module)
+
+        self.__mgr.mod_ccsession.add_remote_exception = \
+            isc.config.ModuleSpecError('faked exception')
+        self.assertRaises(isc.server_common.bind10_server.BIND10ServerFatal,
+                          self.__mgr._setup_module)
+
+    def test_datasrc_config_handler(self):
+        self.__mgr._config_params = {'mapped_file_dir': '/some/path'}
+
+        # A simple (boring) case with real class implementations.  This
+        # confirms the methods are called as expected.
+        cfg_data = MockConfigData(
+            {"classes": {"IN": [{"type": "MasterFiles",
+                                 "cache-enable": True, "params": {}}]}})
+        self.__mgr._datasrc_config_handler({}, cfg_data)
+        self.assertEqual(1, len(self.__mgr._datasrc_info_list))
+        self.assertEqual(1, self.__mgr._datasrc_info_list[0].gen_id)
+
+        # Below we're using a mock DataSrcClientMgr for easier tests
+        class MockDataSrcClientMgr:
+            def __init__(self, status_list, raise_on_reconfig=False):
+                self.__status_list = status_list
+                self.__raise_on_reconfig = raise_on_reconfig
+
+            def reconfigure(self, new_config, config_data):
+                if self.__raise_on_reconfig:
+                    raise isc.server_common.datasrc_clients_mgr.ConfigError(
+                        'test error')
+                # otherwise do nothing
+
+            def get_clients_map(self):
+                return 42, {RRClass.IN: self}
+
+            def get_status(self): # mocking get_clients_map()[1].get_status()
+                return self.__status_list
+
+        # This confirms memmgr's config is passed and handled correctly.
+        # From memmgr's point of view it should be enough we have an object
+        # in segment_info_map.  Note also that the new DataSrcInfo is appended
+        # to the list
+        self.__mgr._datasrc_clients_mgr = \
+            MockDataSrcClientMgr([('sqlite3', 'mapped', None)])
+        self.__mgr._datasrc_config_handler(None, None) # params don't matter
+        self.assertEqual(2, len(self.__mgr._datasrc_info_list))
+        self.assertIsNotNone(
+            self.__mgr._datasrc_info_list[1].segment_info_map[
+                (RRClass.IN, 'sqlite3')])
+
+        # Emulate the case reconfigure() fails.  Exception isn't propagated,
+        # but the list doesn't change.
+        self.__mgr._datasrc_clients_mgr = MockDataSrcClientMgr(None, True)
+        self.__mgr._datasrc_config_handler(None, None)
+        self.assertEqual(2, len(self.__mgr._datasrc_info_list))
+
+if __name__== "__main__":
+    isc.log.resetUnitTestRootLogger()
+    unittest.main()

src/bin/xfrin/tests/xfrin_test.py

@@ -256,11 +256,11 @@ class MockDataSrcClientsMgr():
     def get_client_list(self, rrclass):
         return self.found_datasrc_client_list
 
-    def reconfigure(self, arg1):
+    def reconfigure(self, arg1, arg2):
         # the only current test simply needs to know this is called with
-        # the expected argument and exceptions are handled.  if we need more
+        # the expected arguments and exceptions are handled.  if we need more
         # variations in tests, this mock method should be extended.
-        self.reconfigure_param.append(arg1)
+        self.reconfigure_param.append((arg1, arg2))
         raise isc.server_common.datasrc_clients_mgr.ConfigError(
             'reconfigure failure')
 
@@ -3038,7 +3038,7 @@ class TestXfrin(unittest.TestCase):
         # we just check it's called as expected, and the only possible
         # exception doesn't cause disruption.
         self.xfr._datasrc_config_handler(True, False)
-        self.assertEqual([True],
+        self.assertEqual([(True, False)],
                          self.xfr._datasrc_clients_mgr.reconfigure_param)
 
 def raise_interrupt():

src/bin/xfrin/xfrin.py.in

@@ -1459,7 +1459,7 @@ class Xfrin:
 
         """
         try:
-            self._datasrc_clients_mgr.reconfigure(new_config)
+            self._datasrc_clients_mgr.reconfigure(new_config, config_data)
         except isc.server_common.datasrc_clients_mgr.ConfigError as ex:
             logger.error(XFRIN_DATASRC_CONFIG_ERROR, ex)
 

src/lib/python/isc/Makefile.am

@@ -1,5 +1,5 @@
 SUBDIRS = datasrc util cc config dns log net notify testutils acl bind10
-SUBDIRS += xfrin log_messages server_common ddns sysinfo statistics
+SUBDIRS += xfrin log_messages server_common ddns sysinfo statistics memmgr
 
 python_PYTHON = __init__.py
 

src/lib/python/isc/log_messages/Makefile.am

@@ -4,6 +4,7 @@ EXTRA_DIST = __init__.py
 EXTRA_DIST += init_messages.py
 EXTRA_DIST += cmdctl_messages.py
 EXTRA_DIST += ddns_messages.py
+EXTRA_DIST += memmgr_messages.py
 EXTRA_DIST += stats_messages.py
 EXTRA_DIST += stats_httpd_messages.py
 EXTRA_DIST += xfrin_messages.py
@@ -24,6 +25,7 @@ CLEANFILES = __init__.pyc
 CLEANFILES += init_messages.pyc
 CLEANFILES += cmdctl_messages.pyc
 CLEANFILES += ddns_messages.pyc
+CLEANFILES += memmgr_messages.pyc
 CLEANFILES += stats_messages.pyc
 CLEANFILES += stats_httpd_messages.pyc
 CLEANFILES += xfrin_messages.pyc

src/lib/python/isc/log_messages/memmgr_messages.py

@@ -0,0 +1 @@
+from work.memmgr_messages import *

src/lib/python/isc/memmgr/Makefile.am

@@ -0,0 +1,10 @@
+SUBDIRS = . tests
+
+python_PYTHON = __init__.py datasrc_info.py
+
+pythondir = $(pyexecdir)/isc/memmgr
+
+CLEANDIRS = __pycache__
+
+clean-local:
+	rm -rf $(CLEANDIRS)

src/lib/python/isc/memmgr/datasrc_info.py

@@ -0,0 +1,220 @@
+# Copyright (C) 2013  Internet Systems Consortium.
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
+# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
+# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+import os
+
+class SegmentInfoError(Exception):
+    """An exception raised for general errors in the SegmentInfo class."""
+    pass
+
+class SegmentInfo:
+    """A base class to maintain information about memory segments.
+
+    An instance of this class corresponds to the memory segment used
+    for in-memory cache of a specific single data source.  It manages
+    information to set/reset the latest effective segment (such as
+    path to a memory mapped file) and sets of other modules using the
+    segment.
+
+    Since there can be several different types of memory segments,
+    the top level class provides abstract interfaces independent from
+    segment-type specific details.  Such details are expected to be
+    delegated to subclasses corresponding to specific types of segments.
+
+    The implementation is still incomplete.  It will have more attributes
+    such as a set of current readers, methods for adding or deleting
+    the readers.  These will probably be implemented in this base class
+    as they will be independent from segment-type specific details.
+
+    """
+    # Common constants of user type: reader or writer
+    READER = 0
+    WRITER = 1
+
+    def create(type, genid, rrclass, datasrc_name, mgr_config):
+        """Factory of specific SegmentInfo subclass instance based on the
+        segment type.
+
+        This is specifically for the memmgr, and segments that are not of
+        its interest will be ignored.  This method returns None in these
+        cases.  At least 'local' type segments will be ignored this way.
+
+        If an unknown type of segment is specified, this method throws an
+        SegmentInfoError exception.  The assumption is that this method
+        is called after the corresponding data source configuration has been
+        validated, at which point such unknown segments should have been
+        rejected.
+
+        Parameters:
+          type (str or None): The type of memory segment; None if the segment
+              isn't used.
+          genid (int): The generation ID of the corresponding data source
+              configuration.
+          rrclass (isc.dns.RRClass): The RR class of the data source.
+          datasrc_name (str): The name of the data source.
+          mgr_config (dict): memmgr configuration related to memory segment
+              information.  The content of the dict is type specific; each
+              subclass is expected to know which key is necessary and the
+              semantics of its value.
+
+        """
+        if type == 'mapped':
+            return MappedSegmentInfo(genid, rrclass, datasrc_name, mgr_config)
+        elif type is None or type == 'local':
+            return None
+        raise SegmentInfoError('unknown segment type to create info: ' + type)
+
+    def get_reset_param(self, user_type):
+        """Return parameters to reset the zone table memory segment.
+
+        It returns a dict object that consists of parameter mappings
+        (string to parameter value) for the specified type of user to
+        reset a zone table segment with
+        isc.datasrc.ConfigurableClientList.reset_memory_segment().  It
+        can also be passed to the user module as part of command
+        parameters.  Note that reset_memory_segment() takes a json
+        expression encoded as a string, so the return value of this method
+        will have to be converted with json.dumps().
+
+        Each subclass must implement this method.
+
+        Parameter:
+          user_type (READER or WRITER): specifies the type of user to reset
+              the segment.
+
+        """
+        raise SegmentInfoError('get_reset_param is not implemented')
+
+    def switch_versions(self):
+        """Switch internal information for the reader segment and writer
+        segment.
+
+        This method is expected to be called when the writer on one version
+        of memory segment completes updates and the memmgr is going to
+        have readers switch to the updated version.  Details of the
+        information to be switched would depend on the segment type, and
+        are delegated to the specific subclass.
+
+        Each subclass must implement this method.
+
+        """
+        raise SegmentInfoError('switch_versions is not implemented')
+
+class MappedSegmentInfo(SegmentInfo):
+    """SegmentInfo implementation of 'mapped' type memory segments.
+
+    It maintains paths to mapped files both readers and the writer.
+
+    While objets of this class are expected to be shared by multiple
+    threads, it assumes operations are serialized through message passing,
+    so access to this class itself is not protected by any explicit
+    synchronization mechanism.
+
+    """
+    def __init__(self, genid, rrclass, datasrc_name, mgr_config):
+        super().__init__()
+
+        # Something like "/var/bind10/zone-IN-1-sqlite3-mapped"
+        self.__mapped_file_base = mgr_config['mapped_file_dir'] + os.sep + \
+            'zone-' + str(rrclass) + '-' + str(genid) + '-' + datasrc_name + \
+            '-mapped'
+
+        # Current versions (suffix of the mapped files) for readers and the
+        # writer.  In this initial implementation we assume that all possible
+        # readers are waiting for a new version (not using pre-existing one),
+        # and the writer is expected to build a new segment as version "0".
+        self.__reader_ver = None # => 0 => 1 => 0 => 1 ...
+        self.__writer_ver = 0    # => 1 => 0 => 1 => 0 ...
+
+    def get_reset_param(self, user_type):
+        ver = self.__reader_ver if user_type == self.READER else \
+            self.__writer_ver
+        if ver is None:
+            return None
+        mapped_file = self.__mapped_file_base + '.' + str(ver)
+        return {'mapped-file': mapped_file}
+
+    def switch_versions(self):
+        # Swith the versions as noted in the constructor.
+        self.__writer_ver = 1 - self.__writer_ver
+
+        if self.__reader_ver is None:
+            self.__reader_ver = 0
+        else:
+            self.__reader_ver = 1 - self.__reader_ver
+
+        # Versions should be different
+        assert(self.__reader_ver != self.__writer_ver)
+
+class DataSrcInfo:
+    """A container for datasrc.ConfigurableClientLists and associated
+    in-memory segment information corresponding to a given geration of
+    configuration.
+
+    This class maintains all datasrc.ConfigurableClientLists in a form
+    of dict from RR classes corresponding to a given single generation
+    of data source configuration, along with sets of memory segment
+    information that needs to be used by memmgr.
+
+    Once constructed, mappings do not change (different generation of
+    configuration will result in another DataSrcInfo objects).  Status
+    of SegmentInfo objects stored in this class object may change over time.
+
+    Attributes: these are all constant and read only.  For dict objects,
+          mapping shouldn't be modified either.
+      gen_id (int): The corresponding configuration generation ID.
+      clients_map (dict, isc.dns.RRClass=>isc.datasrc.ConfigurableClientList):
+          The configured client lists for all RR classes of the generation.
+      segment_info_map (dict, (isc.dns.RRClass, str)=>SegmentInfo):
+          SegmentInfo objects managed in the DataSrcInfo objects.  Can be
+          retrieved by (RRClass, <data source name>).
+
+    """
+    def __init__(self, genid, clients_map, mgr_config):
+        """Constructor.
+
+        As long as given parameters are of valid type and have been
+        validated, this constructor shouldn't raise an exception.
+
+        Parameters:
+          genid (int): see gen_id attribute
+          clients_map (dict): see clients_map attribute
+          mgr_config (dict, str=>key-dependent-value): A copy of the current
+            memmgr configuration, in case it's needed to construct a specific
+            type of SegmentInfo.  The specific SegmentInfo class is expected
+            to know the key-value mappings that it needs.
+
+        """
+        self.__gen_id = genid
+        self.__clients_map = clients_map
+        self.__segment_info_map = {}
+        for (rrclass, client_list) in clients_map.items():
+            for (name, sgmt_type, _) in client_list.get_status():
+                sgmt_info = SegmentInfo.create(sgmt_type, genid, rrclass, name,
+                                               mgr_config)
+                if sgmt_info is not None:
+                    self.__segment_info_map[(rrclass, name)] = sgmt_info
+
+    @property
+    def gen_id(self):
+        return self.__gen_id
+
+    @property
+    def clients_map(self):
+        return self.__clients_map
+
+    @property
+    def segment_info_map(self):
+        return self.__segment_info_map

src/lib/python/isc/memmgr/tests/datasrc_info_tests.py

@@ -0,0 +1,192 @@
+# Copyright (C) 2013  Internet Systems Consortium.
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
+# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
+# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+import os
+import unittest
+
+from isc.dns import *
+import isc.config
+import isc.datasrc
+import isc.log
+from isc.server_common.datasrc_clients_mgr import DataSrcClientsMgr
+from isc.memmgr.datasrc_info import *
+
+# Defined for easier tests with DataSrcClientsMgr.reconfigure(), which
+# only needs get_value() method
+class MockConfigData:
+    def __init__(self, data):
+        self.__data = data
+
+    def get_value(self, identifier):
+        return self.__data[identifier], False
+
+class TestSegmentInfo(unittest.TestCase):
+    def setUp(self):
+        self.__mapped_file_dir = os.environ['TESTDATA_PATH']
+        self.__sgmt_info = SegmentInfo.create('mapped', 0, RRClass.IN,
+                                              'sqlite3',
+                                              {'mapped_file_dir':
+                                                   self.__mapped_file_dir})
+
+    def __check_sgmt_reset_param(self, user_type, expected_ver):
+        """Common check on the return value of get_reset_param() for
+        MappedSegmentInfo.
+
+        Unless it's expected to return None, it should be a map that
+        maps "mapped-file" to the expected version of mapped-file.
+
+        """
+        if expected_ver is None:
+            self.assertIsNone(self.__sgmt_info.get_reset_param(user_type))
+            return
+        param = self.__sgmt_info.get_reset_param(user_type)
+        self.assertEqual(self.__mapped_file_dir +
+                         '/zone-IN-0-sqlite3-mapped.' + str(expected_ver),
+                         param['mapped-file'])
+
+    def test_initial_params(self):
+        self.__check_sgmt_reset_param(SegmentInfo.WRITER, 0)
+        self.__check_sgmt_reset_param(SegmentInfo.READER, None)
+
+    def test_swtich_versions(self):
+        self.__sgmt_info.switch_versions()
+        self.__check_sgmt_reset_param(SegmentInfo.WRITER, 1)
+        self.__check_sgmt_reset_param(SegmentInfo.READER, 0)
+
+        self.__sgmt_info.switch_versions()
+        self.__check_sgmt_reset_param(SegmentInfo.WRITER, 0)
+        self.__check_sgmt_reset_param(SegmentInfo.READER, 1)
+
+    def test_init_others(self):
+        # For local type of segment, information isn't needed and won't be
+        # created.
+        self.assertIsNone(SegmentInfo.create('local', 0, RRClass.IN,
+                                             'sqlite3', {}))
+
+        # Unknown type of segment will result in an exception.
+        self.assertRaises(SegmentInfoError, SegmentInfo.create, 'unknown', 0,
+                          RRClass.IN, 'sqlite3', {})
+
+    def test_missing_methods(self):
+        # Bad subclass of SegmentInfo that doesn't implement mandatory methods.
+        class TestSegmentInfo(SegmentInfo):
+            pass
+
+        self.assertRaises(SegmentInfoError,
+                          TestSegmentInfo().get_reset_param,
+                          SegmentInfo.WRITER)
+        self.assertRaises(SegmentInfoError, TestSegmentInfo().switch_versions)
+
+class MockClientList:
+    """A mock ConfigurableClientList class.
+
+    Just providing minimal shortcut interfaces needed for DataSrcInfo class.
+
+    """
+    def __init__(self, status_list):
+        self.__status_list = status_list
+
+    def get_status(self):
+        return self.__status_list
+
+class TestDataSrcInfo(unittest.TestCase):
+    def setUp(self):
+        self.__mapped_file_dir = os.environ['TESTDATA_PATH']
+        self.__mgr_config = {'mapped_file_dir': self.__mapped_file_dir}
+        self.__sqlite3_dbfile = os.environ['TESTDATA_PATH'] + '/' + 'zone.db'
+        self.__clients_map = {
+            # mixture of 'local' and 'mapped' and 'unused' (type =None)
+            # segments
+            RRClass.IN: MockClientList([('datasrc1', 'local', None),
+                                        ('datasrc2', 'mapped', None),
+                                        ('datasrc3', None, None)]),
+            RRClass.CH: MockClientList([('datasrc2', 'mapped', None),
+                                        ('datasrc1', 'local', None)]) }
+
+    def tearDown(self):
+        if os.path.exists(self.__sqlite3_dbfile):
+            os.unlink(self.__sqlite3_dbfile)
+
+    def __check_sgmt_reset_param(self, sgmt_info, writer_file):
+        # Check if the initial state of (mapped) segment info object has
+        # expected values.
+        self.assertIsNone(sgmt_info.get_reset_param(SegmentInfo.READER))
+        param = sgmt_info.get_reset_param(SegmentInfo.WRITER)
+        self.assertEqual(writer_file, param['mapped-file'])
+
+    def test_init(self):
+        """Check basic scenarios of constructing DataSrcInfo."""
+
+        # This checks that all data sources of all RR classes are covered,
+        # "local" segments are ignored, info objects for "mapped" segments
+        # are created and stored in segment_info_map.
+        datasrc_info = DataSrcInfo(42, self.__clients_map, self.__mgr_config)
+        self.assertEqual(42, datasrc_info.gen_id)
+        self.assertEqual(self.__clients_map, datasrc_info.clients_map)
+        self.assertEqual(2, len(datasrc_info.segment_info_map))
+        sgmt_info = datasrc_info.segment_info_map[(RRClass.IN, 'datasrc2')]
+        self.__check_sgmt_reset_param(sgmt_info, self.__mapped_file_dir +
+                                      '/zone-IN-42-datasrc2-mapped.0')
+        sgmt_info = datasrc_info.segment_info_map[(RRClass.CH, 'datasrc2')]
+        self.__check_sgmt_reset_param(sgmt_info, self.__mapped_file_dir +
+                                      '/zone-CH-42-datasrc2-mapped.0')
+
+        # A case where clist.get_status() returns an empty list; shouldn't
+        # cause disruption
+        self.__clients_map = { RRClass.IN: MockClientList([])}
+        datasrc_info = DataSrcInfo(42, self.__clients_map, self.__mgr_config)
+        self.assertEqual(42, datasrc_info.gen_id)
+        self.assertEqual(0, len(datasrc_info.segment_info_map))
+
+        # A case where clients_map is empty; shouldn't cause disruption
+        self.__clients_map = {}
+        datasrc_info = DataSrcInfo(42, self.__clients_map, self.__mgr_config)
+        self.assertEqual(42, datasrc_info.gen_id)
+        self.assertEqual(0, len(datasrc_info.segment_info_map))
+
+    # This test uses real "mmaped" segment and doesn't work without shared
+    # memory support.
+    @unittest.skipIf(os.environ['HAVE_SHARED_MEMORY'] != 'yes',
+                     'shared memory support is not available')
+    def test_production(self):
+        """Check the behavior closer to a production environment.
+
+        Instead of using a mock classes, just for confirming we didn't miss
+        something.
+
+        """
+        cfg_data = MockConfigData(
+            {"classes":
+                 {"IN": [{"type": "sqlite3", "cache-enable": True,
+                          "cache-type": "mapped", "cache-zones": [],
+                          "params": {"database_file": self.__sqlite3_dbfile}}]
+                  }
+             })
+        cmgr = DataSrcClientsMgr(use_cache=True)
+        cmgr.reconfigure({}, cfg_data)
+
+        genid, clients_map = cmgr.get_clients_map()
+        datasrc_info = DataSrcInfo(genid, clients_map, self.__mgr_config)
+
+        self.assertEqual(1, datasrc_info.gen_id)
+        self.assertEqual(clients_map, datasrc_info.clients_map)
+        self.assertEqual(1, len(datasrc_info.segment_info_map))
+        sgmt_info = datasrc_info.segment_info_map[(RRClass.IN, 'sqlite3')]
+        self.assertIsNone(sgmt_info.get_reset_param(SegmentInfo.READER))
+
+if __name__ == "__main__":
+    isc.log.init("bind10-test")
+    isc.log.resetUnitTestRootLogger()
+    unittest.main()

src/lib/python/isc/server_common/Makefile.am

@@ -1,12 +1,14 @@
 SUBDIRS = tests
 
 python_PYTHON = __init__.py tsig_keyring.py auth_command.py dns_tcp.py
-python_PYTHON += datasrc_clients_mgr.py
+python_PYTHON += datasrc_clients_mgr.py bind10_server.py
 python_PYTHON += logger.py
 
 pythondir = $(pyexecdir)/isc/server_common
 
 BUILT_SOURCES = $(PYTHON_LOGMSGPKG_DIR)/work/server_common_messages.py
+BUILT_SOURCES += bind10_server.py
+
 nodist_pylogmessage_PYTHON = $(PYTHON_LOGMSGPKG_DIR)/work/server_common_messages.py
 
 pylogmessagedir = $(pyexecdir)/isc/log_messages/

src/lib/python/isc/server_common/bind10_server.py.in

@@ -0,0 +1,213 @@
+# Copyright (C) 2013  Internet Systems Consortium.
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
+# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
+# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+import errno
+import os
+import select
+import signal
+
+import isc.log
+import isc.config
+from isc.server_common.logger import logger
+from isc.log_messages.server_common_messages import *
+
+class BIND10ServerFatal(Exception):
+    """Exception raised when the server program encounters a fatal error."""
+    pass
+
+class BIND10Server:
+    """A mixin class for common BIND 10 server implementations.
+
+    It takes care of common initialization such as setting up a module CC
+    session, and running main event loop.  It also handles the "shutdown"
+    command for its normal behavior.  If a specific server class wants to
+    handle this command differently or if it does not support the command,
+    it should override the _command_handler method.
+
+    Specific modules can define module-specific class inheriting this class,
+    instantiate it, and call run() with the module name.
+
+    Methods to be implemented in the actual class:
+      _config_handler: config handler method as specified in ModuleCCSession.
+                       must be exception free; errors should be signaled by
+                       the return value.
+      _mod_command_handler: can be optionally defined to handle
+                            module-specific commands.  should conform to
+                            command handlers as specified in ModuleCCSession.
+                            must be exception free; errors should be signaled
+                            by the return value.
+      _setup_module: can be optionally defined for module-specific
+                     initialization.  This is called after the module CC
+                     session has started, and can be used for registering
+                     interest on remote modules, etc.  If it raises an
+                     exception, the server will be immediatelly stopped.
+                     Parameter: None, Return: None
+
+    """
+    # Will be set to True when the server should stop and shut down.
+    # Can be read via accessor method 'shutdown', mainly for testing.
+    __shutdown = False
+
+    # ModuleCCSession used in the server.  Defined as 'protectd' so tests
+    # can refer to it directly; others should access it via the
+    # 'mod_ccsession' accessor.
+    _mod_cc = None
+
+    # Will be set in run().  Define a tentative value so other methods can
+    # be tested directly.
+    __module_name = ''
+
+    # Basically constant, but allow tests to override it.
+    _select_fn = select.select
+
+    @property
+    def shutdown(self):
+        return self.__shutdown
+
+    @property
+    def mod_ccsession(self):
+        return self._mod_cc
+
+    def _setup_ccsession(self):
+        """Create and start module CC session.
+
+        This is essentially private, but allows tests to override it.
+
+        """
+        self._mod_cc = isc.config.ModuleCCSession(
+            self._get_specfile_location(), self._config_handler,
+            self._command_handler)
+        self._mod_cc.start()
+
+    def _get_specfile_location(self):
+        """Return the path to the module spec file following common convetion.
+
+        This method generates the path commonly used by most BIND 10 modules,
+        determined by a well known prefix and the module name.
+
+        A specific module can override this method if it uses a different
+        path for the spec file.
+
+        """
+        # First check if it's running under an 'in-source' environment,
+        # then try commonly used paths and file names.  If found, use it.
+        for ev in ['B10_FROM_SOURCE', 'B10_FROM_BUILD']:
+            if ev in os.environ:
+                specfile = os.environ[ev] + '/src/bin/' + self.__module_name +\
+                    '/' + self.__module_name + '.spec'
+                if os.path.exists(specfile):
+                    return specfile
+        # Otherwise, just use the installed path, whether or not it really
+        # exists; leave error handling to the caller.
+        specfile_path = '${datarootdir}/bind10'\
+            .replace('${datarootdir}', '${prefix}/share')\
+            .replace('${prefix}', '/Users/jinmei/opt')
+        return specfile_path + '/' + self.__module_name + '.spec'
+
+    def _trigger_shutdown(self):
+        """Initiate a shutdown sequence.
+
+        This method is expected to be called in various ways including
+        in the middle of a signal handler, and is designed to be as simple
+        as possible to minimize side effects.  Actual shutdown will take
+        place in a normal control flow.
+
+        This method is defined as 'protected'.  User classes can use it
+        to shut down the server.
+
+        """
+        self.__shutdown = True
+
+    def _run_internal(self):
+        """Main event loop.
+
+        This method is essentially private, but allows tests to override it.
+
+        """
+
+        logger.info(PYSERVER_COMMON_SERVER_STARTED, self.__module_name)
+        cc_fileno = self._mod_cc.get_socket().fileno()
+        while not self.__shutdown:
+            try:
+                (reads, _, _) = self._select_fn([cc_fileno], [], [])
+            except select.error as ex:
+                # ignore intterruption by signal; regard other select errors
+                # fatal.
+                if ex.args[0] == errno.EINTR:
+                    continue
+                else:
+                    raise
+            for fileno in reads:
+                if fileno == cc_fileno:
+                    # this shouldn't raise an exception (if it does, we'll
+                    # propagate it)
+                    self._mod_cc.check_command(True)
+
+        self._mod_cc.send_stopping()
+
+    def _command_handler(self, cmd, args):
+        logger.debug(logger.DBGLVL_TRACE_BASIC, PYSERVER_COMMON_COMMAND,
+                     self.__module_name, cmd)
+        if cmd == 'shutdown':
+            self._trigger_shutdown()
+            answer = isc.config.create_answer(0)
+        else:
+            answer = self._mod_command_handler(cmd, args)
+
+        return answer
+
+    def _mod_command_handler(self, cmd, args):
+        """The default implementation of the module specific command handler"""
+        return isc.config.create_answer(1, "Unknown command: " + str(cmd))
+
+    def _setup_module(self):
+        """The default implementation of the module specific initilization"""
+        pass
+
+    def run(self, module_name):
+        """Start the server and let it run until it's told to stop.
+
+        Usually this must be the first method of this class that is called
+        from its user.
+
+        Parameter:
+          module_name (str): the Python module name for the actual server
+            implementation.  Often identical to the directory name in which
+            the implementation files are placed.
+
+        Returns: values expected to be used as program's exit code.
+          0: server has run and finished successfully.
+          1: some error happens
+
+          """
+        try:
+            self.__module_name = module_name
+            shutdown_sighandler = \
+                lambda signal, frame: self._trigger_shutdown()
+            signal.signal(signal.SIGTERM, shutdown_sighandler)
+            signal.signal(signal.SIGINT, shutdown_sighandler)
+            self._setup_ccsession()
+            self._setup_module()
+            self._run_internal()
+            logger.info(PYSERVER_COMMON_SERVER_STOPPED, self.__module_name)
+            return 0
+        except BIND10ServerFatal as ex:
+            logger.error(PYSERVER_COMMON_SERVER_FATAL, self.__module_name,
+                         ex)
+        except Exception as ex:
+            logger.error(PYSERVER_COMMON_UNCAUGHT_EXCEPTION, type(ex).__name__,
+                         ex)
+
+        return 1

src/lib/python/isc/server_common/datasrc_clients_mgr.py

@@ -40,10 +40,11 @@ class DataSrcClientsMgr:
     def __init__(self, use_cache=False):
         """Constructor.
 
-        In the initial implementation, user applications of this class are
-        generally expected to NOT use in-memory cache; use_cache would be
-        set to True only for tests.  In future, some applications such as
-        outbound zone transfer may want to set it to True.
+        In the initial implementation, most user applications of this class
+        are generally expected to NOT use in-memory cache; the only expected
+        exception is the memory (cache) manager, which, by definition,
+        needs to deal with in-memory data.  In future, some more applications
+        such as outbound zone transfer may want to set it to True.
 
         Parameter:
           use_cache (bool): If set to True, enable in-memory cache on
@@ -61,6 +62,31 @@ class DataSrcClientsMgr:
         self.__clients_map = {}
         self.__map_lock = threading.Lock()
 
+        # The generation ID of the configuration corresponding to
+        # current __clinets_map.  Until we support the concept of generations
+        # in the configuration framework, we tentatively maintain it within
+        # this class.
+        self.__gen_id = 0
+
+    def get_clients_map(self):
+        """Returns a dict from RR class to ConfigurableClientList with gen ID.
+
+        It corresponds to the generation of data source configuration at the
+        time of the call.  It can be safely called while reconfigure() is
+        called from another thread.
+
+        The mapping of the dict should be considered "frozen"; the caller
+        shouldn't modify the mapping (it can use the mapped objects in a
+        way modifying its internal state).
+
+        Note: in a future version we may also need to return the
+        "generation ID" of the corresponding configuration so the caller
+        application can handle migration between generations gradually.
+
+        """
+        with self.__map_lock:
+            return (self.__gen_id, self.__clients_map)
+
     def get_client_list(self, rrclass):
         """Return the configured ConfigurableClientList for the RR class.
 
@@ -91,7 +117,7 @@ class DataSrcClientsMgr:
             client_list = self.__clients_map.get(rrclass)
         return client_list
 
-    def reconfigure(self, config):
+    def reconfigure(self, new_config, config_data):
         """(Re)configure the set of client lists.
 
         This method takes a new set of data source configuration, builds
@@ -116,12 +142,20 @@ class DataSrcClientsMgr:
         at the same time.
 
         Parameter:
-          config (dict): configuration data for the data_sources module.
+          new_config (dict): configuration data for the data_sources module
+            (actually unused in this method).
+          config_data (isc.config.ConfigData): the latest full config data
+            for the data_sources module.  Usually the second parameter of
+            the (remote) configuration update callback for the module.
 
         """
         try:
             new_map = {}
-            for rrclass_cfg, class_cfg in config.get('classes').items():
+            # We only refer to config_data, not new_config (diff from the
+            # previous).  the latter may be empty for the initial default
+            # configuration while the former works for all cases.
+            for rrclass_cfg, class_cfg in \
+                    config_data.get_value('classes')[0].items():
                 rrclass = isc.dns.RRClass(rrclass_cfg)
                 new_client_list = isc.datasrc.ConfigurableClientList(rrclass)
                 new_client_list.configure(json.dumps(class_cfg),
@@ -129,6 +163,10 @@ class DataSrcClientsMgr:
                 new_map[rrclass] = new_client_list
             with self.__map_lock:
                 self.__clients_map = new_map
+
+                # NOTE: when we support the concept of generations this should
+                # be retrieved from the configuration
+                self.__gen_id += 1
         except Exception as ex:
             # Catch all types of exceptions as a whole: there won't be much
             # granularity for exceptions raised from the C++ module anyway.

src/lib/python/isc/server_common/server_common_messages.mes

@@ -21,6 +21,9 @@
 # have that at this moment. So when adding a message, make sure that
 # the name is not already used in src/lib/config/config_messages.mes
 
+% PYSERVER_COMMON_COMMAND %1 server has received '%2' command
+The server process received the shown name of command from other module.
+
 % PYSERVER_COMMON_DNS_TCP_SEND_DONE completed sending TCP message to %1 (%2 bytes in total)
 Debug message.  A complete DNS message has been successfully
 transmitted over a TCP connection, possibly after multiple send
@@ -44,6 +47,18 @@ The destination address and the total size of the message that has
 been transmitted so far (including the 2-byte length field) are shown
 in the log message.
 
+% PYSERVER_COMMON_SERVER_FATAL %1 server has encountered a fatal error: %2
+The BIND 10 server process encountered a fatal error (normally specific to
+the particular program), and is forcing itself to shut down.
+
+% PYSERVER_COMMON_SERVER_STARTED %1 server has started
+The server process has successfully started and is now ready to receive
+commands and configuration updates.
+
+% PYSERVER_COMMON_SERVER_STOPPED %1 server has started
+The server process has successfully stopped and is no longer listening for or
+handling commands.  Normally the process will soon exit.
+
 % PYSERVER_COMMON_TSIG_KEYRING_DEINIT Deinitializing global TSIG keyring
 A debug message noting that the global TSIG keyring is being removed from
 memory. Most programs don't do that, they just exit, which is OK.
@@ -57,3 +72,8 @@ to be loaded from configuration.
 A debug message. The TSIG keyring is being (re)loaded from configuration.
 This happens at startup or when the configuration changes. The old keyring
 is removed and new one created with all the keys.
+
+% PYSERVER_COMMON_UNCAUGHT_EXCEPTION uncaught exception of type %1: %2
+The BIND 10 server process encountered an uncaught exception and will now shut
+down. This is indicative of a programming error and should not happen under
+normal circumstances. The exception type and message are printed.

src/lib/python/isc/server_common/tests/Makefile.am

@@ -1,5 +1,6 @@
 PYCOVERAGE_RUN = @PYCOVERAGE_RUN@
 PYTESTS = tsig_keyring_test.py dns_tcp_test.py datasrc_clients_mgr_test.py
+PYTESTS += bind10_server_test.py
 EXTRA_DIST = $(PYTESTS)
 
 # If necessary (rare cases), explicitly specify paths to dynamic libraries
@@ -29,6 +30,7 @@ endif
 	$(LIBRARY_PATH_PLACEHOLDER) \
 	PYTHONPATH=$(COMMON_PYTHON_PATH):$(abs_top_builddir)/src/lib/dns/python/.libs \
 	B10_LOCKFILE_DIR_FROM_BUILD=$(abs_top_builddir) \
+	B10_FROM_SOURCE=$(abs_top_srcdir) \
 	B10_FROM_BUILD=$(abs_top_builddir) \
 	$(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \
 	done

src/lib/python/isc/server_common/tests/bind10_server_test.py

@@ -0,0 +1,252 @@
+# Copyright (C) 2013  Internet Systems Consortium.
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
+# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
+# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+import unittest
+import errno
+import os
+import signal
+
+import isc.log
+import isc.config
+from isc.server_common.bind10_server import *
+from isc.testutils.ccsession_mock import MockModuleCCSession
+
+TEST_FILENO = 42                # arbitrarily chosen
+
+class TestException(Exception):
+    """A generic exception class specific in this test module."""
+    pass
+
+class MyCCSession(MockModuleCCSession, isc.config.ConfigData):
+    def __init__(self, specfile, config_handler, command_handler):
+        # record parameter for later inspection
+        self.specfile_param = specfile
+        self.config_handler_param = config_handler
+        self.command_handler_param = command_handler
+
+        self.check_command_param = None # used in check_command()
+
+        # Initialize some local attributes of MockModuleCCSession, including
+        # 'stopped'
+        MockModuleCCSession.__init__(self)
+
+    def start(self):
+        pass
+
+    def check_command(self, nonblock):
+        """Mock check_command(). Just record the param for later inspection."""
+        self.check_command_param = nonblock
+
+    def get_socket(self):
+        return self
+
+    def fileno(self):
+        """Pretending get_socket().fileno()
+
+        Returing an arbitrarily chosen constant.
+
+        """
+        return TEST_FILENO
+
+class MockServer(BIND10Server):
+    def __init__(self):
+        self._select_fn = self.select_wrapper
+
+    def _setup_ccsession(self):
+        orig_cls = isc.config.ModuleCCSession
+        isc.config.ModuleCCSession = MyCCSession
+        try:
+            super()._setup_ccsession()
+        except Exception:
+            raise
+        finally:
+            isc.config.ModuleCCSession = orig_cls
+
+    def _config_handler(self):
+        pass
+
+    def mod_command_handler(self, cmd, args):
+        """A sample _mod_command_handler implementation."""
+        self.command_handler_params = (cmd, args) # for inspection
+        return isc.config.create_answer(0)
+
+    def select_wrapper(self, reads, writes, errors):
+        self._trigger_shutdown() # make sure the loop will stop
+        self.select_params = (reads, writes, errors) # record for inspection
+        return [], [], []
+
+class TestBIND10Server(unittest.TestCase):
+    def setUp(self):
+        self.__server = MockServer()
+
+    def test_init(self):
+        """Check initial conditions"""
+        self.assertFalse(self.__server.shutdown)
+
+    def test_trigger_shutdown(self):
+        self.__server._trigger_shutdown()
+        self.assertTrue(self.__server.shutdown)
+
+    def test_sigterm_handler(self):
+        """Check the signal handler behavior.
+
+        SIGTERM and SIGINT should be caught and should call memmgr's
+        _trigger_shutdown().  This test also indirectly confirms run() calls
+        run_internal().
+
+        """
+        def checker():
+            self.__shutdown_called = True
+
+        self.__server._run_internal = lambda: os.kill(os.getpid(),
+                                                      signal.SIGTERM)
+        self.__server._trigger_shutdown = lambda: checker()
+        self.assertEqual(0, self.__server.run('test'))
+        self.assertTrue(self.__shutdown_called)
+
+        self.__shutdown_called = False
+        self.__server._run_internal = lambda: os.kill(os.getpid(),
+                                                      signal.SIGINT)
+        self.assertEqual(0, self.__server.run('test'))
+        self.assertTrue(self.__shutdown_called)
+
+    def test_exception(self):
+        """Check exceptions are handled, not leaked."""
+        def exception_raiser(ex_cls):
+            raise ex_cls('test')
+
+        # Test all possible exceptions that are explicitly caught
+        for ex in [TestException, BIND10ServerFatal]:
+            self.__server._run_internal = lambda: exception_raiser(ex)
+            self.assertEqual(1, self.__server.run('test'))
+
+    def test_run(self):
+        """Check other behavior of run()"""
+        self.__server._run_internal = lambda: None # prevent looping
+        self.assertEqual(0, self.__server.run('test'))
+        # module CC session should have been setup.
+        # The exact path to the spec file can vary, so we simply check
+        # it works and it's the expected name stripping the path.
+        self.assertEqual(
+            self.__server.mod_ccsession.specfile_param.split('/')[-1],
+            'test.spec')
+        self.assertEqual(self.__server.mod_ccsession.config_handler_param,
+                         self.__server._config_handler)
+        self.assertEqual(self.__server.mod_ccsession.command_handler_param,
+                         self.__server._command_handler)
+
+    def test_run_with_setup_module(self):
+        """Check run() with module specific setup method."""
+        self.setup_called = False
+        def check_called():
+            self.setup_called = True
+        self.__server._run_internal = lambda: None
+        self.__server._setup_module = check_called
+        self.assertEqual(0, self.__server.run('test'))
+        self.assertTrue(self.setup_called)
+
+    def test_shutdown_command(self):
+        answer = self.__server._command_handler('shutdown', None)
+        self.assertTrue(self.__server.shutdown)
+        self.assertEqual((0, None), isc.config.parse_answer(answer))
+
+    def test_other_command(self):
+        self.__server._mod_command_handler = self.__server.mod_command_handler
+        answer = self.__server._command_handler('other command', None)
+        # shouldn't be confused with shutdown
+        self.assertFalse(self.__server.shutdown)
+        self.assertEqual((0, None), isc.config.parse_answer(answer))
+        self.assertEqual(('other command', None),
+                         self.__server.command_handler_params)
+
+    def test_other_command_nohandler(self):
+        """Similar to test_other_command, but without explicit handler"""
+        # In this case "unknown command" error should be returned.
+        answer = self.__server._command_handler('other command', None)
+        self.assertEqual(1, isc.config.parse_answer(answer)[0])
+
+    def test_run_internal(self):
+        self.__server._setup_ccsession()
+        self.__server._run_internal()
+        self.assertEqual(([TEST_FILENO], [], []), self.__server.select_params)
+
+    def select_wrapper(self, r, w, e, ex=None, ret=None):
+        """Mock select() function used some of the tests below.
+
+        If ex is not None and it's first call to this method, it raises ex
+        assuming it's an exception.
+
+        If ret is not None, it returns the given value; otherwise it returns
+        all empty lists.
+
+        """
+        self.select_params.append((r, w, e))
+        if ex is not None and len(self.select_params) == 1:
+            raise ex
+        else:
+            self.__server._trigger_shutdown()
+        if ret is not None:
+            return ret
+        return [], [], []
+
+    def test_select_for_command(self):
+        """A normal event iteration, handling one command."""
+        self.select_params = []
+        self.__server._select_fn = \
+            lambda r, w, e: self.select_wrapper(r, w, e,
+                                                ret=([TEST_FILENO], [], []))
+        self.__server._setup_ccsession()
+        self.__server._run_internal()
+        # select should be called only once.
+        self.assertEqual([([TEST_FILENO], [], [])], self.select_params)
+        # check_command should have been called.
+        self.assertTrue(self.__server.mod_ccsession.check_command_param)
+        # module CC session should have been stopped explicitly.
+        self.assertTrue(self.__server.mod_ccsession.stopped)
+
+    def test_select_interrupted(self):
+        """Emulating case select() raises EINTR."""
+        self.select_params = []
+        self.__server._select_fn = \
+            lambda r, w, e: self.select_wrapper(r, w, e,
+                                                ex=select.error(errno.EINTR))
+        self.__server._setup_ccsession()
+        self.__server._run_internal()
+        # EINTR will be ignored and select() will be called again.
+        self.assertEqual([([TEST_FILENO], [], []), ([TEST_FILENO], [], [])],
+                          self.select_params)
+        # check_command() shouldn't have been called (select_wrapper returns
+        # empty lists by default).
+        self.assertIsNone(self.__server.mod_ccsession.check_command_param)
+        self.assertTrue(self.__server.mod_ccsession.stopped)
+
+    def test_select_other_exception(self):
+        """Emulating case select() raises other select error."""
+        self.select_params = []
+        self.__server._select_fn = \
+            lambda r, w, e: self.select_wrapper(r, w, e,
+                                                ex=select.error(errno.EBADF))
+        self.__server._setup_ccsession()
+        # the exception will be propagated.
+        self.assertRaises(select.error, self.__server._run_internal)
+        self.assertEqual([([TEST_FILENO], [], [])], self.select_params)
+        # in this case module CC session hasn't been stopped explicitly
+        # others will notice it due to connection reset.
+        self.assertFalse(self.__server.mod_ccsession.stopped)
+
+if __name__== "__main__":
+    isc.log.init("bind10_server_test")
+    isc.log.resetUnitTestRootLogger()
+    unittest.main()

src/lib/python/isc/server_common/tests/datasrc_clients_mgr_test.py

@@ -32,6 +32,8 @@ class DataSrcClientsMgrTest(unittest.TestCase):
         # We construct the manager with enabling in-memory cache for easier
         # tests.  There should be no risk of inter-thread issues in the tests.
         self.__mgr = DataSrcClientsMgr(use_cache=True)
+        self.__datasrc_cfg = isc.config.ConfigData(
+            isc.config.module_spec_from_file(DATASRC_SPECFILE))
 
     def test_init(self):
         """Check some initial state.
@@ -52,39 +54,40 @@ class DataSrcClientsMgrTest(unittest.TestCase):
         # There should be at least in-memory only data for the static
         # bind/CH zone. (We don't assume the existence of SQLite3 datasrc,
         # so it'll still work if and when we make the default DB-independent).
-        self.__mgr.reconfigure(DEFAULT_CONFIG)
+        self.__mgr.reconfigure({}, self.__datasrc_cfg)
         clist = self.__mgr.get_client_list(RRClass.CH)
         self.assertIsNotNone(clist)
         self.assertTrue(clist.find(Name('bind'), True, False)[2])
 
         # Reconfigure it with a simple new config: the list for CH will be
         # gone, and and an empty list for IN will be installed.
-        self.__mgr.reconfigure({"classes": {"IN": []}})
+        self.__datasrc_cfg.set_local_config({"classes": {"IN": []}})
+        self.__mgr.reconfigure({}, self.__datasrc_cfg)
         self.assertIsNone(self.__mgr.get_client_list(RRClass.CH))
         self.assertIsNotNone(self.__mgr.get_client_list(RRClass.IN))
 
     def test_reconfigure_error(self):
         """Check reconfigure failure preserves the old config."""
         # Configure it with the default
-        self.__mgr.reconfigure(DEFAULT_CONFIG)
+        self.__mgr.reconfigure({}, self.__datasrc_cfg)
         self.assertIsNotNone(self.__mgr.get_client_list(RRClass.CH))
 
         # Then try invalid configuration
-        self.assertRaises(ConfigError, self.__mgr.reconfigure, 42)
+        self.assertRaises(ConfigError, self.__mgr.reconfigure, {}, 42)
         self.assertIsNotNone(self.__mgr.get_client_list(RRClass.CH))
 
         # Another type of invalid configuration: exception would come from
         # the C++ wrapper.
+        self.__datasrc_cfg.set_local_config({"classes": {"IN": 42}})
         self.assertRaises(ConfigError,
-                          self.__mgr.reconfigure, {"classes": {"IN": 42}})
+                          self.__mgr.reconfigure, {}, self.__datasrc_cfg)
         self.assertIsNotNone(self.__mgr.get_client_list(RRClass.CH))
 
-    def test_reconfig_while_using_old(self):
-        """Check datasrc client and finder can work even after list is gone."""
-        self.__mgr.reconfigure(DEFAULT_CONFIG)
-        clist = self.__mgr.get_client_list(RRClass.CH)
-        self.__mgr.reconfigure({"classes": {"IN": []}})
+    def check_client_list_content(self, clist):
+        """Some set of checks on given data source client list.
 
+        Used by a couple of tests below.
+        """
         datasrc_client, finder, exact = clist.find(Name('bind'))
         self.assertTrue(exact)
 
@@ -104,6 +107,35 @@ class DataSrcClientsMgrTest(unittest.TestCase):
         rrsets = datasrc_client.get_iterator(Name('bind'))
         self.assertNotEqual(0, len(list(rrsets)))
 
+    def test_reconfig_while_using_old(self):
+        """Check datasrc client and finder can work even after list is gone."""
+        self.__mgr.reconfigure({}, self.__datasrc_cfg)
+        clist = self.__mgr.get_client_list(RRClass.CH)
+
+        self.__datasrc_cfg.set_local_config({"classes": {"IN": []}})
+        self.__mgr.reconfigure({}, self.__datasrc_cfg)
+        self.check_client_list_content(clist)
+
+    def test_get_clients_map(self):
+        # This is basically a trivial getter, so it should be sufficient
+        # to check we can call it as we expect.
+
+        # Initially map iss empty, the generation ID is 0.
+        self.assertEqual((0, {}), self.__mgr.get_clients_map())
+
+        self.__mgr.reconfigure({}, self.__datasrc_cfg)
+        genid, clients_map = self.__mgr.get_clients_map()
+        self.assertEqual(1, genid)
+        self.assertEqual(2, len(clients_map)) # should contain 'IN' and 'CH'
+
+        # Check the retrieved map is usable even after further reconfig().
+        self.__datasrc_cfg.set_local_config({"classes": {"IN": []}})
+        self.__mgr.reconfigure({}, self.__datasrc_cfg)
+        self.check_client_list_content(clients_map[RRClass.CH])
+
+        # generation ID should be incremented again
+        self.assertEqual(2, self.__mgr.get_clients_map()[0])
+
 if __name__ == "__main__":
     isc.log.init("bind10")
     isc.log.resetUnitTestRootLogger()