[2467] Merge branch 'master' into trac2467

Conflicts:
	src/bin/dhcp4/tests/dhcp4_srv_unittest.cc
	src/lib/dhcp/tests/iface_mgr_unittest.cc

ChangeLog

@@ -1,13 +1,90 @@
+611.	[func]		naokikambe
+	Added Xfrin statistics items such as the number of successful
+	transfers.  These are per-zone type counters.  Their values can be
+	obtained with zone names by invoking "Stats show Xfrin" via bindctl
+	while Xfrin is running.
+	(Trac #2252, git e1a0ea8ef5c51b9b25afa111fbfe9347afbe5413)
+
+bind10-1.0.0beta2 released on May 3, 2013
+
+610.	[bug]		muks
+	When the sqlite3 program is not available on the system (in
+	PATH), we no longer attempt to run some tests which depend
+	on it.
+	(Trac #1909, git f85b274b85b57a094d33ca06dfbe12ae67bb47df)
+
+609.	[bug]		jinmei
+	Handled some rare error cases in DNS server classes correctly.
+	This fix specifically solves occasional crash of b10-auth due to
+	errors caused by TCP DNS clients.  Also, as a result of cleanups
+	with the fix, b10-auth should now be a little bit faster in
+	handling UDP queries: in some local experiments it ran about 5%
+	faster.
+	(Trac #2903, git 6d3e0f4b36a754248f8a03a29e2c36aef644cdcc)
+
+608.	[bug]		jinmei
+	b10-cmdctl: fixed a hangup problem on receiving the shutdown
+	command from bindctl.  Note, however, that cmdctl is defined as
+	a "needed" module by default, so shutting down cmdctl would cause
+	shutdown of the entire BIND 10 system anyway, and is therefore
+	still not very useful in practice.
+	(Trac #2712, git fa392e8eb391a17d30550d4b290c975710651d98)
+
+607.	[bug]		jinmei
+	Worked around some unit test regressions on FreeBSD 9.1 due to
+	a binary compatibility issue between standard and system
+	libraries (http://www.freebsd.org/cgi/query-pr.cgi?pr=175453).
+	While not all tests still pass, main BIND 10 programs should
+	generally work correctly.  Still, there can be odd run time
+	behavior such as abrupt crash instead of graceful shutdown
+	when some fatal event happens, so it's generally discouraged to
+	use BIND 10 on FreeBSD 9.1 RELEASE.  According to the above
+	bug report for FreeBSD, it seems upgrading or downgrading the
+	FreeBSD version will solve this problem.
+	(Trac #2887, git 69dfb4544d9ded3c10cffbbfd573ae05fdeb771f)
+
+606.	[bug]		jinmei
+	b10-xfrout now correctly stops sending notify requests once it
+	receives a valid response.  It previously handled it as if the
+	requests are timed out and resent it a few times in a short
+	period.
+	(Trac #2879, git 4c45f29f28ae766a9f7dc3142859f1d0000284e1)
+
+605.	[bug]		tmark
+	Modified perfdhcp to calculate the times displayed for packet sent 
+	and received as time elapsed since perfdhcp process start time.  
+	Previously these were times since the start of the epoch.
+	However the large numbers involved caused loss of precision
+	in the calculation of the test statistics.
+	(Trac #2785, git e9556924dcd1cf285dc358c47d65ed7c413e02cf)
+
+604.	[func]		marcin
+	libdhcp++: abstracted methods which open sockets and send/receive
+	DHCP4 packets to a separate class. Other classes will be derived
+	from it to implement OS-specific methods of DHCPv4 packets filtering.
+	The primary purpose for this change is to add support for Direct
+	DHCPv4 response to a client which doesn't have an address yet on
+	different OSes.
+	(Trac #991, git 33ffc9a750cd3fb34158ef676aab6b05df0302e2)
+
+603.	[func]		tmark
+	The directory in which the b10-dchp4 and b10-dhcp6 server id files has
+	been changed from the local state directory (set by the "configure"
+	--localstatedir switch) to the "bind10" subdirectory of it. After an
+	upgrade, server id files in the former location will be orphaned and
+	should be manually removed.
+	(Trac #2770, git a622140d411b3f07a68a1451e19df36118a80650)
+
 602.	[bug]		tmark
-	Perdhcp will now exit gracefully if the command line argument for IP version
-	(-4 or -6) does not match the command line argument given for the server.
-	Prior to this perfdhcp would core when given an IP version of -6 but a valid
-	IPv4 address for server.
+	Perfdhcp will now exit gracefully if the command line argument for
+	IP version (-4 or -6) does not match the command line argument
+	given for the server. Prior to this perfdhcp would core when given
+	an IP version of -6 but a valid IPv4 address for server.
 	(Trac #2784, git 96b66c0c79dccf9a0206a45916b9b23fe9b94f74)
 
 601.	[bug]*		jinmei, vorner
 	The "delete record" interface of the database based data source
-	was extended do that the parameter includes reversed name in
+	was extended so that the parameter includes reversed name in
 	addition to the actual name.  This may help the underlying
 	accessor implementation if reversed names are more convenient
 	for the delete operation.  This was the case for the SQLite3

Makefile.am

@@ -2,7 +2,7 @@ ACLOCAL_AMFLAGS = -I m4macros -I examples/m4 ${ACLOCAL_FLAGS}
 # ^^^^^^^^ This has to be the first line and cannot come later in this
 # Makefile.am due to some bork in some versions of autotools.
 
-SUBDIRS = compatcheck doc . src tests
+SUBDIRS = compatcheck doc . src tests m4macros
 USE_LCOV=@USE_LCOV@
 LCOV=@LCOV@
 GENHTML=@GENHTML@
@@ -46,7 +46,7 @@ endif
 clean-cpp-coverage:
 	@if [ $(USE_LCOV) = yes ] ; then \
 		$(LCOV) --directory . --zerocounters; \
-		rm -rf coverage/; \
+		rm -rf $(abs_top_srcdir)/coverage-cpp-html/; \
 	else \
 		echo "C++ code coverage not enabled at configuration time." ; \
 		echo "Use: ./configure --with-lcov" ; \

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.59])
-AC_INIT(bind10, 20130221, bind10-dev@isc.org)
+AC_INIT(bind10, 20130503, bind10-dev@isc.org)
 AC_CONFIG_SRCDIR(README)
 # serial-tests is not available in automake version before 1.13. In
 # automake 1.13 and higher, AM_PROG_INSTALL is undefined, so we'll check
@@ -388,8 +388,6 @@ In this case we will continue, but naming of python processes will not work.])
     fi
 fi
 
-# TODO: check for _sqlite3.py module
-
 # (g++ only check)
 # Python 3.2 has an unused parameter in one of its headers. This
 # has been reported, but not fixed as of yet, so we check if we need
@@ -884,6 +882,17 @@ if test "$BOOST_NUMERIC_CAST_WOULDFAIL" = "yes" -a X"$werror_ok" = X1 -a $CLANGP
     AC_MSG_ERROR([Failed to compile a required header file.  If you are using FreeBSD and Boost installed via ports, retry with specifying --without-werror.  See the ChangeLog entry for Trac no. 1991 for more details.])
 fi
 
+use_shared_memory=yes
+AC_ARG_WITH(shared-memory,
+    AC_HELP_STRING([--with-shared-memory],
+    [Build with Boost shared memory support; for large scale authoritative DNS servers]),
+    [use_shared_memory=$withval])
+if test X$use_shared_memory = Xyes -a "$BOOST_MAPPED_FILE_WOULDFAIL" = "yes"; then
+    AC_MSG_ERROR([Boost shared memory does not compile on this system.  If you don't need it (most normal users won't) build without it by rerunning this script with --without-shared-memory; using a different compiler or a different version of Boost may also help.])
+fi
+AM_CONDITIONAL([USE_SHARED_MEMORY], [test x$use_shared_memory = xyes])
+AC_SUBST(BOOST_MAPPED_FILE_CXXFLAG)
+
 # Add some default CPP flags needed for Boost, identified by the AX macro.
 CPPFLAGS="$CPPFLAGS $CPPFLAGS_BOOST_THREADCONF"
 
@@ -1032,12 +1041,16 @@ AC_SUBST(GTEST_LDFLAGS)
 AC_SUBST(GTEST_LDADD)
 AC_SUBST(GTEST_SOURCE)
 
-dnl check for pkg-config itself so we don't try the m4 macro without pkg-config
+dnl check for pkg-config itself
 AC_CHECK_PROG(HAVE_PKG_CONFIG, pkg-config, yes, no)
 if test "x$HAVE_PKG_CONFIG" = "xno" ; then
   AC_MSG_ERROR(Please install pkg-config)
 fi
-PKG_CHECK_MODULES(SQLITE, sqlite3 >= 3.3.9, enable_features="$enable_features SQLite3")
+
+AX_SQLITE3_FOR_BIND10
+if test "x$have_sqlite" = "xyes" ; then
+  enable_features="$enable_features SQLite3"
+fi
 
 #
 # ASIO: we extensively use it as the C++ event management module.
@@ -1190,6 +1203,7 @@ AC_CONFIG_FILES([Makefile
                  src/bin/dhcp4/tests/Makefile
                  src/bin/resolver/Makefile
                  src/bin/resolver/tests/Makefile
+                 src/bin/resolver/bench/Makefile
                  src/bin/sysinfo/Makefile
                  src/bin/sockcreator/Makefile
                  src/bin/sockcreator/tests/Makefile
@@ -1311,6 +1325,7 @@ AC_CONFIG_FILES([Makefile
                  tests/tools/perfdhcp/Makefile
                  tests/tools/perfdhcp/tests/Makefile
                  tests/tools/perfdhcp/tests/testdata/Makefile
+                 m4macros/Makefile
                  dns++.pc
                ])
 AC_OUTPUT([doc/version.ent

doc/design/ipc-high.txt

@@ -0,0 +1,382 @@
+The IPC protocol
+================
+
+While the cc-protocol.txt describes the low-level primitives, here we
+describe how the whole IPC should work and how to use it.
+
+Definitions
+-----------
+
+system::
+  The system that moves data between the users and does bookkeeping.
+  In our current implementation, it is implemented as the MsgQ daemon,
+  which the users connect to and it routes the data.
+user::
+  Usually a process; generally an entity that wants to communicate
+  with the other users.
+session::
+  Session is the interface by which the user communicates with the
+  system. Single user may have multiple sessions, a session belongs to
+  single user.
+message::
+  A data blob sent by one user. The recipient might be the system
+  itself, other session or set of sessions (called group, see below,
+  it is possibly empty). Message is either a response or an original
+  message (TODO: Better name?).
+group::
+  A named set of sessions. Conceptually, all the possible groups
+  exist, there's no explicit creation and deletion of groups.
+session id::
+  Unique identifier of a session. It is not reused for the whole
+  lifetime of the system. Historically called `lname` in the code.
+undelivery signal::
+  While sending an original message, a client may request an
+  undelivery signal. If the recipient specification yields no
+  sessions to deliver the message to, the system informs user about
+  the situation.
+sequence number::
+  Each message sent through the system carries a sequence number. The
+  number should be unique per sender. It can be used to pair a
+  response to the original message, since the response specifies which
+  sequence number had the message it response to. Even responses and
+  messages not expecting answer have their sequence number, but it is
+  generally unused.
+non-blocking operation::
+  Operation that will complete without waiting for anything.
+fast operation::
+  Operation that may wait for other process, but only for a very short
+  time. Generally, this includes communication between the user and
+  system, but not between two clients. It can be expected to be fast
+  enough to use this inside an interactive session, but may be too
+  heavy in the middle of query processing, for example. Every
+  non-blocking operation is considered fast.
+
+The session
+-----------
+
+The session interface allows for several operations interacting with
+the system. In the code, it is represented by a class.
+
+Possible operations include:
+
+Opening a session::
+  The session is created and connects to the system. This operation is
+  fast. The session receives session id from the system.
+
+Group management::
+  A user may subscribe (become member) of a group, or unsubscribe from
+  a group. These are fast operations.
+
+Send::
+  A user may send a message, addressed to the system, or other
+  session(s). This operation is expected to be non-blocking
+  (current implementation is based on assumption of how OS handles the
+  sends, which may need to be revisited if it turns out to be false).
+
+Receive synchronously::
+  User may wait for an incoming message in blocking mode. It is
+  possible to specify the kind of message to wait for, either original
+  message or response to a message. This interface has a timeout.
+
+Receive asynchronously::
+  Similar to previous, but non-blocking. It terminates immediately.
+  The user provides a callback that is invoked when the requested
+  message arrives.
+
+Terminate::
+  A session may be terminated. No more messages are sent or received
+  over it, the session is automatically unsubscribed from all the
+  groups. This operation is non-blocking. A session is terminated
+  automatically if the user exits.
+
+Assumptions
+-----------
+
+We assume reliability and order of delivery. Messages sent from user A
+to B are all delivered unchanged in original order as long as B
+exists.
+
+All above operations are expected to always succeed. If there's an
+error reported, it should be considered fatal and user should
+exit. In case a user still wants to continue, the session must be
+considered terminated and a new one must be created. Care must be
+taken not to use any information obtained from the previous session,
+since the state in other users and the system may have changed during
+the reconnect.
+
+Addressing
+----------
+
+Addressing happens in three ways:
+
+By group name::
+  The message is routed to all the sessions subscribed to this group.
+  It is legal to address an empty group; such message is then
+  delivered to no sessions.
+By session ID::
+  The message is sent to the single session, if it is still alive.
+By an alias::
+  A session may have any number of aliases - well known names. Only
+  single session may hold given alias (but it is not yet enforced by
+  the system). The message is delivered to the one session owning the
+  alias, if any. Internally, the aliases are implemented as groups
+  with single subscribed session, so it is the same as the first
+  option on the protocol level, but semantically it is different.
+
+The system
+----------
+
+The system performs these goals:
+
+ * Maintains the open sessions and allows creating new ones.
+ * Keeps information about groups and which sessions are subscribed to
+   which group.
+ * Routes the messages between users.
+
+Also, the system itself is a user of the system. It can be reached by
+the alias `Msgq` and provides following high-level services (see
+below):
+
+Notifications about sessions::
+  When a session is opened to the system or when a session is
+  terminated, a notification is sent to interested users. The
+  notification contains the session ID of the session in question.
+  The termination notification is probably more useful (if a user
+  communicated with a given session before, it might be interested it
+  is no longer available), the opening notification is provided mostly
+  for completeness.
+Notifications about group subscriptions::
+  When a session subscribes to a group or unsubscribes from a group, a
+  notification is sent to interested users. The notification contains
+  both the session ID of the session subscribing/unsubscribing and
+  name of the group. This includes notifications about aliases (since
+  aliases are groups internally).
+Commands to list sessions::
+  There's a command to list session IDs of all currently opened sessions
+  and a command to list session IDs of all sessions subscribed to a
+  given group. Note that using these lists might need some care, as
+  the information might be outdated at the time it is delivered to the
+  user.
+
+User shows interest in notifications about sessions and group
+subscriptions by subscribing to a group with well-known name (as with
+any notification).
+
+Note that due to implementation details, the `Msgq` alias is not yet
+available during early stage of the bootstrap of bind10 system. This
+means some very core services can't rely on the above services of the
+system. The alias is guaranteed to be working before the first
+non-core module is started.
+
+Higher-level services
+---------------------
+
+While the system is able to send any kind of data, the payload sent by
+users in bind10 is structured data encoded as JSON. The messages sent
+are of three general types:
+
+Command::
+  A message sent to single destination, with the undeliverable
+  signal turned on and expecting an answer. This is a request
+  to perform some operation on the recipient (it can have side effects
+  or not). The command is identified by a name and it can have
+  parameters. A command with the same name may behave differently (or
+  have different parameters) on different receiving users.
+Reply::
+  An answer to the `Command`. It is sent directly to the session where
+  the command originated from, does not expect further answer and the
+  undeliverable notification is not set. It either confirms the
+  command was run successfully and contains an optional result, or
+  notifies the sender of failure to run the command. Success and
+  failure differ only in the payload sent through the system, not in
+  the way it is sent. The undeliverable signal is failure
+  reply sent by the system on behalf of the missing recipient.
+Notification::
+  A message sent to any number of destinations (eg. sent to a group),
+  not expecting an answer. It notifies other users about an event or
+  change of state.
+
+Details of the higher-level
+---------------------------
+
+While there are libraries implementing the communication in convenient
+way, it is useful to know what happens inside.
+
+The notifications are probably the simplest. Users interested in
+receiving notifications of some family subscribe to corresponding
+group. Then, a client sends a message to the group. For example, if
+clients `receiver-A` and `receiver-B` want to receive notifications
+about changes to zone data, they'd subscribe to the
+`Notifications/ZoneUpdates` group. Then, other client (let's say
+`XfrIn`, with session ID `s12345`) would send something like:
+
+  s12345 -> Notifications/ZoneUpdates
+  {"notification": ["zone-update", {
+      "class": "IN",
+      "origin": "example.org.",
+      "serial": 123456
+  }]}
+
+Both receivers would receive the message and know that the
+`example.org` zone is now at version 123456. Note that multiple users
+may produce the same kind of notification. Also, single group may be
+used to send multiple notification names (but they should be related;
+in our example, the `Notifications/ZoneUpdates` could be used for
+`zone-update`, `zone-available` and `zone-unavailable` notifications
+for change in zone data, configuration of new zone in the system and
+removal of a zone from configuration).
+
+Sending a command to single recipient is slightly more complex. The
+sending user sends a message to the receiving one, addressed either by
+session ID or by an alias (group to which at most one session may be
+subscribed). The message contains the name of the command and
+parameters. It is sent with the undeliverable signals turned on.
+The user also starts a timer (with reasonably long timeout). The
+sender also subscribes to notifications about terminated sessions or
+unsubscription from the alias group.
+
+The receiving user gets the message, runs the command and sends a
+response back, with the result. The response has the undeliverable
+signal turned off and it is marked as response to the message
+containing the command. The sending user receives the answer and pairs
+it with the command.
+
+There are several things that may go wrong.
+
+* There might be an error on the receiving user (bad parameters, the
+  operation failed, the recipient doesn't know command of that name).
+  The receiving side sends the response as previous, the only
+  difference is the content of the payload. The sending user is
+  notified about it, without delays.
+* The recipient user doesn't exist (either the session ID is wrong or
+  terminated already, or the alias is empty). The system sends a
+  failure response and the sending user knows immediately the command
+  failed.
+* The recipient disconnects while processing the command (possibly
+  crashes). The sender gets a notification about disconnection or
+  unsubscription from the alias group and knows the answer won't come.
+* The recipient ``blackholes'' the command. It receives it, but never
+  answers. The timeout in sender times out. As this is a serious
+  programmer error in the recipient and should be rare, the sender
+  should at least log an error to notify about the case.
+
+One example would be asking the question of life, universe and
+everything (all the examples assume the sending user is already
+subscribed to the notifications):
+
+  s12345 -> DeepThought
+  {"command": ["question", {
+      "what": ["Life", "Universe", "*"]
+  }]}
+  s23456 -> s12345
+  {"reply": [0, 42]}
+
+The deep thought had an alias. But the answer is sent from its session
+ID. The `0` in the reply means ``success''.
+
+Another example might be asking for some data at a bureau and getting
+an error:
+
+  s12345 -> Burreau
+  {"command": ["provide-information", {
+      "about": "me",
+      "topic": "taxes"
+  }]}
+  s23456 -> s12345
+  {"reply": [1, "You need to fill in other form"]}
+
+And, in this example, the sender is trying to reach an non-existent
+session. The `msgq` here is not the alias `Msgq`, but a special
+``phantom'' session ID that is not listed anywhere.
+
+  s12345 -> s0
+  {"command": ["ping"]}
+  msgq -> s12345
+  {"reply": [-1, "No such recipient"]}
+
+Last, an example when the other user disconnects while processing the
+command.
+
+  s12345 -> s23456
+  {"command": ["shutdown"]}
+  msgq -> s12345
+  {"notification": ["disconnected", {
+    "lname": "s23456"
+  }]}
+
+The system does not support sending a command to multiple users
+directly. It can be accomplished as this:
+
+* The sending user calls a command on the system to get list of
+  sessions in given group. This is command to alias, so it can be done
+  by the previous way.
+* After receiving the list of session IDs, multiple copies of the
+  command are sent by the sending user, one to each of the session
+  IDs.
+* Successes and failures are handled the same as above, since these
+  are just single-recipient commands.
+
+So, this would be an example with unhelpful war council.
+
+  s12345 -> Msgq
+  {"command": ["get-subscriptions", {
+      "group": "WarCouncil"
+  }]}
+  msgq -> s12345
+  {"reply": [0, ["s1", "s2", "s3"]]}
+  s12345 -> s1
+  {"command": ["advice", {
+      "topic": "Should we attack?"
+  }]}
+  s12345 -> s2
+  {"command": ["advice", {
+      "topic": "Should we attack?"
+  }]}
+  s12345 -> s3
+  {"command": ["advice", {
+      "topic": "Should we attack?"
+  }]}
+  s1 -> s12345
+  {"reply": [0, true]}
+  s2 -> s12345
+  {"reply": [0, false]}
+  s3 -> s12345
+  {"reply": [1, "Advice feature not implemented"]}
+
+Users
+-----
+
+While there's a lot of flexibility for the behaviour of a user, it
+usually comes to something like this (during the lifetime of the
+user):
+
+* The user starts up.
+* Then it creates one or more sessions (there may be technical reasons
+  to have more than one session, such as threads, but it is not
+  required by the system).
+* It subscribes to some groups to receive notifications in future.
+* It binds to some aliases if it wants to be reachable by others by a
+  nice name.
+* It invokes some start-up commands (to get the configuration, for
+  example).
+* During the lifetime, it listens for notifications and answers
+  commands. It also invokes remote commands and sends notifications
+  about things that are happening.
+* Eventually, the user terminates, closing all the sessions it had
+  opened.
+
+Known limitations
+-----------------
+
+It is meant mostly as signalling protocol. Sending millions of
+messages or messages of several tens of megabytes is probably a bad
+idea. While there's no architectural limitation with regards of the
+number of transferred messages and the maximum size of message is 4GB,
+the code is not optimised and it would probably be very slow.
+
+We currently expect the system not to be at heavy load. Therefore, we
+expect the system to keep up with users sending messages. The
+libraries write in blocking mode, which is no problem if the
+expectation is true, as the write buffers will generally be empty and
+the write wouldn't block, but if it turns out it is not the case, we
+might need to reconsider.

m4macros/Makefile.am

@@ -0,0 +1,2 @@
+EXTRA_DIST  = ax_boost_for_bind10.m4
+EXTRA_DIST += ax_sqlite3_for_bind10.m4

m4macros/ax_boost_for_bind10.m4

@@ -23,7 +23,11 @@ dnl   BOOST_OFFSET_PTR_WOULDFAIL set to "yes" if offset_ptr would cause build
 dnl                              error; otherwise set to "no"
 dnl   BOOST_NUMERIC_CAST_WOULDFAIL set to "yes" if numeric_cast would cause
 dnl                                build error; otherwise set to "no"
-dnl
+dnl   BOOST_MAPPED_FILE_WOULDFAIL set to "yes" if managed_mapped_file would
+dnl                               cause build failure; otherwise set to "no"
+dnl   BOOST_MAPPED_FILE_CXXFLAG set to the compiler flag that would need to
+dnl                             compile managed_mapped_file (can be empty).
+dnl                             It is of no use if "WOULDFAIL" is yes.
 
 AC_DEFUN([AX_BOOST_FOR_BIND10], [
 AC_LANG_SAVE
@@ -101,10 +105,49 @@ if test "X$GXX" = "Xyes"; then
 
    CXXFLAGS="$CXXFLAGS_SAVED"
 else
-  # This doesn't matter for non-g++
-  BOOST_NUMERIC_CAST_WOULDFAIL=no
+   # This doesn't matter for non-g++
+   BOOST_NUMERIC_CAST_WOULDFAIL=no
 fi
 
+# Boost interprocess::managed_mapped_file is highly system dependent and
+# can cause many portability issues.  We are going to check if it could
+# compile at all, possibly with being lenient about compiler warnings.
+BOOST_MAPPED_FILE_WOULDFAIL=yes
+BOOST_MAPPED_FILE_CXXFLAG=
+CXXFLAGS_SAVED="$CXXFLAGS"
+try_flags="no"
+if test "X$GXX" = "Xyes"; then
+  CXXFLAGS="$CXXFLAGS -Wall -Wextra -Werror"
+  try_flags="$try_flags -Wno-error"
+fi
+# clang can cause false positives with -Werror without -Qunused-arguments
+AC_CHECK_DECL([__clang__], [CXXFLAGS="$CXXFLAGS -Qunused-arguments"], [])
+
+AC_MSG_CHECKING([Boost managed_mapped_file compiles])
+CXXFLAGS_SAVED2="$CXXFLAGS"
+for flag in $try_flags; do
+  if test "$flag" != no; then
+    BOOST_MAPPED_FILE_CXXFLAG="$flag"
+  fi
+  CXXFLAGS="$CXXFLAGS $BOOST_MAPPED_FILE_CXXFLAG"
+  AC_TRY_COMPILE([
+  #include <boost/interprocess/managed_mapped_file.hpp>
+  ],[
+  return (boost::interprocess::managed_mapped_file().all_memory_deallocated());
+  ],[AC_MSG_RESULT([yes, with $flag flag])
+     BOOST_MAPPED_FILE_WOULDFAIL=no
+     break
+  ],[])
+
+  CXXFLAGS="$CXXFLAGS_SAVED2"
+done
+
+if test $BOOST_MAPPED_FILE_WOULDFAIL = yes; then
+  AC_MSG_RESULT(no)
+fi
+
+CXXFLAGS="$CXXFLAGS_SAVED"
+
 AC_SUBST(BOOST_INCLUDES)
 
 CPPFLAGS="$CPPFLAGS_SAVED"

m4macros/ax_sqlite3_for_bind10.m4

@@ -0,0 +1,25 @@
+dnl @synopsis AX_SQLITE3_FOR_BIND10
+dnl
+dnl Test for the sqlite3 library and program, intended to be used within
+dnl BIND 10, and to test BIND 10.
+dnl
+dnl We use pkg-config to look for the sqlite3 library, so the sqlite3
+dnl development package with the .pc file must be installed.
+dnl
+dnl This macro sets SQLITE_CFLAGS and SQLITE_LIBS. It also sets
+dnl SQLITE3_PROGRAM to the path of the sqlite3 program, if it is found
+dnl in PATH.
+
+AC_DEFUN([AX_SQLITE3_FOR_BIND10], [
+
+PKG_CHECK_MODULES(SQLITE, sqlite3 >= 3.3.9,
+    have_sqlite="yes",
+    have_sqlite="no (sqlite3 not detected)")
+
+# Check for sqlite3 program
+AC_PATH_PROG(SQLITE3_PROGRAM, sqlite3, no)
+AM_CONDITIONAL(HAVE_SQLITE3_PROGRAM, test "x$SQLITE3_PROGRAM" != "xno")
+
+# TODO: check for _sqlite3.py module
+
+])dnl AX_SQLITE3_FOR_BIND10

src/bin/auth/auth_srv.cc

@@ -42,7 +42,7 @@
 
 #include <asiodns/dns_service.h>
 
-#include <datasrc/data_source.h>
+#include <datasrc/exceptions.h>
 #include <datasrc/client_list.h>
 
 #include <xfr/xfrout_client.h>

src/bin/auth/datasrc_clients_mgr.h

@@ -25,7 +25,7 @@
 
 #include <cc/data.h>
 
-#include <datasrc/data_source.h>
+#include <datasrc/exceptions.h>
 #include <datasrc/client_list.h>
 #include <datasrc/memory/zone_writer.h>
 

src/bin/auth/tests/config_unittest.cc

@@ -21,7 +21,7 @@
 
 #include <cc/data.h>
 
-#include <datasrc/data_source.h>
+#include <datasrc/exceptions.h>
 
 #include <xfr/xfrout_client.h>
 

src/bin/cfgmgr/plugins/datasrc.spec.pre.in

@@ -68,6 +68,12 @@
                                 "item_name": "name",
                                 "item_type": "string",
                                 "item_optional": true
+                            },
+                            {
+                                "item_name": "cache-type",
+                                "item_type": "string",
+                                "item_optional": true,
+                                "item_default": "local"
                             }
                         ]
                     }

src/bin/cmdctl/cmdctl.py.in

@@ -247,6 +247,7 @@ class CommandControl():
         CommandControl to communicate with other modules. '''
         self._verbose = verbose
         self._httpserver = httpserver
+        self.__msg_handler_thread = None # set in _start_msg_handle_thread
         self._lock = threading.Lock()
         self._setup_session()
         self.modules_spec = self._get_modules_specification()
@@ -326,7 +327,26 @@ class CommandControl():
                     self._cmdctl_config_data[key] = new_config[key]
         return answer
 
+    def _get_current_thread(self):
+        """A simple wrapper of returning the 'current' thread object.
+
+        This is extracted as a 'protected' method so tests can override for
+        their convenience.
+
+        """
+        return threading.currentThread()
+
     def command_handler(self, command, args):
+        """Handle commands from other modules.
+
+        This method must not be called by any other threads than
+        __msg_handler_thread invoked at the intialization of the class;
+        otherwise it would cause critical race or dead locks.
+
+        """
+        # Check the restriction described above.
+        assert self._get_current_thread() == self.__msg_handler_thread
+
         answer = ccsession.create_answer(0)
         if command == ccsession.COMMAND_MODULE_SPECIFICATION_UPDATE:
             # The 'value' of a specification update can be either
@@ -362,6 +382,7 @@ class CommandControl():
         ''' Start one thread to handle received message from msgq.'''
         td = threading.Thread(target=self._handle_msg_from_msgq)
         td.daemon = True
+        self.__msg_handler_thread = td
         td.start()
 
     def _handle_msg_from_msgq(self):
@@ -402,7 +423,7 @@ class CommandControl():
         rcode, reply = self.send_command('ConfigManager', ccsession.COMMAND_GET_MODULE_SPEC)
         return self._parse_command_result(rcode, reply)
 
-    def send_command_with_check(self, module_name, command_name, params = None):
+    def send_command_with_check(self, module_name, command_name, params=None):
         '''Before send the command to modules, check if module_name, command_name
         parameters are legal according the spec file of the module.
         Return rcode, dict. TODO, the rcode should be defined properly.
@@ -424,31 +445,34 @@ class CommandControl():
 
         return self.send_command(module_name, command_name, params)
 
-    def send_command(self, module_name, command_name, params = None):
-        '''Send the command from bindctl to proper module. '''
+    def send_command(self, module_name, command_name, params=None):
+        """Send the command from bindctl to proper module.
+
+        Note that commands sent to Cmdctl itself are also delivered via the
+        CC session.  Since this method is called from a thread handling a
+        particular HTTP session, it cannot directly call command_handler().
+
+        """
         errstr = 'unknown error'
         answer = None
         logger.debug(DBG_CMDCTL_MESSAGING, CMDCTL_SEND_COMMAND,
                      command_name, module_name)
 
-        if module_name == self._module_name:
-            # Process the command sent to cmdctl directly.
-            answer = self.command_handler(command_name, params)
-        else:
-            # FIXME: Due to the fact that we use a separate session
-            # from the module one (due to threads and blocking), and
-            # because the plain cc session does not have the high-level
-            # rpc-call method, we use the low-level way and create the
-            # command ourselves.
-            msg = ccsession.create_command(command_name, params)
-            seq = self._cc.group_sendmsg(msg, module_name, want_answer=True)
-            logger.debug(DBG_CMDCTL_MESSAGING, CMDCTL_COMMAND_SENT,
-                         command_name, module_name)
-            #TODO, it may be blocked, msqg need to add a new interface waiting in timeout.
-            try:
-                answer, env = self._cc.group_recvmsg(False, seq)
-            except isc.cc.session.SessionTimeout:
-                errstr = "Module '%s' not responding" % module_name
+        # FIXME: Due to the fact that we use a separate session
+        # from the module one (due to threads and blocking), and
+        # because the plain cc session does not have the high-level
+        # rpc-call method, we use the low-level way and create the
+        # command ourselves.
+        msg = ccsession.create_command(command_name, params)
+        seq = self._cc.group_sendmsg(msg, module_name, want_answer=True)
+        logger.debug(DBG_CMDCTL_MESSAGING, CMDCTL_COMMAND_SENT, command_name,
+                     module_name)
+        # TODO, it may be blocked, msqg need to add a new interface waiting
+        # in timeout.
+        try:
+            answer, env = self._cc.group_recvmsg(False, seq)
+        except isc.cc.session.SessionTimeout:
+            errstr = "Module '%s' not responding" % module_name
 
         if answer:
             try:

src/bin/cmdctl/tests/cmdctl_test.py

@@ -373,7 +373,30 @@ class TestSecureHTTPRequestHandler(unittest.TestCase):
             self.handler._is_user_logged_in = orig_is_user_logged_in
             self.handler._check_user_name_and_pwd = orig_check_user_name_and_pwd
 
+class MockSession:
+    """Act like isc.cc.Session, stealing group_sendmsg/recvmsg().
+
+    The initial simple version only records given parameters in
+    group_sendmsg() for later inspection and raise a timeout exception
+    from recvmsg().  As we see the need for more test cases these methods
+    should be extended.
+
+    """
+    def __init__(self, sent_messages):
+        self.__sent_messages = sent_messages
+
+    def group_sendmsg(self, msg, module_name, want_answer):
+        self.__sent_messages.append((msg, module_name))
+
+    def group_recvmsg(self, nonblock, seq):
+        raise isc.cc.session.SessionTimeout('dummy timeout')
+
 class MyCommandControl(CommandControl):
+    def __init__(self, httpserver, verbose):
+        super().__init__(httpserver, verbose)
+        self.sent_messages = [] # for inspection; allow tests to see it
+        self._cc = MockSession(self.sent_messages)
+
     def _get_modules_specification(self):
         return {}
 
@@ -390,6 +413,12 @@ class MyCommandControl(CommandControl):
     def _handle_msg_from_msgq(self):
         pass
 
+    def _start_msg_handle_thread(self): # just not bother to be threads
+        pass
+
+    def _get_current_thread(self):
+        return None
+
 class TestCommandControl(unittest.TestCase):
 
     def setUp(self):
@@ -502,8 +531,24 @@ class TestCommandControl(unittest.TestCase):
         os.remove(file_name)
 
     def test_send_command(self):
-        rcode, value = self.cmdctl.send_command('Cmdctl', 'print_settings', None)
-        self.assertEqual(rcode, 0)
+        # Send a command to other module.  We check an expected message
+        # is sent via the session (cmdct._cc).  Due to the behavior of
+        # our mock session object the anser will be "fail", but it's not
+        # the subject of this test, and so it's okay.
+        # TODO: more detailed cases should be tested.
+        rcode, value = self.cmdctl.send_command('Init', 'shutdown', None)
+        self.assertEqual(1, len(self.cmdctl.sent_messages))
+        self.assertEqual(({'command': ['shutdown']}, 'Init'),
+                         self.cmdctl.sent_messages[-1])
+        self.assertEqual(1, rcode)
+
+        # Send a command to cmdctl itself.  Should be the same effect.
+        rcode, value = self.cmdctl.send_command('Cmdctl', 'print_settings',
+                                                None)
+        self.assertEqual(2, len(self.cmdctl.sent_messages))
+        self.assertEqual(({'command': ['print_settings']}, 'Cmdctl'),
+                         self.cmdctl.sent_messages[-1])
+        self.assertEqual(1, rcode)
 
 class MySecureHTTPServer(SecureHTTPServer):
     def server_bind(self):

src/bin/dbutil/tests/Makefile.am

@@ -5,5 +5,11 @@ SUBDIRS = . testdata
 noinst_SCRIPTS = dbutil_test.sh
 
 check-local:
+if HAVE_SQLITE3_PROGRAM
 	B10_LOCKFILE_DIR_FROM_BUILD=$(abs_top_builddir) \
 	$(SHELL) $(abs_builddir)/dbutil_test.sh
+else
+	@echo ""
+	@echo " **** The sqlite3 program is required to run dbutil tests **** "
+	@echo ""
+endif

src/bin/dhcp4/dhcp4_srv.cc

@@ -57,7 +57,7 @@ static const char* SERVER_ID_FILE = "b10-dhcp4-serverid";
 // These are hardcoded parameters. Currently this is a skeleton server that only
 // grants those options and a single, fixed, hardcoded lease.
 
-Dhcpv4Srv::Dhcpv4Srv(uint16_t port, const char* dbconfig) {
+Dhcpv4Srv::Dhcpv4Srv(uint16_t port, const char* dbconfig, const bool use_bcast) {
     LOG_DEBUG(dhcp4_logger, DBG_DHCP4_START, DHCP4_OPEN_SOCKET).arg(port);
     try {
         // First call to instance() will create IfaceMgr (it's a singleton)
@@ -67,7 +67,7 @@ Dhcpv4Srv::Dhcpv4Srv(uint16_t port, const char* dbconfig) {
         if (port) {
             // open sockets only if port is non-zero. Port 0 is used
             // for non-socket related testing.
-            IfaceMgr::instance().openSockets4(port);
+            IfaceMgr::instance().openSockets4(port, use_bcast);
         }
 
         string srvid_file = CfgMgr::instance().getDataDir() + "/" + string(SERVER_ID_FILE);
@@ -287,9 +287,9 @@ Dhcpv4Srv::generateServerID() {
             continue;
         }
 
-        const IfaceMgr::AddressCollection addrs = iface->getAddresses();
+        const Iface::AddressCollection addrs = iface->getAddresses();
 
-        for (IfaceMgr::AddressCollection::const_iterator addr = addrs.begin();
+        for (Iface::AddressCollection::const_iterator addr = addrs.begin();
              addr != addrs.end(); ++addr) {
             if (addr->getFamily() != AF_INET) {
                 continue;
@@ -317,7 +317,7 @@ Dhcpv4Srv::writeServerID(const std::string& file_name) {
     return (true);
 }
 
-string 
+string
 Dhcpv4Srv::srvidToString(const OptionPtr& srvid) {
     if (!srvid) {
         isc_throw(BadValue, "NULL pointer passed to srvidToString()");
@@ -517,6 +517,28 @@ Dhcpv4Srv::assignLease(const Pkt4Ptr& question, Pkt4Ptr& answer) {
 
         answer->setYiaddr(lease->addr_);
 
+        // If remote address is not set, we are dealing with a directly
+        // connected client requesting new lease. We can send response to
+        // the address assigned in the lease, but first we have to make sure
+        // that IfaceMgr supports responding directly to the client when
+        // client doesn't have address assigned to its interface yet.
+        if (answer->getRemoteAddr().toText() == "0.0.0.0") {
+            if (IfaceMgr::instance().isDirectResponseSupported()) {
+                answer->setRemoteAddr(lease->addr_);
+            } else {
+                // Since IfaceMgr does not support direct responses to
+                // clients not having IP addresses, we have to send response
+                // to broadcast. We don't check whether the use_bcast flag
+                // was set in the constructor, because this flag is only used
+                // by unit tests to prevent opening broadcast sockets, as
+                // it requires root privileges. If this function is invoked by
+                // unit tests, we expect that it sets broadcast address if
+                // direct response is not supported, so as a test can verify
+                // function's behavior, regardless of the use_bcast flag's value.
+                answer->setRemoteAddr(IOAddress("255.255.255.255"));
+            }
+        }
+
         // IP Address Lease time (type 51)
         opt = OptionPtr(new Option(Option::V4, DHO_DHCP_LEASE_TIME));
         opt->setUint32(lease->valid_lft_);

src/bin/dhcp4/dhcp4_srv.h

@@ -66,8 +66,10 @@ class Dhcpv4Srv : public boost::noncopyable {
     /// @param port specifies port number to listen on
     /// @param dbconfig Lease manager configuration string.  The default
     ///        of the "memfile" manager is used for testing.
+    /// @param use_bcast configure sockets to support broadcast messages.
     Dhcpv4Srv(uint16_t port = DHCP4_SERVER_PORT,
-              const char* dbconfig = "type=memfile");
+              const char* dbconfig = "type=memfile",
+              const bool use_bcast = true);
 
     /// @brief Destructor. Used during DHCPv4 service shutdown.
     ~Dhcpv4Srv();

src/bin/dhcp4/tests/dhcp4_srv_unittest.cc

@@ -17,6 +17,7 @@
 
 #include <asiolink/io_address.h>
 #include <dhcp/dhcp4.h>
+#include <dhcp/iface_mgr.h>
 #include <dhcp/option.h>
 #include <dhcp/option4_addrlst.h>
 #include <dhcp/option_custom.h>
@@ -46,7 +47,16 @@ namespace {
 class NakedDhcpv4Srv: public Dhcpv4Srv {
     // "Naked" DHCPv4 server, exposes internal fields
 public:
-    NakedDhcpv4Srv(uint16_t port = 0):Dhcpv4Srv(port) { }
+
+    /// @brief Constructor.
+    ///
+    /// It disables configuration of broadcast options on
+    /// sockets that are opened by the Dhcpv4Srv constructor.
+    /// Setting broadcast options requires root privileges
+    /// which is not the case when running unit tests.
+    NakedDhcpv4Srv(uint16_t port = 0)
+        : Dhcpv4Srv(port, "type=memfile", false) {
+    }
 
     using Dhcpv4Srv::processDiscover;
     using Dhcpv4Srv::processRequest;
@@ -170,6 +180,8 @@ public:
         EXPECT_TRUE(a->getOption(DHO_DHCP_SERVER_IDENTIFIER));
         EXPECT_TRUE(a->getOption(DHO_DHCP_LEASE_TIME));
         EXPECT_TRUE(a->getOption(DHO_SUBNET_MASK));
+        EXPECT_TRUE(a->getOption(DHO_DOMAIN_NAME));
+        EXPECT_TRUE(a->getOption(DHO_DOMAIN_NAME_SERVERS));
 
         // Check that something is offered
         EXPECT_TRUE(a->getYiaddr().toText() != "0.0.0.0");
@@ -345,6 +357,120 @@ public:
         EXPECT_TRUE(expected_clientid->getData() == opt->getData());
     }
 
+    /// @brief Tests if Discover or Request message is processed correctly
+    ///
+    /// @param msg_type DHCPDISCOVER or DHCPREQUEST
+    /// @param client_addr client address
+    /// @param relay_addr relay address
+    void testDiscoverRequest(const uint8_t msg_type,
+                             const IOAddress& client_addr,
+                             const IOAddress& relay_addr) {
+
+        boost::scoped_ptr<NakedDhcpv4Srv> srv(new NakedDhcpv4Srv(0));
+        vector<uint8_t> mac(6);
+        for (int i = 0; i < 6; i++) {
+            mac[i] = i*10;
+        }
+
+        boost::shared_ptr<Pkt4> req(new Pkt4(msg_type, 1234));
+        boost::shared_ptr<Pkt4> rsp;
+
+        req->setIface("eth0");
+        req->setIndex(17);
+        req->setHWAddr(1, 6, mac);
+        req->setRemoteAddr(IOAddress(client_addr));
+        req->setGiaddr(relay_addr);
+
+        // We are going to test that certain options are returned
+        // in the response message when requested using 'Parameter
+        // Request List' option. Let's configure those options that
+        // are returned when requested.
+        configureRequestedOptions();
+
+        if (msg_type == DHCPDISCOVER) {
+            ASSERT_NO_THROW(
+                rsp = srv->processDiscover(req);
+            );
+
+            // Should return OFFER
+            ASSERT_TRUE(rsp);
+            EXPECT_EQ(DHCPOFFER, rsp->getType());
+
+        } else {
+            ASSERT_NO_THROW(
+                rsp = srv->processRequest(req);
+            );
+
+            // Should return ACK
+            ASSERT_TRUE(rsp);
+            EXPECT_EQ(DHCPACK, rsp->getType());
+
+        }
+
+        if (relay_addr.toText() != "0.0.0.0") {
+            // This is relayed message. It should be sent brsp to relay address.
+            EXPECT_EQ(req->getGiaddr().toText(),
+                      rsp->getRemoteAddr().toText());
+
+        } else if (client_addr.toText() != "0.0.0.0") {
+            // This is a message from a client having an IP address.
+            EXPECT_EQ(req->getRemoteAddr().toText(),
+                      rsp->getRemoteAddr().toText());
+
+        } else {
+            // This is a message from a client having no IP address yet.
+            // If IfaceMgr supports direct traffic the response should
+            // be sent to the new address assigned to the client.
+            if (IfaceMgr::instance().isDirectResponseSupported()) {
+                EXPECT_EQ(rsp->getYiaddr(),
+                          rsp->getRemoteAddr().toText());
+
+            // If direct response to the client having no IP address is
+            // not supported, response should go to broadcast.
+            } else {
+                EXPECT_EQ("255.255.255.255", rsp->getRemoteAddr().toText());
+
+            }
+
+        }
+
+        messageCheck(req, rsp);
+
+        // We did not request any options so these should not be present
+        // in the RSP.
+        EXPECT_FALSE(rsp->getOption(DHO_LOG_SERVERS));
+        EXPECT_FALSE(rsp->getOption(DHO_COOKIE_SERVERS));
+        EXPECT_FALSE(rsp->getOption(DHO_LPR_SERVERS));
+
+        // Repeat the test but request some options.
+        // Add 'Parameter Request List' option.
+        addPrlOption(req);
+
+        if (msg_type == DHCPDISCOVER) {
+            ASSERT_NO_THROW(
+                rsp = srv->processDiscover(req);
+            );
+
+            // Should return non-NULL packet.
+            ASSERT_TRUE(rsp);
+            EXPECT_EQ(DHCPOFFER, rsp->getType());
+
+        } else {
+            ASSERT_NO_THROW(
+                rsp = srv->processRequest(req);
+            );
+
+            // Should return non-NULL packet.
+            ASSERT_TRUE(rsp);
+            EXPECT_EQ(DHCPACK, rsp->getType());
+
+        }
+
+        // Check that the requested options are returned.
+        optionsCheck(rsp);
+
+    }
+
     ~Dhcpv4SrvTest() {
         CfgMgr::instance().deleteSubnets4();
 
@@ -381,7 +507,7 @@ TEST_F(Dhcpv4SrvTest, basic) {
     EXPECT_TRUE(naked_srv->getServerID());
 }
 
-// Verifies that received DISCOVER can be processed correctly,
+// Verifies that DISCOVER received via relay can be processed correctly,
 // that the OFFER message generated in response is valid and
 // contains necessary options.
 //
@@ -389,199 +515,56 @@ TEST_F(Dhcpv4SrvTest, basic) {
 // are other tests that verify correctness of the allocation
 // engine. See DiscoverBasic, DiscoverHint, DiscoverNoClientId
 // and DiscoverInvalidHint.
-TEST_F(Dhcpv4SrvTest, processDiscover) {
-    boost::scoped_ptr<NakedDhcpv4Srv> srv(new NakedDhcpv4Srv(0));
-    vector<uint8_t> mac(6);
-    for (int i = 0; i < 6; i++) {
-        mac[i] = 255 - i;
-    }
-
-    Pkt4Ptr pkt(new Pkt4(DHCPDISCOVER, 1234));
-    Pkt4Ptr offer;
-
-    pkt->setIface("eth0");
-    pkt->setIndex(17);
-    pkt->setHWAddr(1, 6, mac);
-    pkt->setRemoteAddr(IOAddress("192.0.2.56"));
-    pkt->setGiaddr(IOAddress("192.0.2.67"));
-
-    // Let's make it a relayed message
-    pkt->setHops(3);
-    pkt->setRemotePort(DHCP4_SERVER_PORT);
-
-    // We are going to test that certain options are returned
-    // (or not returned) in the OFFER message when requested
-    // using 'Parameter Request List' option. Let's configure
-    // those options that are returned when requested.
-    configureRequestedOptions();
-
-    // Should not throw
-    EXPECT_NO_THROW(
-        offer = srv->processDiscover(pkt);
-    );
-
-    // Should return something
-    ASSERT_TRUE(offer);
-
-    EXPECT_EQ(DHCPOFFER, offer->getType());
-
-    // This is relayed message. It should be sent back to relay address.
-    EXPECT_EQ(pkt->getGiaddr(), offer->getRemoteAddr());
-
-    messageCheck(pkt, offer);
-
-    // There are some options that are always present in the
-    // message, even if not requested.
-    EXPECT_TRUE(offer->getOption(DHO_DOMAIN_NAME));
-    EXPECT_TRUE(offer->getOption(DHO_DOMAIN_NAME_SERVERS));
-
-    // We did not request any options so they should not be present
-    // in the OFFER.
-    EXPECT_FALSE(offer->getOption(DHO_LOG_SERVERS));
-    EXPECT_FALSE(offer->getOption(DHO_COOKIE_SERVERS));
-    EXPECT_FALSE(offer->getOption(DHO_LPR_SERVERS));
-
-    // Add 'Parameter Request List' option.
-    addPrlOption(pkt);
-
-    // Now repeat the test but request some options.
-    EXPECT_NO_THROW(
-        offer = srv->processDiscover(pkt);
-    );
-
-    // Should return something
-    ASSERT_TRUE(offer);
-
-    EXPECT_EQ(DHCPOFFER, offer->getType());
-
-    // This is relayed message. It should be sent back to relay address.
-    EXPECT_EQ(pkt->getGiaddr(), offer->getRemoteAddr());
-
-    messageCheck(pkt, offer);
-
-    // Check that the requested options are returned.
-    optionsCheck(offer);
-
-    // Now repeat the test for directly sent message
-    pkt->setHops(0);
-    pkt->setGiaddr(IOAddress("0.0.0.0"));
-    pkt->setRemotePort(DHCP4_CLIENT_PORT);
-
-    EXPECT_NO_THROW(
-        offer = srv->processDiscover(pkt);
-    );
-
-    // Should return something
-    ASSERT_TRUE(offer);
-
-    EXPECT_EQ(DHCPOFFER, offer->getType());
-
-    // This is direct message. It should be sent back to origin, not
-    // to relay.
-    EXPECT_EQ(pkt->getRemoteAddr(), offer->getRemoteAddr());
+TEST_F(Dhcpv4SrvTest, processDiscoverRelay) {
+    testDiscoverRequest(DHCPDISCOVER,
+                        IOAddress("192.0.2.56"),
+                        IOAddress("192.0.2.67"));
+}
 
-    messageCheck(pkt, offer);
+// Verifies that the non-relayed DISCOVER is processed correctly when
+// client source address is specified.
+TEST_F(Dhcpv4SrvTest, processDiscoverNoRelay) {
+    testDiscoverRequest(DHCPDISCOVER,
+                        IOAddress("0.0.0.0"),
+                        IOAddress("192.0.2.67"));
+}
 
-    // Check that the requested options are returned.
-    optionsCheck(offer);
+// Verified that the non-relayed DISCOVER is processed correctly when
+// client source address is not specified.
+TEST_F(Dhcpv4SrvTest, processDiscoverNoClientAddr) {
+    testDiscoverRequest(DHCPDISCOVER,
+                        IOAddress("0.0.0.0"),
+                        IOAddress("0.0.0.0"));
 }
 
-// Verifies that received REQUEST can be processed correctly,
-// that the ACK message generated in response is valid and
+// Verifies that REQUEST received via relay can be processed correctly,
+// that the OFFER message generated in response is valid and
 // contains necessary options.
 //
 // Note: this test focuses on the packet correctness. There
 // are other tests that verify correctness of the allocation
-// engine. See RequestBasic.
-TEST_F(Dhcpv4SrvTest, processRequest) {
-    boost::scoped_ptr<NakedDhcpv4Srv> srv(new NakedDhcpv4Srv(0));
-    vector<uint8_t> mac(6);
-    for (int i = 0; i < 6; i++) {
-        mac[i] = i * 10;
-    }
-
-    Pkt4Ptr req(new Pkt4(DHCPREQUEST, 1234));
-    Pkt4Ptr ack;
-
-    req->setIface("eth0");
-    req->setIndex(17);
-    req->setHWAddr(1, 6, mac);
-    req->setRemoteAddr(IOAddress("192.0.2.56"));
-    req->setGiaddr(IOAddress("192.0.2.67"));
-
-    // We are going to test that certain options are returned
-    // in the ACK message when requested using 'Parameter
-    // Request List' option. Let's configure those options that
-    // are returned when requested.
-    configureRequestedOptions();
-
-    // Should not throw
-    ASSERT_NO_THROW(
-        ack = srv->processRequest(req);
-    );
-
-    // Should return something
-    ASSERT_TRUE(ack);
-
-    EXPECT_EQ(DHCPACK, ack->getType());
-
-    // This is relayed message. It should be sent back to relay address.
-    EXPECT_EQ(req->getGiaddr(), ack->getRemoteAddr());
-
-    messageCheck(req, ack);
-
-    // There are some options that are always present in the
-    // message, even if not requested.
-    EXPECT_TRUE(ack->getOption(DHO_DOMAIN_NAME));
-    EXPECT_TRUE(ack->getOption(DHO_DOMAIN_NAME_SERVERS));
-
-    // We did not request any options so these should not be present
-    // in the ACK.
-    EXPECT_FALSE(ack->getOption(DHO_LOG_SERVERS));
-    EXPECT_FALSE(ack->getOption(DHO_COOKIE_SERVERS));
-    EXPECT_FALSE(ack->getOption(DHO_LPR_SERVERS));
-
-    // Add 'Parameter Request List' option.
-    addPrlOption(req);
-
-    // Repeat the test but request some options.
-    ASSERT_NO_THROW(
-        ack = srv->processRequest(req);
-    );
-
-    // Should return something
-    ASSERT_TRUE(ack);
-
-    EXPECT_EQ(DHCPACK, ack->getType());
-
-    // This is relayed message. It should be sent back to relay address.
-    EXPECT_EQ(req->getGiaddr(), ack->getRemoteAddr());
-
-    // Check that the requested options are returned.
-    optionsCheck(ack);
-
-    // Now repeat the test for directly sent message
-    req->setHops(0);
-    req->setGiaddr(IOAddress("0.0.0.0"));
-    req->setRemotePort(DHCP4_CLIENT_PORT);
-
-    EXPECT_NO_THROW(
-        ack = srv->processDiscover(req);
-    );
-
-    // Should return something
-    ASSERT_TRUE(ack);
-
-    EXPECT_EQ(DHCPOFFER, ack->getType());
-
-    // This is direct message. It should be sent back to origin, not
-    // to relay.
-    EXPECT_EQ(ack->getRemoteAddr(), req->getRemoteAddr());
+// engine. See DiscoverBasic, DiscoverHint, DiscoverNoClientId
+// and DiscoverInvalidHint.
+TEST_F(Dhcpv4SrvTest, processRequestRelay) {
+    testDiscoverRequest(DHCPREQUEST,
+                        IOAddress("192.0.2.56"),
+                        IOAddress("192.0.2.67"));
+}
 
-    messageCheck(req, ack);
+// Verifies that the non-relayed REQUEST is processed correctly when
+// client source address is specified.
+TEST_F(Dhcpv4SrvTest, processRequestNoRelay) {
+    testDiscoverRequest(DHCPREQUEST,
+                        IOAddress("0.0.0.0"),
+                        IOAddress("192.0.2.67"));
+}
 
-    // Check that the requested options are returned.
-    optionsCheck(ack);
+// Verified that the non-relayed REQUEST is processed correctly when
+// client source address is not specified.
+TEST_F(Dhcpv4SrvTest, processRequestNoClientAddr) {
+    testDiscoverRequest(DHCPREQUEST,
+                        IOAddress("0.0.0.0"),
+                        IOAddress("0.0.0.0"));
 }
 
 TEST_F(Dhcpv4SrvTest, processRelease) {

src/bin/resolver/Makefile.am

@@ -1,4 +1,4 @@
-SUBDIRS = . tests
+SUBDIRS = . tests bench
 
 AM_CPPFLAGS = -I$(top_srcdir)/src/lib -I$(top_builddir)/src/lib
 AM_CPPFLAGS += -I$(top_srcdir)/src/bin -I$(top_builddir)/src/bin

src/bin/resolver/bench/Makefile.am

@@ -0,0 +1,25 @@
+AM_CPPFLAGS = -I$(top_srcdir)/src/lib -I$(top_builddir)/src/lib
+AM_CPPFLAGS += -I$(top_builddir)/src/lib/dns -I$(top_srcdir)/src/bin
+AM_CPPFLAGS += -I$(top_builddir)/src/lib/cc
+AM_CPPFLAGS += -I$(top_builddir)/src/bin/resolver
+AM_CPPFLAGS += $(BOOST_INCLUDES)
+
+AM_CXXFLAGS = $(B10_CXXFLAGS)
+
+if USE_STATIC_LINK
+AM_LDFLAGS = -static
+endif
+
+CLEANFILES = *.gcno *.gcda
+
+noinst_PROGRAMS = resolver-bench
+
+resolver_bench_SOURCES = main.cc
+resolver_bench_SOURCES += fake_resolution.h fake_resolution.cc
+resolver_bench_SOURCES += dummy_work.h dummy_work.cc
+resolver_bench_SOURCES += naive_resolver.h naive_resolver.cc
+
+resolver_bench_LDADD  = $(GTEST_LDADD)
+resolver_bench_LDADD += $(top_builddir)/src/lib/exceptions/libb10-exceptions.la
+resolver_bench_LDADD += $(top_builddir)/src/lib/asiolink/libb10-asiolink.la
+

src/bin/resolver/bench/dummy_work.cc

@@ -0,0 +1,28 @@
+// Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <resolver/bench/dummy_work.h>
+
+namespace isc {
+namespace resolver {
+namespace bench {
+
+void
+dummy_work() {
+    // Function left intentonally blank.
+};
+
+}
+}
+}

src/bin/resolver/bench/dummy_work.h

@@ -0,0 +1,36 @@
+// Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#ifndef DUMMY_WORK_H
+#define DUMMY_WORK_H
+
+namespace isc {
+namespace resolver {
+namespace bench {
+
+/// \brief An empty function.
+///
+/// An empty function, to fill the CPU with something during the benchmark.
+/// It is expected to be called many times by whatever simulates doing some
+/// real CPU-bound work.
+///
+/// It is defined in separate translation unit, so the compiler does not
+/// know it is empty and can't optimise the call out.
+void dummy_work();
+
+}
+}
+}
+
+#endif

src/bin/resolver/bench/fake_resolution.cc

@@ -0,0 +1,172 @@
+// Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <resolver/bench/fake_resolution.h>
+#include <resolver/bench/dummy_work.h>
+
+#include <asiolink/interval_timer.h>
+
+#include <boost/bind.hpp>
+#include <boost/foreach.hpp>
+#include <algorithm>
+#include <stdlib.h> // not cstdlib, which doesn't officially have random()
+
+namespace isc {
+namespace resolver {
+namespace bench {
+
+// Parameters of the generated queries.
+// How much work is each operation?
+const size_t parse_size = 100000;
+const size_t render_size = 100000;
+const size_t send_size = 1000;
+const size_t cache_read_size = 10000;
+const size_t cache_write_size = 10000;
+// How large a change is to terminate in this iteration (either by getting
+// the complete answer, or by finding it in the cache). With 0.5, half the
+// queries are found in the cache directly. Half of the rest needs just one
+// upstream query. Etc.
+const float chance_complete = 0.5;
+// Number of milliseconds an upstream query can take. It picks a random number
+// in between.
+const size_t upstream_time_min = 2;
+const size_t upstream_time_max = 50;
+
+FakeQuery::FakeQuery(FakeInterface& interface) :
+    interface_(&interface),
+    outstanding_(false)
+{
+    // Schedule what tasks are needed.
+    // First, parse the query
+    steps_.push_back(Step(Compute, parse_size));
+    // Look into the cache if it is there
+    steps_.push_back(Step(CacheRead, cache_read_size));
+    while ((1.0 * random()) / RAND_MAX > chance_complete) {
+        // Needs another step of recursion. Render the upstream query.
+        steps_.push_back(Step(Compute, render_size));
+        // Send it and wait for the answer.
+        steps_.push_back(Step(Upstream, upstream_time_min +
+                              (random() *
+                               (upstream_time_max - upstream_time_min) /
+                               RAND_MAX)));
+        // After it comes, parse the answer and store it in the cache.
+        steps_.push_back(Step(Compute, parse_size));
+        steps_.push_back(Step(CacheWrite, cache_write_size));
+    }
+    // Last, render the answer and send it.
+    steps_.push_back(Step(Compute, render_size));
+    steps_.push_back(Step(Send, send_size));
+    // Reverse it, so we can pop_back the tasks as we work on them.
+    std::reverse(steps_.begin(), steps_.end());
+}
+
+void
+FakeQuery::performTask(const StepCallback& callback) {
+    // nextTask also does all the sanity checking we need.
+    if (nextTask() == Upstream) {
+        outstanding_ = true;
+        interface_->scheduleUpstreamAnswer(this, callback,
+                                           steps_.back().second);
+        steps_.pop_back();
+    } else {
+        for (size_t i = 0; i < steps_.back().second; ++i) {
+            dummy_work();
+        }
+        steps_.pop_back();
+        callback();
+    }
+}
+
+FakeInterface::FakeInterface(size_t query_count) :
+    queries_(query_count)
+{
+    BOOST_FOREACH(FakeQueryPtr& query, queries_) {
+        query = FakeQueryPtr(new FakeQuery(*this));
+    }
+}
+
+void
+FakeInterface::processEvents() {
+    service_.run_one();
+}
+
+namespace {
+
+void
+processDone(bool* flag) {
+    *flag = true;
+}
+
+}
+
+FakeQueryPtr
+FakeInterface::receiveQuery() {
+    // Handle all the events that are already scheduled.
+    // As processEvents blocks until an event happens and we want to terminate
+    // if there are no events, we do a small trick. We post an event to the end
+    // of the queue and work until it is found. This should process all the
+    // events that were there already.
+    bool processed = false;
+    service_.post(boost::bind(&processDone, &processed));
+    while (!processed) {
+        processEvents();
+    }
+
+    // Now, look if there are more queries to return.
+    if (queries_.empty()) {
+        return (FakeQueryPtr());
+    } else {
+        // Take from the back. The order doesn't matter and it's faster from
+        // there.
+        FakeQueryPtr result(queries_.back());
+        queries_.pop_back();
+        return (result);
+    }
+}
+
+class FakeInterface::UpstreamQuery {
+public:
+    UpstreamQuery(FakeQuery* query, const FakeQuery::StepCallback& callback,
+                  const boost::shared_ptr<asiolink::IntervalTimer> timer) :
+        query_(query),
+        callback_(callback),
+        timer_(timer)
+    {}
+    void trigger() {
+        query_->answerReceived();
+        callback_();
+        // We are not needed any more.
+        delete this;
+    }
+private:
+    FakeQuery* const query_;
+    const FakeQuery::StepCallback callback_;
+    // Just to hold it alive before the callback is called.
+    const boost::shared_ptr<asiolink::IntervalTimer> timer_;
+};
+
+void
+FakeInterface::scheduleUpstreamAnswer(FakeQuery* query,
+                                      const FakeQuery::StepCallback& callback,
+                                      size_t msec)
+{
+    const boost::shared_ptr<asiolink::IntervalTimer>
+        timer(new asiolink::IntervalTimer(service_));
+    UpstreamQuery* q(new UpstreamQuery(query, callback, timer));
+    timer->setup(boost::bind(&UpstreamQuery::trigger, q), msec);
+}
+
+}
+}
+}

src/bin/resolver/bench/fake_resolution.h

@@ -0,0 +1,228 @@
+// Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#ifndef FAKE_RESOLUTION_H
+#define FAKE_RESOLUTION_H
+
+#include <exceptions/exceptions.h>
+#include <asiolink/io_service.h>
+
+#include <boost/function.hpp>
+#include <boost/shared_ptr.hpp>
+
+#include <utility>
+#include <vector>
+
+namespace isc {
+namespace resolver {
+namespace bench {
+
+/// \brief The kind of task a FakeQuery might want to perform.
+///
+/// The benchmark should examine which kind of task the query needs to perform
+/// to progress forward. According to the task, some resources might need to be
+/// locked, something re-scheduled, or such.
+enum Task {
+    /// \brief Some CPU-bound computation.
+    ///
+    /// The query needs to do some computation without any shared resources.
+    /// This might be parsing or rendering of the query, verification of
+    /// signatures, etc.
+    Compute,
+    /// \brief The query needs to read data from cache.
+    CacheRead,
+    /// \brief The query needs to modify the cache.
+    CacheWrite,
+    /// \brief A response is to be sent.
+    ///
+    /// This needs to access the interface/socket. If the socket is shared
+    /// between threads, it might need to lock it.
+    Send,
+    /// \brief An answer from upstream server is needed.
+    ///
+    /// The query needs to send a query to some authoritative server and wait
+    /// for the answer. Something might need to be locked (or not, depending
+    /// on the architecture of the thing that sends and receives). Also, the
+    /// task will not complete immediately, the callback of performTask
+    /// will be called at later time.
+    Upstream
+};
+
+class FakeInterface;
+
+/// \brief Imitation of the work done to resolve a query.
+///
+/// An object of this class represents some fake work that should look like
+/// the work needed to perform resolution of one query. No real work is done,
+/// but several steps are scheduled, with characteristics hopefully
+/// corresponding to steps of the real query.
+///
+/// The idea is that benchmark will repeatedly check if the query is done.
+/// If not, it examines the next task by calling nextTask(). Depending on
+/// the result, it'd lock or prepare any shared resources. After that, it'd
+/// call performTask() to do the task. Once the query calls the callback
+/// passed, it can proceed to the next step.
+///
+/// See naive_resolver.cc for example code how this could be done.
+class FakeQuery {
+private:
+    // The queries come only through an interface. Don't let others create.
+    friend class FakeInterface;
+    /// \brief Constructor
+    FakeQuery(FakeInterface& interface);
+public:
+    /// \brief Is work on the query completely done?
+    ///
+    /// If this returns true, do not call performTask or nextTask any more.
+    /// The resolution is done.
+    ///
+    /// \throw isc::InvalidOperation if upstream query is still in progress.
+    bool done() const {
+        if (outstanding_) {
+            isc_throw(isc::InvalidOperation, "Upstream query outstanding");
+        }
+        return (steps_.empty());
+    }
+    /// \brief Callback to signify a task has been performed.
+    typedef boost::function<void()> StepCallback;
+    /// \brief Perform next step in the resolution.
+    ///
+    /// Do whatever is needed to be done for the next step of resolution.
+    /// Once the step is done, the callback is called.
+    ///
+    /// The callback is usually called from within this call. However, in
+    /// the case when the nextTask() returned `Upstream`, the call to the
+    /// callback is delayed for some period of time after the method
+    /// returns.
+    ///
+    /// \throw isc::InvalidOperation if it is called when done() is true, or
+    ///     if an upstream query is still in progress (performTask was called
+    ///     before and the callback was not called by the query yet).
+    void performTask(const StepCallback& callback);
+    /// \brief Examine the kind of the next resolution process.
+    ///
+    /// Call this to know what kind of task will performTask do next.
+    ///
+    /// \throw isc::InvalidOperation if it is called when done() is true, or
+    ///     if an upstream query is still in progress (performTask was called
+    ///     before and the callback was not called by the query yet).
+    Task nextTask() const {
+        // Will check for outstanding_ internally too
+        if (done()) {
+            isc_throw(isc::InvalidOperation, "We are done, no more tasks");
+        }
+        return (steps_.back().first);
+    }
+    /// \brief Move network communication to different interface.
+    ///
+    /// By default, a query does all the "communication" on the interface
+    /// it was born on. This may be used to move a query from one interface
+    /// to another.
+    ///
+    /// You don't have to lock either of the interfaces to do so, this
+    /// only switches the data in the query.
+    ///
+    /// \throw isc::InvalidOperation if it is called while an upstream query
+    ///     is in progress.
+    void migrateTo(FakeInterface& dst_interface) {
+        if (outstanding_) {
+            isc_throw(isc::InvalidOperation,
+                      "Can't migrate in the middle of query");
+        }
+        interface_ = &dst_interface;
+    }
+    /// \brief The answer for upstream query was received
+    ///
+    /// This should be called from within the FakeInterface only.
+    /// It marks that the query from upstream was answered.
+    void answerReceived() {
+        outstanding_ = false;
+    }
+private:
+    // The scheduled steps for this task.
+    typedef std::pair<Task, size_t> Step;
+    // The scheduled steps. Reversed (first to be done at the end), so we can
+    // pop_back() the completed steps.
+    std::vector<Step> steps_;
+    // The interface to schedule timeouts on.
+    FakeInterface* interface_;
+    // Is an upstream query outstanding?
+    bool outstanding_;
+};
+
+typedef boost::shared_ptr<FakeQuery> FakeQueryPtr;
+
+/// \brief An imitation of interface for receiving queries.
+///
+/// This is effectively a little bit smarter factory for queries. You can
+/// request a new query from it, or let process events (incoming answers).
+///
+/// It contains its own event loop. If the benchmark has more threads, have
+/// one in each of the threads (if the threads ever handles network
+/// communication -- if it accepts queries, sends answers or does upstream
+/// queries).
+///
+/// If the model simulated would share the same interface between multiple
+/// threads, it is better to have one in each thread as well, but lock
+/// access to receiveQuery() so only one is used at once (no idea what happens
+/// if ASIO loop is accessed from multiple threads).
+///
+/// Note that the creation of the queries is not thread safe (due to
+/// the random() function inside). The interface generates all its queries
+/// in advance, on creation time. But you need to create all the needed
+/// interfaces from single thread and then distribute them to your threads.
+class FakeInterface {
+public:
+    /// \brief Constructor
+    ///
+    /// Initiarile the interface and create query_count queries for the
+    /// benchmark. They will be handed out one by one with receiveQuery().
+    FakeInterface(size_t query_count);
+    /// \brief Wait for answers from upstream servers.
+    ///
+    /// Wait until at least one "answer" comes from the remote server. This
+    /// will effectively block the calling thread until it is time to call
+    /// a callback of performTask.
+    ///
+    /// It is not legal to call it without any outstanding upstream queries
+    /// on this interface. However, the situation is not explicitly checked.
+    ///
+    /// \note Due to internal implementation, it is not impossible no or more
+    ///    than one callbacks to be called from within this method.
+    void processEvents();
+    /// \brief Accept another query.
+    ///
+    /// Generate a new fake query to resolve.
+    ///
+    /// This method might call callbacks of other queries waiting for upstream
+    /// answer.
+    ///
+    /// This returns a NULL pointer when there are no more queries to answer
+    /// (the number designated for the benchmark was reached).
+    FakeQueryPtr receiveQuery();
+private:
+    class UpstreamQuery;
+    friend class FakeQuery;
+    void scheduleUpstreamAnswer(FakeQuery* query,
+                                const FakeQuery::StepCallback& callback,
+                                size_t msec);
+    asiolink::IOService service_;
+    std::vector<FakeQueryPtr> queries_;
+};
+
+}
+}
+}
+
+#endif

src/bin/resolver/bench/main.cc

@@ -0,0 +1,27 @@
+// Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <resolver/bench/naive_resolver.h>
+
+#include <bench/benchmark.h>
+
+const size_t count = 1000; // TODO: We may want to read this from argv.
+
+int main(int, const char**) {
+    // Run the naive implementation
+    isc::resolver::bench::NaiveResolver naive_resolver(count);
+    isc::bench::BenchMark<isc::resolver::bench::NaiveResolver>
+        (1, naive_resolver, true);
+    return 0;
+}

src/bin/resolver/bench/naive_resolver.cc

@@ -0,0 +1,66 @@
+// Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <resolver/bench/naive_resolver.h>
+
+#include <cassert>
+#include <boost/bind.hpp>
+
+namespace isc {
+namespace resolver {
+namespace bench {
+
+NaiveResolver::NaiveResolver(size_t query_count) :
+    interface_(query_count),
+    processed_(false)
+{}
+
+namespace {
+
+void
+stepDone(bool* flag) {
+    *flag = true;
+}
+
+}
+
+size_t
+NaiveResolver::run() {
+    assert(!processed_);
+    size_t count = 0;
+    FakeQueryPtr query;
+    // Process a query at a time. As the previous is already handled, the
+    // receiveQuery may never trigger other events.
+    while ((query = interface_.receiveQuery())) {
+        // Handle each step
+        while (!query->done()) {
+            bool done = false; // This step is not yet done.
+            // If there were more queries/threads/whatever, we would examine
+            // the query->nextTask() and lock or prepare resources accordingly.
+            // But as there's just one, we simply do the task, without caring.
+            query->performTask(boost::bind(&stepDone, &done));
+            // We may need to wait for the upstream query.
+            while (!done) {
+                interface_.processEvents();
+            }
+        }
+        count ++;
+    }
+    processed_ = true;
+    return (count);
+}
+
+}
+}
+}

src/bin/resolver/bench/naive_resolver.h

@@ -0,0 +1,44 @@
+// Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#ifndef RESOLVER_BENCH_NAIVE_H
+#define RESOLVER_BENCH_NAIVE_H
+
+#include <resolver/bench/fake_resolution.h>
+
+namespace isc {
+namespace resolver {
+namespace bench {
+
+/// \brief Naive implementation of resolver for the benchmark
+///
+/// This is here mostly to show how to implement the other benchmark
+/// implementations. Look at the code inside how to use the fake
+/// resolution.
+class NaiveResolver {
+public:
+    /// \brief Constructor. Initializes the data.
+    NaiveResolver(size_t query_count);
+    /// \brief Run the resolution.
+    size_t run();
+private:
+    FakeInterface interface_;
+    bool processed_;
+};
+
+}
+}
+}
+
+#endif

src/bin/stats/tests/Makefile.am

@@ -1,7 +1,7 @@
 SUBDIRS = testdata .
 
 PYCOVERAGE_RUN = @PYCOVERAGE_RUN@
-PYTESTS = b10-stats_test.py b10-stats-httpd_test.py
+PYTESTS = stats_test.py stats-httpd_test.py
 EXTRA_DIST = $(PYTESTS) test_utils.py
 CLEANFILES = test_utils.pyc
 

src/bin/stats/tests/b10-stats-httpd_test.py

@@ -46,10 +46,10 @@ import isc
 import isc.log
 import stats_httpd
 import stats
-from test_utils import BaseModules, ThreadingServerManager, MyStats,\
-                       MyStatsHttpd, SignalHandler,\
-                       send_command, CONST_BASETIME
+from test_utils import ThreadingServerManager, SignalHandler, \
+    MyStatsHttpd, CONST_BASETIME
 from isc.testutils.ccsession_mock import MockModuleCCSession
+from isc.config import RPCRecipientMissing, RPCError
 
 # This test suite uses xml.etree.ElementTree.XMLParser via
 # xml.etree.ElementTree.parse. On the platform where expat isn't
@@ -104,6 +104,11 @@ DUMMY_DATA = {
         }
     }
 
+# Bad practice: this should be localized
+stats._BASETIME = CONST_BASETIME
+stats.get_timestamp = lambda: time.mktime(CONST_BASETIME)
+stats.get_datetime = lambda x=None: time.strftime("%Y-%m-%dT%H:%M:%SZ", CONST_BASETIME)
+
 def get_availaddr(address='127.0.0.1', port=8001):
     """returns a tuple of address and port which is available to
     listen on the platform. The first argument is a address for
@@ -230,13 +235,11 @@ class TestHttpHandler(unittest.TestCase):
     def setUp(self):
         # set the signal handler for deadlock
         self.sig_handler = SignalHandler(self.fail)
-        self.base = BaseModules()
-        self.stats_server = ThreadingServerManager(MyStats)
-        self.stats = self.stats_server.server
-        DUMMY_DATA['Stats']['lname'] = self.stats.cc_session.lname
-        self.stats_server.run()
+        DUMMY_DATA['Stats']['lname'] = 'test-lname'
         (self.address, self.port) = get_availaddr()
-        self.stats_httpd_server = ThreadingServerManager(MyStatsHttpd, (self.address, self.port))
+        self.stats_httpd_server = ThreadingServerManager(MyStatsHttpd,
+                                                         (self.address,
+                                                          self.port))
         self.stats_httpd = self.stats_httpd_server.server
         self.stats_httpd_server.run()
         self.client = http.client.HTTPConnection(self.address, self.port)
@@ -245,13 +248,9 @@ class TestHttpHandler(unittest.TestCase):
 
     def tearDown(self):
         self.client.close()
-        self.stats_httpd_server.shutdown()
-        self.stats_server.shutdown()
-        self.base.shutdown()
         # reset the signal handler
         self.sig_handler.reset()
 
-    @unittest.skipIf(sys.version_info >= (3, 3), "Unsupported in Python 3.3 or higher")
     @unittest.skipUnless(xml_parser, "skipping the test using XMLParser")
     def test_do_GET(self):
         self.assertTrue(type(self.stats_httpd.httpd) is list)
@@ -456,15 +455,10 @@ class TestHttpHandler(unittest.TestCase):
         self.assertEqual(response.status, 404)
 
     def test_do_GET_failed1(self):
-        # checks status
-        self.assertEqual(send_command("status", "Stats"),
-                         (0, "Stats is up. (PID " + str(os.getpid()) + ")"))
-        # failure case(Stats is down)
-        self.assertTrue(self.stats.running)
-        self.assertEqual(send_command("shutdown", "Stats"),
-                         (0, None)) # Stats is down
-        self.assertFalse(self.stats.running)
-        self.stats_httpd.cc_session.set_timeout(milliseconds=100)
+        # failure case (Stats is down, so rpc_call() results in an exception)
+        # Note: this should eventually be RPCRecipientMissing.
+        self.stats_httpd._rpc_answers.append(
+            isc.cc.session.SessionTimeout('timeout'))
 
         # request XML
         self.client.putrequest('GET', stats_httpd.XML_URL_PATH + '/')
@@ -486,10 +480,8 @@ class TestHttpHandler(unittest.TestCase):
 
     def test_do_GET_failed2(self):
         # failure case(Stats replies an error)
-        self.stats.mccs.set_command_handler(
-            lambda cmd, args: \
-                isc.config.ccsession.create_answer(1, "specified arguments are incorrect: I have an error.")
-            )
+        self.stats_httpd._rpc_answers.append(
+            RPCError(1, "specified arguments are incorrect: I have an error."))
 
         # request XML
         self.client.putrequest('GET', stats_httpd.XML_URL_PATH + '/')
@@ -498,12 +490,16 @@ class TestHttpHandler(unittest.TestCase):
         self.assertEqual(response.status, 404)
 
         # request XSD
+        self.stats_httpd._rpc_answers.append(
+            RPCError(1, "specified arguments are incorrect: I have an error."))
         self.client.putrequest('GET', stats_httpd.XSD_URL_PATH)
         self.client.endheaders()
         response = self.client.getresponse()
         self.assertEqual(response.status, 200)
 
         # request XSL
+        self.stats_httpd._rpc_answers.append(
+            RPCError(1, "specified arguments are incorrect: I have an error."))
         self.client.putrequest('GET', stats_httpd.XSL_URL_PATH)
         self.client.endheaders()
         response = self.client.getresponse()
@@ -567,12 +563,10 @@ class TestHttpServer(unittest.TestCase):
     def setUp(self):
         # set the signal handler for deadlock
         self.sig_handler = SignalHandler(self.fail)
-        self.base = BaseModules()
 
     def tearDown(self):
         if hasattr(self, "stats_httpd"):
             self.stats_httpd.stop()
-        self.base.shutdown()
         # reset the signal handler
         self.sig_handler.reset()
 
@@ -604,9 +598,6 @@ class TestStatsHttpd(unittest.TestCase):
     def setUp(self):
         # set the signal handler for deadlock
         self.sig_handler = SignalHandler(self.fail)
-        self.base = BaseModules()
-        self.stats_server = ThreadingServerManager(MyStats)
-        self.stats_server.run()
         # checking IPv6 enabled on this platform
         self.ipv6_enabled = is_ipv6_enabled()
         # instantiation of StatsHttpd indirectly calls gethostbyaddr(), which
@@ -617,71 +608,80 @@ class TestStatsHttpd(unittest.TestCase):
         self.__gethostbyaddr_orig = socket.gethostbyaddr
         socket.gethostbyaddr = lambda x: ('test.example.', [], None)
 
+        # Some tests replace this library function.  Keep the original for
+        # restor
+        self.__orig_select_select = select.select
+
     def tearDown(self):
         socket.gethostbyaddr = self.__gethostbyaddr_orig
         if hasattr(self, "stats_httpd"):
             self.stats_httpd.stop()
-        self.stats_server.shutdown()
-        self.base.shutdown()
         # reset the signal handler
         self.sig_handler.reset()
 
+        # restore original of replaced library
+        select.select = self.__orig_select_select
+
     def test_init(self):
         server_address = get_availaddr()
         self.stats_httpd = MyStatsHttpd(server_address)
         self.assertEqual(self.stats_httpd.running, False)
         self.assertEqual(self.stats_httpd.poll_intval, 0.5)
         self.assertNotEqual(len(self.stats_httpd.httpd), 0)
-        self.assertEqual(type(self.stats_httpd.mccs), isc.config.ModuleCCSession)
-        self.assertEqual(type(self.stats_httpd.cc_session), isc.cc.Session)
-        self.assertEqual(len(self.stats_httpd.config), 2)
+        self.assertIsNotNone(self.stats_httpd.mccs)
+        self.assertIsNotNone(self.stats_httpd.cc_session)
+        # The real CfgMgr would return 'version', but our test mock omits it,
+        # so the len(config) should be 1
+        self.assertEqual(len(self.stats_httpd.config), 1)
         self.assertTrue('listen_on' in self.stats_httpd.config)
         self.assertEqual(len(self.stats_httpd.config['listen_on']), 1)
         self.assertTrue('address' in self.stats_httpd.config['listen_on'][0])
         self.assertTrue('port' in self.stats_httpd.config['listen_on'][0])
         self.assertTrue(server_address in set(self.stats_httpd.http_addrs))
-        ans = send_command(
-            isc.config.ccsession.COMMAND_GET_MODULE_SPEC,
-            "ConfigManager", {"module_name":"StatsHttpd"})
-        # assert StatsHttpd is added to ConfigManager
-        self.assertNotEqual(ans, (0,{}))
-        self.assertTrue(ans[1]['module_name'], 'StatsHttpd')
+        self.assertEqual('StatsHttpd', self.stats_httpd.mccs.\
+                             get_module_spec().get_module_name())
 
     def test_init_hterr(self):
-        orig_open_httpd = stats_httpd.StatsHttpd.open_httpd
-        def err_open_httpd(arg): raise stats_httpd.HttpServerError
-        stats_httpd.StatsHttpd.open_httpd = err_open_httpd
-        self.assertRaises(stats_httpd.HttpServerError, stats_httpd.StatsHttpd)
-        ans = send_command(
-            isc.config.ccsession.COMMAND_GET_MODULE_SPEC,
-            "ConfigManager", {"module_name":"StatsHttpd"})
-        # assert StatsHttpd is removed from ConfigManager
-        self.assertEqual(ans, (0,{}))
-        stats_httpd.StatsHttpd.open_httpd = orig_open_httpd
+        """Test the behavior of StatsHttpd constructor when open_httpd fails.
+
+        We specifically check the following two:
+        - close_mccs() is called (so stats-httpd tells ConfigMgr it's shutting
+          down)
+        - the constructor results in HttpServerError exception.
+
+        """
+        self.__mccs_closed = False
+        def call_checker():
+            self.__mccs_closed = True
+        class FailingStatsHttpd(MyStatsHttpd):
+            def open_httpd(self):
+                raise stats_httpd.HttpServerError
+            def close_mccs(self):
+                call_checker()
+        self.assertRaises(stats_httpd.HttpServerError, FailingStatsHttpd)
+        self.assertTrue(self.__mccs_closed)
 
     def test_openclose_mccs(self):
         self.stats_httpd = MyStatsHttpd(get_availaddr())
-        mccs = MockModuleCCSession()
-        self.stats_httpd.mccs = mccs
+        mccs = self.stats_httpd.mccs
         self.assertFalse(self.stats_httpd.mccs.stopped)
         self.assertFalse(self.stats_httpd.mccs.closed)
         self.stats_httpd.close_mccs()
         self.assertTrue(mccs.stopped)
         self.assertTrue(mccs.closed)
-        self.assertEqual(self.stats_httpd.mccs, None)
+        self.assertIsNone(self.stats_httpd.mccs)
         self.stats_httpd.open_mccs()
         self.assertIsNotNone(self.stats_httpd.mccs)
         self.stats_httpd.mccs = None
-        self.assertEqual(self.stats_httpd.mccs, None)
-        self.assertEqual(self.stats_httpd.close_mccs(), None)
+        self.assertIsNone(self.stats_httpd.mccs)
+        self.assertIsNone(self.stats_httpd.close_mccs())
 
     def test_mccs(self):
         self.stats_httpd = MyStatsHttpd(get_availaddr())
         self.assertIsNotNone(self.stats_httpd.mccs.get_socket())
         self.assertTrue(
             isinstance(self.stats_httpd.mccs.get_socket(), socket.socket))
-        self.assertTrue(
-            isinstance(self.stats_httpd.cc_session, isc.cc.session.Session))
+        self.assertIsNotNone(self.stats_httpd.cc_session)
         statistics_spec = self.stats_httpd.get_stats_spec()
         for mod in DUMMY_DATA:
             self.assertTrue(mod in statistics_spec)
@@ -699,8 +699,11 @@ class TestStatsHttpd(unittest.TestCase):
             self.stats_httpd = MyStatsHttpd(*server_addresses)
             for ht in self.stats_httpd.httpd:
                 self.assertTrue(isinstance(ht, stats_httpd.HttpServer))
-                self.assertTrue(ht.address_family in set([socket.AF_INET, socket.AF_INET6]))
+                self.assertTrue(ht.address_family in set([socket.AF_INET,
+                                                          socket.AF_INET6]))
                 self.assertTrue(isinstance(ht.socket, socket.socket))
+                ht.socket.close() # to silence warning about resource leak
+            self.stats_httpd.close_mccs() # ditto
 
         # dual stack (address is ipv6)
         if self.ipv6_enabled:
@@ -710,6 +713,8 @@ class TestStatsHttpd(unittest.TestCase):
                 self.assertTrue(isinstance(ht, stats_httpd.HttpServer))
                 self.assertEqual(ht.address_family, socket.AF_INET6)
                 self.assertTrue(isinstance(ht.socket, socket.socket))
+                ht.socket.close()
+            self.stats_httpd.close_mccs() # ditto
 
         # dual/single stack (address is ipv4)
         server_addresses = get_availaddr()
@@ -718,6 +723,8 @@ class TestStatsHttpd(unittest.TestCase):
             self.assertTrue(isinstance(ht, stats_httpd.HttpServer))
             self.assertEqual(ht.address_family, socket.AF_INET)
             self.assertTrue(isinstance(ht.socket, socket.socket))
+            ht.socket.close()
+        self.stats_httpd.close_mccs()
 
     def test_httpd_anyIPv4(self):
         # any address (IPv4)
@@ -744,39 +751,69 @@ class TestStatsHttpd(unittest.TestCase):
                           get_availaddr(address='localhost'))
 
         # nonexistent hostname
-        self.assertRaises(stats_httpd.HttpServerError, MyStatsHttpd, ('my.host.domain', 8000))
+        self.assertRaises(stats_httpd.HttpServerError, MyStatsHttpd,
+                          ('my.host.domain', 8000))
 
         # over flow of port number
-        self.assertRaises(stats_httpd.HttpServerError, MyStatsHttpd, ('127.0.0.1', 80000))
+        self.assertRaises(stats_httpd.HttpServerError, MyStatsHttpd,
+                          ('127.0.0.1', 80000))
 
         # negative
-        self.assertRaises(stats_httpd.HttpServerError, MyStatsHttpd, ('127.0.0.1', -8000))
+        self.assertRaises(stats_httpd.HttpServerError, MyStatsHttpd,
+                          ('127.0.0.1', -8000))
 
         # alphabet
-        self.assertRaises(stats_httpd.HttpServerError, MyStatsHttpd, ('127.0.0.1', 'ABCDE'))
+        self.assertRaises(stats_httpd.HttpServerError, MyStatsHttpd,
+                          ('127.0.0.1', 'ABCDE'))
 
         # Address already in use
         server_addresses = get_availaddr()
-        self.stats_httpd_server = ThreadingServerManager(MyStatsHttpd, server_addresses)
-        self.stats_httpd_server.run()
-        self.assertRaises(stats_httpd.HttpServerError, MyStatsHttpd, server_addresses)
-        send_command("shutdown", "StatsHttpd")
+        server = MyStatsHttpd(server_addresses)
+        self.assertRaises(stats_httpd.HttpServerError, MyStatsHttpd,
+                          server_addresses)
+
+    def __faked_select(self, ex=None):
+        """A helper subroutine for tests using faked select.select.
+
+        See test_running() for basic features.  If ex is not None,
+        it's assumed to be an exception object and will be raised on the
+        first call.
+
+        """
+        self.assertTrue(self.stats_httpd.running)
+        self.__call_count += 1
+        if ex is not None and self.__call_count == 1:
+            raise ex
+        if self.__call_count == 2:
+            self.stats_httpd.running  = False
+        assert self.__call_count <= 2 # safety net to avoid infinite loop
+        return ([], [], [])
 
     def test_running(self):
-        self.stats_httpd_server = ThreadingServerManager(MyStatsHttpd, get_availaddr())
-        self.stats_httpd = self.stats_httpd_server.server
+        # Previous version of this test checks the result of "status" and
+        # "shutdown" commands; however, they are more explicitly tested
+        # in specific tests.  In this test we only have to check:
+        # - start() will set 'running' to True
+        # - as long as 'running' is True, it keeps calling select.select
+        # - when running becomes False, it exists from the loop and calls
+        #   stop()
+        self.stats_httpd = MyStatsHttpd(get_availaddr())
         self.assertFalse(self.stats_httpd.running)
-        self.stats_httpd_server.run()
-        self.assertEqual(send_command("status", "StatsHttpd"),
-                         (0, "Stats Httpd is up. (PID " + str(os.getpid()) + ")"))
-        self.assertTrue(self.stats_httpd.running)
-        self.assertEqual(send_command("shutdown", "StatsHttpd"), (0, None))
+
+        # In this test we'll call select.select() 2 times: on the first call
+        # stats_httpd.running should be True; on the second call the faked
+        # select() will set it to False.
+        self.__call_count = 0
+        select.select = lambda r, w, x, t: self.__faked_select()
+        self.stats_httpd.start()
         self.assertFalse(self.stats_httpd.running)
-        self.stats_httpd_server.shutdown()
+        self.assertIsNone(self.stats_httpd.mccs) # stop() clears .mccs
 
-        # failure case
+    def test_running_fail(self):
+        # A failure case of start(): we close the (real but dummy) socket for
+        # the CC session.  This breaks the select-loop due to exception
         self.stats_httpd = MyStatsHttpd(get_availaddr())
-        self.stats_httpd.cc_session.close()
+        self.stats_httpd.mccs.get_socket().close()
         self.assertRaises(ValueError, self.stats_httpd.start)
 
     def test_failure_with_a_select_error (self):
@@ -784,28 +821,26 @@ class TestStatsHttpd(unittest.TestCase):
         errno.EINTR is raised while it's selecting"""
         def raise_select_except(*args):
             raise select.error('dummy error')
-        orig_select = stats_httpd.select.select
-        stats_httpd.select.select = raise_select_except
+        select.select = raise_select_except
         self.stats_httpd = MyStatsHttpd(get_availaddr())
         self.assertRaises(select.error, self.stats_httpd.start)
-        stats_httpd.select.select = orig_select
 
     def test_nofailure_with_errno_EINTR(self):
         """checks no exception is raised if errno.EINTR is raised
         while it's selecting"""
-        def raise_select_except(*args):
-            raise select.error(errno.EINTR)
-        orig_select = stats_httpd.select.select
-        stats_httpd.select.select = raise_select_except
-        self.stats_httpd_server = ThreadingServerManager(MyStatsHttpd, get_availaddr())
-        self.stats_httpd_server.run()
-        self.stats_httpd_server.shutdown()
-        stats_httpd.select.select = orig_select
+        self.__call_count = 0
+        select.select = lambda r, w, x, t: self.__faked_select(
+            select.error(errno.EINTR))
+        self.stats_httpd = MyStatsHttpd(get_availaddr())
+        self.stats_httpd.start() # shouldn't leak the exception
+        self.assertFalse(self.stats_httpd.running)
+        self.assertIsNone(self.stats_httpd.mccs)
 
     def test_open_template(self):
         self.stats_httpd = MyStatsHttpd(get_availaddr())
         # successful conditions
-        tmpl = self.stats_httpd.open_template(stats_httpd.XML_TEMPLATE_LOCATION)
+        tmpl = self.stats_httpd.open_template(
+            stats_httpd.XML_TEMPLATE_LOCATION)
         self.assertTrue(isinstance(tmpl, string.Template))
         opts = dict(
             xml_string="<dummy></dummy>",
@@ -813,13 +848,15 @@ class TestStatsHttpd(unittest.TestCase):
         lines = tmpl.substitute(opts)
         for n in opts:
             self.assertGreater(lines.find(opts[n]), 0)
-        tmpl = self.stats_httpd.open_template(stats_httpd.XSD_TEMPLATE_LOCATION)
+        tmpl = self.stats_httpd.open_template(
+            stats_httpd.XSD_TEMPLATE_LOCATION)
         self.assertTrue(isinstance(tmpl, string.Template))
         opts = dict(xsd_namespace="http://host/path/to/")
         lines = tmpl.substitute(opts)
         for n in opts:
             self.assertGreater(lines.find(opts[n]), 0)
-        tmpl = self.stats_httpd.open_template(stats_httpd.XSL_TEMPLATE_LOCATION)
+        tmpl = self.stats_httpd.open_template(
+            stats_httpd.XSL_TEMPLATE_LOCATION)
         self.assertTrue(isinstance(tmpl, string.Template))
         opts = dict(xsd_namespace="http://host/path/to/")
         lines = tmpl.substitute(opts)
@@ -1067,7 +1104,15 @@ class TestStatsHttpd(unittest.TestCase):
         self.assertEqual('@description', stats_xsl[2].find('%sif' % nst).attrib['test'])
         self.assertEqual('@description', stats_xsl[2].find('%sif/%svalue-of' % ((nst,)*2)).attrib['select'])
 
+class Z_TestOSEnv(unittest.TestCase):
     def test_for_without_B10_FROM_SOURCE(self):
+        # Note: this test is sensitive due to its substantial side effect of
+        # reloading.  For exmaple, it affects tests that tweak module
+        # attributes (such as test_init_hterr).  It also breaks logging
+        # setting for unit tests.  To minimize these effects, we use
+        # workaround: make it very likely to run at the end of the tests
+        # by naming the test class "Z_".
+
         # just lets it go through the code without B10_FROM_SOURCE env
         # variable
         if "B10_FROM_SOURCE" in os.environ:

src/bin/stats/tests/b10-stats_test.py

@@ -23,7 +23,6 @@ to real environment.
 
 import unittest
 import os
-import threading
 import io
 import time
 import imp
@@ -31,10 +30,7 @@ import sys
 
 import stats
 import isc.log
-import isc.cc.session
-from test_utils import BaseModules, ThreadingServerManager, MyStats, \
-    SimpleStats, SignalHandler, MyModuleCCSession, send_command
-from isc.testutils.ccsession_mock import MockModuleCCSession
+from test_utils import MyStats
 
 class TestUtilties(unittest.TestCase):
     items = [
@@ -91,9 +87,15 @@ class TestUtilties(unittest.TestCase):
         self.const_timestamp = 1308730448.965706
         self.const_timetuple = (2011, 6, 22, 8, 14, 8, 2, 173, 0)
         self.const_datetime = '2011-06-22T08:14:08Z'
+        self.__orig_time = stats.time
+        self.__orig_gmtime = stats.gmtime
         stats.time = lambda : self.const_timestamp
         stats.gmtime = lambda : self.const_timetuple
 
+    def tearDown(self):
+        stats.time = self.__orig_time
+        stats.gmtime = self.__orig_gmtime
+
     def test_get_spec_defaults(self):
         self.assertEqual(
             stats.get_spec_defaults(self.items), {
@@ -243,8 +245,6 @@ class TestCallback(unittest.TestCase):
 class TestStats(unittest.TestCase):
     def setUp(self):
         # set the signal handler for deadlock
-        self.sig_handler = SignalHandler(self.fail)
-        self.base = BaseModules()
         self.const_timestamp = 1308730448.965706
         self.const_datetime = '2011-06-22T08:14:08Z'
         self.const_default_datetime = '1970-01-01T00:00:00Z'
@@ -253,15 +253,12 @@ class TestStats(unittest.TestCase):
         self.__orig_get_datetime = stats.get_datetime
 
     def tearDown(self):
-        self.base.shutdown()
-        # reset the signal handler
-        self.sig_handler.reset()
         # restore the stored original function in case we replaced them
         stats.get_timestamp = self.__orig_timestamp
         stats.get_datetime = self.__orig_get_datetime
 
     def test_init(self):
-        self.stats = stats.Stats()
+        self.stats = MyStats()
         self.assertEqual(self.stats.module_name, 'Stats')
         self.assertFalse(self.stats.running)
         self.assertTrue('command_show' in self.stats.callbacks)
@@ -291,7 +288,7 @@ class TestStats(unittest.TestCase):
 """
         orig_spec_location = stats.SPECFILE_LOCATION
         stats.SPECFILE_LOCATION = io.StringIO(spec_str)
-        self.assertRaises(stats.StatsError, stats.Stats)
+        self.assertRaises(stats.StatsError, MyStats)
         stats.SPECFILE_LOCATION = orig_spec_location
 
     def __send_command(self, stats, command_name, params=None):
@@ -310,13 +307,13 @@ class TestStats(unittest.TestCase):
             raise CheckException # terminate the loop
 
         # start without err
-        stats = SimpleStats()
-        self.assertFalse(stats.running)
-        stats._check_command = lambda: __check_start(stats)
+        self.stats = MyStats()
+        self.assertFalse(self.stats.running)
+        self.stats._check_command = lambda: __check_start(self.stats)
         # We are going to confirm start() will set running to True, avoiding
         # to fall into a loop with the exception trick.
-        self.assertRaises(CheckException, stats.start)
-        self.assertEqual(self.__send_command(stats, "status"),
+        self.assertRaises(CheckException, self.stats.start)
+        self.assertEqual(self.__send_command(self.stats, "status"),
                          (0, "Stats is up. (PID " + str(os.getpid()) + ")"))
 
     def test_shutdown(self):
@@ -328,15 +325,15 @@ class TestStats(unittest.TestCase):
             # override get_interval() so it won't go poll statistics
             tested_stats.get_interval = lambda : 0
 
-        stats = SimpleStats()
-        stats._check_command = lambda: __check_shutdown(stats)
-        stats.start()
-        self.assertTrue(stats.mccs.stopped)
+        self.stats = MyStats()
+        self.stats._check_command = lambda: __check_shutdown(self.stats)
+        self.stats.start()
+        self.assertTrue(self.stats.mccs.stopped)
 
     def test_handlers(self):
         """Test command_handler"""
 
-        __stats = SimpleStats()
+        __stats = MyStats()
 
         # 'show' command.  We're going to check the expected methods are
         # called in the expected order, and check the resulting response.
@@ -433,7 +430,7 @@ class TestStats(unittest.TestCase):
                         }]}
             return answer_value
 
-        self.stats = SimpleStats()
+        self.stats = MyStats()
         self.stats.cc_session.rpc_call = __check_rpc_call
 
         self.stats.update_modules()
@@ -480,7 +477,7 @@ class TestStats(unittest.TestCase):
         where we set the expected data in statistics_data.
 
         """
-        self.stats = SimpleStats()
+        self.stats = MyStats()
         def __faked_update_modules():
             self.stats.statistics_data = { \
                 'Stats': {
@@ -539,7 +536,7 @@ class TestStats(unittest.TestCase):
 
     def test_update_statistics_data(self):
         """test for list-type statistics"""
-        self.stats = SimpleStats()
+        self.stats = MyStats()
         _test_exp1 = {
               'zonename': 'test1.example',
               'queries.tcp': 5,
@@ -616,7 +613,7 @@ class TestStats(unittest.TestCase):
 
     def test_update_statistics_data_pt2(self):
         """test for named_set-type statistics"""
-        self.stats = SimpleStats()
+        self.stats = MyStats()
         _test_exp1 = \
             { 'test10.example': { 'queries.tcp': 5, 'queries.udp': 4 } }
         _test_exp2 = \
@@ -686,7 +683,7 @@ class TestStats(unittest.TestCase):
                 'Foo', 'foo1', _test_exp6), ['unknown module name: Foo'])
 
     def test_update_statistics_data_withmid(self):
-        self.stats = SimpleStats()
+        self.stats = MyStats()
 
         # This test relies on existing statistics data at the Stats object.
         # This version of test prepares the data using the do_polling() method;
@@ -701,7 +698,7 @@ class TestStats(unittest.TestCase):
         # We use the knowledge of what kind of messages are sent via
         # do_polling, and return the following faked answer directly.
         create_answer = isc.config.ccsession.create_answer # shortcut
-        self.stats._answers = [\
+        self.stats._answers = [
             # Answer for "show_processes"
             (create_answer(0, [[1034, 'b10-auth-1', 'Auth'],
                                [1035, 'b10-auth-2', 'Auth']]),  None),
@@ -754,7 +751,6 @@ class TestStats(unittest.TestCase):
         self.assertEqual(self.stats.statistics_data_bymid['Auth']['bar2@foo'],
                          {'queries.tcp': bar2_tcp})
         # kill running Auth but the statistics data doesn't change
-        self.base.auth2.server.shutdown()
         self.stats.update_statistics_data()
         self.assertTrue('Auth' in self.stats.statistics_data)
         self.assertTrue('queries.tcp' in self.stats.statistics_data['Auth'])
@@ -765,7 +761,6 @@ class TestStats(unittest.TestCase):
                          sum_qudp)
         self.assertTrue('Auth' in self.stats.statistics_data_bymid)
         # restore statistics data of killed auth
-        # self.base.b10_init.server.pid_list = [ killed ] + self.base.b10_init.server.pid_list[:]
         self.stats.update_statistics_data('Auth',
                                           "bar1@foo",
                                           {'queries.tcp': bar1_tcp})
@@ -794,7 +789,7 @@ class TestStats(unittest.TestCase):
     def test_config(self):
         orig_get_timestamp = stats.get_timestamp
         stats.get_timestamp = lambda : self.const_timestamp
-        stat = SimpleStats()
+        stat = MyStats()
 
         # test updating poll-interval
         self.assertEqual(stat.config['poll-interval'], 60)
@@ -840,7 +835,7 @@ class TestStats(unittest.TestCase):
             (0, {'Init': {'boot_time': self.const_datetime}}))
 
     def test_commands(self):
-        self.stats = stats.Stats()
+        self.stats = MyStats()
 
         # status
         self.assertEqual(self.stats.command_status(),
@@ -853,39 +848,57 @@ class TestStats(unittest.TestCase):
                          isc.config.create_answer(0))
         self.assertFalse(self.stats.running)
 
-    @unittest.skipIf(sys.version_info >= (3, 3), "Unsupported in Python 3.3 or higher")
-    def test_command_show(self):
-        # two auth instances invoked
-        list_auth = [ self.base.auth.server,
-                      self.base.auth2.server ]
-        sum_qtcp = 0
-        sum_qudp = 0
-        sum_qtcp_perzone1 = 0
-        sum_qudp_perzone1 = 0
-        sum_qtcp_perzone2 = 4 * len(list_auth)
-        sum_qudp_perzone2 = 3 * len(list_auth)
-        sum_qtcp_nds_perzone10 = 0
-        sum_qudp_nds_perzone10 = 0
-        sum_qtcp_nds_perzone20 = 4 * len(list_auth)
-        sum_qudp_nds_perzone20 = 3 * len(list_auth)
-        self.stats = stats.Stats()
+    def test_command_show_error(self):
+        self.stats = MyStats()
         self.assertEqual(self.stats.command_show(owner='Foo', name=None),
                          isc.config.create_answer(
-                1, "specified arguments are incorrect: owner: Foo, name: None"))
+                1,
+                "specified arguments are incorrect: owner: Foo, name: None"))
         self.assertEqual(self.stats.command_show(owner='Foo', name='_bar_'),
                          isc.config.create_answer(
-                1, "specified arguments are incorrect: owner: Foo, name: _bar_"))
+                1,
+                "specified arguments are incorrect: owner: Foo, name: _bar_"))
         self.assertEqual(self.stats.command_show(owner='Foo', name='bar'),
                          isc.config.create_answer(
-                1, "specified arguments are incorrect: owner: Foo, name: bar"))
+                1,
+                "specified arguments are incorrect: owner: Foo, name: bar"))
 
-        for a in list_auth:
-            sum_qtcp += a.queries_tcp
-            sum_qudp += a.queries_udp
-            sum_qtcp_perzone1 += a.queries_per_zone[0]['queries.tcp']
-            sum_qudp_perzone1 += a.queries_per_zone[0]['queries.udp']
-            sum_qtcp_nds_perzone10 += a.nds_queries_per_zone['test10.example']['queries.tcp']
-            sum_qudp_nds_perzone10 += a.nds_queries_per_zone['test10.example']['queries.udp']
+    def test_command_show_auth(self):
+        self.stats = MyStats()
+        self.stats.update_modules = lambda: None
+
+        # Test data borrowed from test_update_statistics_data_withmid
+        create_answer = isc.config.ccsession.create_answer # shortcut
+        self.stats._answers = [
+            (create_answer(0, [[1034, 'b10-auth-1', 'Auth'],
+                               [1035, 'b10-auth-2', 'Auth']]),  None),
+            (create_answer(0, self.stats._auth_sdata), {'from': 'auth1'}),
+            (create_answer(0, self.stats._auth_sdata), {'from': 'auth2'}),
+            (create_answer(0, self.stats._auth_sdata), {'from': 'auth3'})
+            ]
+
+        num_instances = 2
+        sum_qtcp = 0
+        sum_qudp = 0
+        sum_qtcp_perzone1 = 0
+        sum_qudp_perzone1 = 0
+        sum_qtcp_perzone2 = 4 * num_instances
+        sum_qudp_perzone2 = 3 * num_instances
+        sum_qtcp_nds_perzone10 = 0
+        sum_qudp_nds_perzone10 = 0
+        sum_qtcp_nds_perzone20 = 4 * num_instances
+        sum_qudp_nds_perzone20 = 3 * num_instances
+
+        self.maxDiff = None
+        for a in (0, num_instances):
+            sum_qtcp += self.stats._queries_tcp
+            sum_qudp += self.stats._queries_udp
+            sum_qtcp_perzone1 += self.stats._queries_per_zone[0]['queries.tcp']
+            sum_qudp_perzone1 += self.stats._queries_per_zone[0]['queries.udp']
+            sum_qtcp_nds_perzone10 += \
+                self.stats._nds_queries_per_zone['test10.example']['queries.tcp']
+            sum_qudp_nds_perzone10 += \
+                self.stats._nds_queries_per_zone['test10.example']['queries.udp']
 
         self.assertEqual(self.stats.command_show(owner='Auth'),
                          isc.config.create_answer(
@@ -926,26 +939,33 @@ class TestStats(unittest.TestCase):
                             'test20.example': {
                                 'queries.udp': sum_qudp_nds_perzone20,
                                 'queries.tcp': sum_qtcp_nds_perzone20 }}}}))
+
+    def test_command_show_stats(self):
+        self.stats = MyStats()
         orig_get_datetime = stats.get_datetime
         orig_get_timestamp = stats.get_timestamp
         stats.get_datetime = lambda x=None: self.const_datetime
         stats.get_timestamp = lambda : self.const_timestamp
-        self.assertEqual(self.stats.command_show(owner='Stats', name='report_time'),
+        self.assertEqual(self.stats.command_show(owner='Stats',
+                                                 name='report_time'),
                          isc.config.create_answer(
                 0, {'Stats': {'report_time':self.const_datetime}}))
-        self.assertEqual(self.stats.command_show(owner='Stats', name='timestamp'),
+        self.assertEqual(self.stats.command_show(owner='Stats',
+                                                 name='timestamp'),
                          isc.config.create_answer(
                 0, {'Stats': {'timestamp':self.const_timestamp}}))
         stats.get_datetime = orig_get_datetime
         stats.get_timestamp = orig_get_timestamp
-        self.stats.modules[self.stats.module_name] = isc.config.module_spec.ModuleSpec(
-            { "module_name": self.stats.module_name,
-              "statistics": [] } )
+        self.stats.do_polling = lambda : None
+        self.stats.modules[self.stats.module_name] = \
+            isc.config.module_spec.ModuleSpec(
+            { "module_name": self.stats.module_name, "statistics": [] } )
         self.assertRaises(
-            stats.StatsError, self.stats.command_show, owner=self.stats.module_name, name='bar')
+            stats.StatsError, self.stats.command_show,
+            owner=self.stats.module_name, name='bar')
 
     def test_command_showchema(self):
-        self.stats = stats.Stats()
+        self.stats = MyStats()
         (rcode, value) = isc.config.ccsession.parse_answer(
             self.stats.command_showschema())
         self.assertEqual(rcode, 0)
@@ -1261,98 +1281,130 @@ class TestStats(unittest.TestCase):
                          isc.config.create_answer(
                 1, "module name is not specified"))
 
-    @unittest.skipIf(sys.version_info >= (3, 3), "Unsupported in Python 3.3 or higher")
-    def test_polling(self):
-        stats_server = ThreadingServerManager(MyStats)
-        stat = stats_server.server
-        stats_server.run()
-        self.assertEqual(
-            send_command('show', 'Stats'),
-            (0, stat.statistics_data))
-        # check statistics data of 'Init'
-        b10_init = self.base.b10_init.server
-        self.assertEqual(
-            stat.statistics_data_bymid['Init'][b10_init.cc_session.lname],
-            {'boot_time': self.const_datetime})
-        self.assertEqual(
-            len(stat.statistics_data_bymid['Init']), 1)
+    def test_polling_init(self):
+        """check statistics data of 'Init'."""
+
+        stat = MyStats()
+        stat.update_modules = lambda: None
+        create_answer = isc.config.ccsession.create_answer # shortcut
+
+        stat._answers = [
+            # Answer for "show_processes"
+            (create_answer(0, []),  None),
+            # Answers for "getstats" for Init (type of boot_time is invalid)
+            (create_answer(0, {'boot_time': self.const_datetime}),
+             {'from': 'init'}),
+            ]
+
+        stat.do_polling()
         self.assertEqual(
-            stat.statistics_data['Init'],
+            stat.statistics_data_bymid['Init']['init'],
             {'boot_time': self.const_datetime})
-        # check statistics data of each 'Auth' instances
-        list_auth = ['', '2']
-        for i in list_auth:
-            auth = getattr(self.base,"auth"+i).server
-            for s in stat.statistics_data_bymid['Auth'].values():
-                self.assertEqual(
-                    s, {'queries.perzone': auth.queries_per_zone,
-                        'nds_queries.perzone': auth.nds_queries_per_zone,
-                        'queries.tcp': auth.queries_tcp,
-                        'queries.udp': auth.queries_udp})
-            n = len(stat.statistics_data_bymid['Auth'])
-            self.assertEqual(n, len(list_auth))
-            # check consolidation of statistics data of the auth
-            # instances
+
+    def test_polling_consolidate(self):
+        """check statistics data of multiple instances of same module."""
+        stat = MyStats()
+        stat.update_modules = lambda: None
+        create_answer = isc.config.ccsession.create_answer # shortcut
+
+        # Test data borrowed from test_update_statistics_data_withmid
+        stat._answers = [
+            (create_answer(0, [[1034, 'b10-auth-1', 'Auth'],
+                               [1035, 'b10-auth-2', 'Auth']]),  None),
+            (create_answer(0, stat._auth_sdata), {'from': 'auth1'}),
+            (create_answer(0, stat._auth_sdata), {'from': 'auth2'}),
+            (create_answer(0, stat._auth_sdata), {'from': 'auth3'})
+            ]
+
+        stat.do_polling()
+
+        # check statistics data of each 'Auth' instances.  expected data
+        # for 'nds_queries.perzone' is special as it needs data merge.
+        self.assertEqual(2, len(stat.statistics_data_bymid['Auth'].values()))
+        for s in stat.statistics_data_bymid['Auth'].values():
             self.assertEqual(
-                stat.statistics_data['Auth'],
-                {'queries.perzone': [
-                        {'zonename':
-                             auth.queries_per_zone[0]['zonename'],
-                         'queries.tcp':
-                             auth.queries_per_zone[0]['queries.tcp']*n,
-                         'queries.udp':
-                             auth.queries_per_zone[0]['queries.udp']*n},
-                        {'zonename': "test2.example",
-                         'queries.tcp': 4*n,
-                         'queries.udp': 3*n },
-                        ],
-                 'nds_queries.perzone': {
-                         'test10.example': {
-                             'queries.tcp':
-                                 auth.nds_queries_per_zone['test10.example']['queries.tcp']*n,
-                             'queries.udp':
-                                 auth.nds_queries_per_zone['test10.example']['queries.udp']*n},
-                         'test20.example': {
-                             'queries.tcp':
-                                 4*n,
-                             'queries.udp':
-                                 3*n},
-                         },
-                 'queries.tcp': auth.queries_tcp*n,
-                 'queries.udp': auth.queries_udp*n})
-        # check statistics data of 'Stats'
+                s, {'queries.perzone': stat._auth_sdata['queries.perzone'],
+                    'nds_queries.perzone': stat._nds_queries_per_zone,
+                    'queries.tcp': stat._auth_sdata['queries.tcp'],
+                    'queries.udp': stat._auth_sdata['queries.udp']})
+
+        # check consolidation of statistics data of the auth instances.
+        # it's union of the reported data and the spec default.
+        n = len(stat.statistics_data_bymid['Auth'].values())
+        self.maxDiff = None
         self.assertEqual(
-            len(stat.statistics_data['Stats']), 5)
-        self.assertTrue('boot_time' in
-            stat.statistics_data['Stats'])
-        self.assertTrue('last_update_time' in
-            stat.statistics_data['Stats'])
-        self.assertTrue('report_time' in
-            stat.statistics_data['Stats'])
-        self.assertTrue('timestamp' in
-            stat.statistics_data['Stats'])
-        self.assertEqual(
-            stat.statistics_data['Stats']['lname'],
-            stat.mccs._session.lname)
-        stats_server.shutdown()
+            stat.statistics_data['Auth'],
+            {'queries.perzone': [
+                    {'zonename': 'test1.example',
+                     'queries.tcp': 5 * n,
+                     'queries.udp': 4 * n},
+                    {'zonename': 'test2.example',
+                     'queries.tcp': 4 * n,
+                     'queries.udp': 3 * n},
+                    ],
+             'nds_queries.perzone': {
+                    'test10.example': {
+                        'queries.tcp': 5 * n,
+                        'queries.udp': 4 * n
+                        },
+                    'test20.example': {
+                        'queries.tcp': 4 * n,
+                        'queries.udp': 3 * n
+                        },
+                    },
+             'queries.tcp': 3 * n,
+             'queries.udp': 2 * n})
+
+    def test_polling_stats(self):
+        """Check statistics data of 'Stats'
+
+        This is actually irrelevant to do_polling(), but provided to
+        compatibility of older tests.
+
+        """
+        stat = MyStats()
+        self.assertEqual(len(stat.statistics_data['Stats']), 5)
+        self.assertTrue('boot_time' in stat.statistics_data['Stats'])
+        self.assertTrue('last_update_time' in stat.statistics_data['Stats'])
+        self.assertTrue('report_time' in stat.statistics_data['Stats'])
+        self.assertTrue('timestamp' in stat.statistics_data['Stats'])
+        self.assertEqual(stat.statistics_data['Stats']['lname'],
+                         stat.mccs._session.lname)
 
     def test_polling2(self):
-        # set invalid statistics
-        b10_init = self.base.b10_init.server
-        b10_init.statistics_data = {'boot_time':1}
-        stats_server = ThreadingServerManager(MyStats)
-        stat = stats_server.server
-        stats_server.run()
-        self.assertEqual(
-            send_command('status', 'Stats'),
-            (0, "Stats is up. (PID " + str(os.getpid()) + ")"))
+        """Test do_polling() doesn't incorporate broken statistics data.
+
+        Actually, this is not a test for do_polling() itself.  It's bad, but
+        fixing that is a subject of different ticket.
+
+        """
+        stat = MyStats()
         # check default statistics data of 'Init'
         self.assertEqual(
-            stat.statistics_data['Init'],
-            {'boot_time': self.const_default_datetime})
-        stats_server.shutdown()
+             stat.statistics_data['Init'],
+             {'boot_time': self.const_default_datetime})
+
+        # set invalid statistics
+        create_answer = isc.config.ccsession.create_answer # shortcut
+        stat._answers = [
+            # Answer for "show_processes"
+            (create_answer(0, []),  None),
+            # Answers for "getstats" for Init (type of boot_time is invalid)
+            (create_answer(0, {'boot_time': 1}), {'from': 'init'}),
+            ]
+        stat.update_modules = lambda: None
+
+        # do_polling() should ignore the invalid answer;
+        # default data shouldn't be replaced.
+        stat.do_polling()
+        self.assertEqual(
+             stat.statistics_data['Init'],
+             {'boot_time': self.const_default_datetime})
 
-class TestOSEnv(unittest.TestCase):
+class Z_TestOSEnv(unittest.TestCase):
+    # Running this test would break logging setting.  To prevent it from
+    # affecting other tests we use the same workaround as
+    # Z_TestStatsHttpdError.
     def test_osenv(self):
         """
         test for the environ variable "B10_FROM_SOURCE"

src/bin/stats/tests/test_utils.py

@@ -20,13 +20,11 @@ Utilities and mock modules for unittests of statistics modules
 import os
 import io
 import time
-import sys
 import threading
-import tempfile
 import json
 import signal
+import socket
 
-import msgq
 import isc.config.cfgmgr
 import stats
 import stats_httpd
@@ -51,19 +49,6 @@ class SignalHandler():
         """invokes unittest.TestCase.fail as a signal handler"""
         self.fail_handler("A deadlock might be detected")
 
-def send_command(command_name, module_name, params=None):
-    cc_session = isc.cc.Session()
-    command = isc.config.ccsession.create_command(command_name, params)
-    seq = cc_session.group_sendmsg(command, module_name)
-    try:
-        (answer, env) = cc_session.group_recvmsg(False, seq)
-        if answer:
-            return isc.config.ccsession.parse_answer(answer)
-    except isc.cc.SessionTimeout:
-        pass
-    finally:
-        cc_session.close()
-
 class ThreadingServerManager:
     def __init__(self, server, *args, **kwargs):
         self.server = server(*args, **kwargs)
@@ -91,45 +76,7 @@ class ThreadingServerManager:
         else:
             self.server._thread.join(0) # timeout is 0
 
-class MockMsgq:
-    def __init__(self):
-        self._started = threading.Event()
-        self.msgq = msgq.MsgQ(verbose=False)
-        result = self.msgq.setup()
-        if result:
-            sys.exit("Error on Msgq startup: %s" % result)
-
-    def run(self):
-        self._started.set()
-        try:
-            self.msgq.run()
-        finally:
-            # Make sure all the sockets, etc, are removed once it stops.
-            self.msgq.shutdown()
-
-    def shutdown(self):
-        # Ask it to terminate nicely
-        self.msgq.stop()
-
-class MockCfgmgr:
-    def __init__(self):
-        self._started = threading.Event()
-        self.cfgmgr = isc.config.cfgmgr.ConfigManager(
-            os.environ['CONFIG_TESTDATA_PATH'], "b10-config.db")
-        self.cfgmgr.read_config()
-
-    def run(self):
-        self._started.set()
-        try:
-            self.cfgmgr.run()
-        except Exception:
-            pass
-
-    def shutdown(self):
-        self.cfgmgr.running = False
-
-class MockInit:
-    spec_str = """\
+INIT_SPEC_STR = """\
 {
   "module_spec": {
     "module_name": "Init",
@@ -221,56 +168,12 @@ class MockInit:
   }
 }
 """
-    _BASETIME = CONST_BASETIME
 
-    def __init__(self):
-        self._started = threading.Event()
-        self.running = False
-        self.spec_file = io.StringIO(self.spec_str)
-        # create ModuleCCSession object
-        self.mccs = isc.config.ModuleCCSession(
-            self.spec_file,
-            self.config_handler,
-            self.command_handler)
-        self.spec_file.close()
-        self.cc_session = self.mccs._session
-        self.got_command_name = ''
-        self.pid_list = [[ 9999, "b10-auth", "Auth" ],
-                         [ 9998, "b10-auth-2", "Auth" ]]
-        self.statistics_data = {
-            'boot_time': time.strftime('%Y-%m-%dT%H:%M:%SZ', self._BASETIME)
-            }
-
-    def run(self):
-        self.mccs.start()
-        self.running = True
-        self._started.set()
-        try:
-            while self.running:
-                self.mccs.check_command(False)
-        except Exception:
-            pass
-
-    def shutdown(self):
-        self.running = False
-
-    def config_handler(self, new_config):
-        return isc.config.create_answer(0)
-
-    def command_handler(self, command, *args, **kwargs):
-        self._started.set()
-        self.got_command_name = command
-        sdata = self.statistics_data
-        if command == 'getstats':
-            return isc.config.create_answer(0, sdata)
-        elif command == 'show_processes':
-            # Return dummy pids
-            return isc.config.create_answer(
-                0, self.pid_list)
-        return isc.config.create_answer(1, "Unknown Command")
-
-class MockAuth:
-    spec_str = """\
+# Note: this is derived of the spec for the DNS authoritative server, but
+# for the purpose of this test, it's completely irrelevant to DNS.
+# Some statisittics specs do not make sense for practical sense but used
+# just cover various types of statistics data (list, map/dict, etc).
+AUTH_SPEC_STR = """\
 {
   "module_spec": {
     "module_name": "Auth",
@@ -392,68 +295,6 @@ class MockAuth:
   }
 }
 """
-    def __init__(self):
-        self._started = threading.Event()
-        self.running = False
-        self.spec_file = io.StringIO(self.spec_str)
-        # create ModuleCCSession object
-        self.mccs = isc.config.ModuleCCSession(
-            self.spec_file,
-            self.config_handler,
-            self.command_handler)
-        self.spec_file.close()
-        self.cc_session = self.mccs._session
-        self.got_command_name = ''
-        self.queries_tcp = 3
-        self.queries_udp = 2
-        self.queries_per_zone = [{
-                'zonename': 'test1.example',
-                'queries.tcp': 5,
-                'queries.udp': 4
-                }]
-        self.nds_queries_per_zone = {
-            'test10.example': {
-                'queries.tcp': 5,
-                'queries.udp': 4
-                }
-            }
-
-    def run(self):
-        self.mccs.start()
-        self.running = True
-        self._started.set()
-        try:
-            while self.running:
-                self.mccs.check_command(False)
-        except Exception:
-            pass
-
-    def shutdown(self):
-        self.running = False
-
-    def config_handler(self, new_config):
-        return isc.config.create_answer(0)
-
-    def command_handler(self, command, *args, **kwargs):
-        self.got_command_name = command
-        sdata = { 'queries.tcp': self.queries_tcp,
-                  'queries.udp': self.queries_udp,
-                  'queries.perzone' : self.queries_per_zone,
-                  'nds_queries.perzone' : {
-                    'test10.example': {
-                    'queries.tcp': \
-                      isc.cc.data.find(
-                        self.nds_queries_per_zone,
-                        'test10.example/queries.tcp')
-                    }
-                  },
-                  'nds_queries.perzone/test10.example/queries.udp' :
-                      isc.cc.data.find(self.nds_queries_per_zone,
-                                       'test10.example/queries.udp')
-                }
-        if command == 'getstats':
-            return isc.config.create_answer(0, sdata)
-        return isc.config.create_answer(1, "Unknown Command")
 
 class MyModuleCCSession(isc.config.ConfigData):
     """Mocked ModuleCCSession class.
@@ -468,6 +309,7 @@ class MyModuleCCSession(isc.config.ConfigData):
         isc.config.ConfigData.__init__(self, module_spec)
         self._session = self
         self.stopped = False
+        self.closed = False
         self.lname = 'mock_mod_ccs'
 
     def start(self):
@@ -476,10 +318,13 @@ class MyModuleCCSession(isc.config.ConfigData):
     def send_stopping(self):
         self.stopped = True     # just record it's called to inspect it later
 
-class SimpleStats(stats.Stats):
+    def close(self):
+        self.closed = True
+
+class MyStats(stats.Stats):
     """A faked Stats class for unit tests.
 
-    This class inherits most of the real Stats class, but replace the
+    This class inherits most of the real Stats class, but replaces the
     ModuleCCSession with a fake one so we can avoid network I/O in tests,
     and can also inspect or tweak messages via the session more easily.
     This class also maintains some faked module information and statistics
@@ -500,9 +345,9 @@ class SimpleStats(stats.Stats):
         # the default answer from faked recvmsg if _answers is empty
         self.__default_answer = isc.config.ccsession.create_answer(
             0, {'Init':
-                    json.loads(MockInit.spec_str)['module_spec']['statistics'],
+                    json.loads(INIT_SPEC_STR)['module_spec']['statistics'],
                 'Auth':
-                    json.loads(MockAuth.spec_str)['module_spec']['statistics']
+                    json.loads(AUTH_SPEC_STR)['module_spec']['statistics']
                 })
         # setup faked auth statistics
         self.__init_auth_stat()
@@ -530,24 +375,24 @@ class SimpleStats(stats.Stats):
     def __init_auth_stat(self):
         self._queries_tcp = 3
         self._queries_udp = 2
-        self.__queries_per_zone = [{
+        self._queries_per_zone = [{
                 'zonename': 'test1.example', 'queries.tcp': 5, 'queries.udp': 4
                 }]
-        self.__nds_queries_per_zone = \
+        self._nds_queries_per_zone = \
             { 'test10.example': { 'queries.tcp': 5, 'queries.udp': 4 } }
         self._auth_sdata = \
             { 'queries.tcp': self._queries_tcp,
               'queries.udp': self._queries_udp,
-              'queries.perzone' : self.__queries_per_zone,
+              'queries.perzone' : self._queries_per_zone,
               'nds_queries.perzone' : {
                 'test10.example': {
                     'queries.tcp': isc.cc.data.find(
-                        self.__nds_queries_per_zone,
+                        self._nds_queries_per_zone,
                         'test10.example/queries.tcp')
                     }
                 },
               'nds_queries.perzone/test10.example/queries.udp' :
-                  isc.cc.data.find(self.__nds_queries_per_zone,
+                  isc.cc.data.find(self._nds_queries_per_zone,
                                    'test10.example/queries.udp')
               }
 
@@ -589,32 +434,62 @@ class SimpleStats(stats.Stats):
         answer, _ = self.__group_recvmsg(None, None)
         return isc.config.ccsession.parse_answer(answer)[1]
 
-class MyStats(stats.Stats):
-
-    stats._BASETIME = CONST_BASETIME
-    stats.get_timestamp = lambda: time.mktime(CONST_BASETIME)
-    stats.get_datetime = lambda x=None: time.strftime("%Y-%m-%dT%H:%M:%SZ", CONST_BASETIME)
-
-    def __init__(self):
-        self._started = threading.Event()
-        stats.Stats.__init__(self)
+class MyStatsHttpd(stats_httpd.StatsHttpd):
+    """A faked StatsHttpd class for unit tests.
 
-    def run(self):
-        self._started.set()
-        try:
-            self.start()
-        except Exception:
-            pass
+    This class inherits most of the real StatsHttpd class, but replaces the
+    ModuleCCSession with a fake one so we can avoid network I/O in tests,
+    and can also inspect or tweak messages via the session more easily.
 
-    def shutdown(self):
-        self.command_shutdown()
+    """
 
-class MyStatsHttpd(stats_httpd.StatsHttpd):
     ORIG_SPECFILE_LOCATION = stats_httpd.SPECFILE_LOCATION
     def __init__(self, *server_address):
         self._started = threading.Event()
+        self.__dummy_sock = None # see below
+
+        # Prepare commonly used statistics schema and data requested in
+        # stats-httpd tests.  For the purpose of these tests, the content of
+        # statistics data is not so important (they don't test whther the
+        # counter values are correct, etc), so hardcoding the common case
+        # should suffice.  Note also that some of the statistics values and
+        # specs don't make sense in practice (see also comments on
+        # AUTH_SPEC_STR).
+        with open(stats.SPECFILE_LOCATION) as f:
+            stat_spec_str = f.read()
+        self.__default_spec_answer = {
+            'Init': json.loads(INIT_SPEC_STR)['module_spec']['statistics'],
+            'Auth': json.loads(AUTH_SPEC_STR)['module_spec']['statistics'],
+            'Stats': json.loads(stat_spec_str)['module_spec']['statistics']
+            }
+        self.__default_data_answer = {
+            'Init': {'boot_time':
+                         time.strftime('%Y-%m-%dT%H:%M:%SZ', CONST_BASETIME)},
+            'Stats': {'last_update_time':
+                          time.strftime('%Y-%m-%dT%H:%M:%SZ', CONST_BASETIME),
+                      'report_time':
+                          time.strftime('%Y-%m-%dT%H:%M:%SZ', CONST_BASETIME),
+                      'lname': 'test-lname',
+                      'boot_time':
+                          time.strftime('%Y-%m-%dT%H:%M:%SZ', CONST_BASETIME),
+                      'timestamp': time.mktime(CONST_BASETIME)},
+            'Auth': {'queries.udp': 4, 'queries.tcp': 6,
+                     'queries.perzone': [
+                    {'queries.udp': 8, 'queries.tcp': 10,
+                     'zonename': 'test1.example'},
+                    {'queries.udp': 6, 'queries.tcp': 8,
+                     'zonename': 'test2.example'}],
+                     'nds_queries.perzone': {
+                    'test10.example': {'queries.udp': 8, 'queries.tcp': 10},
+                    'test20.example': {'queries.udp': 6, 'queries.tcp': 8}}}}
+
+        # if set, use them as faked response to rpc_call (see below).
+        # it's a list of answer data of rpc_call.
+        self._rpc_answers = []
+
         if server_address:
-            stats_httpd.SPECFILE_LOCATION = self.create_specfile(*server_address)
+            stats_httpd.SPECFILE_LOCATION = \
+                self.__create_specfile(*server_address)
             try:
                 stats_httpd.StatsHttpd.__init__(self)
             finally:
@@ -624,7 +499,51 @@ class MyStatsHttpd(stats_httpd.StatsHttpd):
         else:
             stats_httpd.StatsHttpd.__init__(self)
 
-    def create_specfile(self, *server_address):
+        # replace some (faked) ModuleCCSession methods so we can inspect/fake.
+        # in order to satisfy select.select() we need some real socket.  We
+        # use an unusable AF_UNIX socket; we won't actually use it for
+        # communication.
+        self.cc_session.rpc_call = self.__rpc_call
+        self.__dummy_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        self.mccs.get_socket = lambda: self.__dummy_sock
+
+    def open_mccs(self):
+        self.mccs = MyModuleCCSession(stats_httpd.SPECFILE_LOCATION,
+                                      self.config_handler,
+                                      self.command_handler)
+        self.cc_session = self.mccs._session
+        self.mccs.start = self.load_config # force reload
+
+    def close_mccs(self):
+        super().close_mccs()
+        if self.__dummy_sock is not None:
+            self.__dummy_sock.close()
+            self.__dummy_sock = None
+
+    def __rpc_call(self, command, group, params={}):
+        """Faked ModuleCCSession.rpc_call for tests.
+
+        The stats httpd module only issues two commands: 'showschema' and
+        'show'.  In most cases we can simply use the prepared default
+        answer.  If customization is needed, the test case can add a
+        faked answer by appending it to _rpc_answers.  If the added object
+        is of Exception type this method raises it instead of return it,
+        emulating the situation where rpc_call() results in an exception.
+
+        """
+        if len(self._rpc_answers) == 0:
+            if command == 'showschema':
+                return self.__default_spec_answer
+            elif command == 'show':
+                return self.__default_data_answer
+            assert False, "unexpected command for faked rpc_call: " + command
+
+        answer = self._rpc_answers.pop(0)
+        if issubclass(type(answer), Exception):
+            raise answer
+        return answer
+
+    def __create_specfile(self, *server_address):
         spec_io = open(self.ORIG_SPECFILE_LOCATION)
         try:
             spec = json.load(spec_io)
@@ -633,7 +552,8 @@ class MyStatsHttpd(stats_httpd.StatsHttpd):
             for i in range(len(config)):
                 if config[i]['item_name'] == 'listen_on':
                     config[i]['item_default'] = \
-                        [ dict(address=a[0], port=a[1]) for a in server_address ]
+                        [ dict(address=a[0], port=a[1])
+                          for a in server_address ]
                     break
             return io.StringIO(json.dumps(spec))
         finally:
@@ -641,53 +561,4 @@ class MyStatsHttpd(stats_httpd.StatsHttpd):
 
     def run(self):
         self._started.set()
-        try:
-            self.start()
-        except Exception:
-            pass
-
-    def shutdown(self):
-        self.command_handler('shutdown', None)
-
-class BaseModules:
-    def __init__(self):
-        # MockMsgq
-        self.msgq = ThreadingServerManager(MockMsgq)
-        self.msgq.run()
-        # Check whether msgq is ready. A SessionTimeout is raised here if not.
-        isc.cc.session.Session().close()
-        # MockCfgmgr
-        self.cfgmgr = ThreadingServerManager(MockCfgmgr)
-        self.cfgmgr.run()
-        # MockInit
-        self.b10_init = ThreadingServerManager(MockInit)
-        self.b10_init.run()
-        # MockAuth
-        self.auth = ThreadingServerManager(MockAuth)
-        self.auth.run()
-        self.auth2 = ThreadingServerManager(MockAuth)
-        self.auth2.run()
-
-
-    def shutdown(self):
-        # MockMsgq. We need to wait (blocking) for it, otherwise it'll wipe out
-        # a socket for another test during its shutdown.
-        self.msgq.shutdown(True)
-
-        # We also wait for the others, but these are just so we don't create
-        # too many threads in parallel.
-
-        # MockAuth
-        self.auth2.shutdown(True)
-        self.auth.shutdown(True)
-        # MockInit
-        self.b10_init.shutdown(True)
-        # MockCfgmgr
-        self.cfgmgr.shutdown(True)
-        # remove the unused socket file
-        socket_file = self.msgq.server.msgq.socket_file
-        try:
-            if os.path.exists(socket_file):
-                os.remove(socket_file)
-        except OSError:
-            pass
+        self.start()

src/bin/xfrin/b10-xfrin.xml

@@ -213,6 +213,132 @@ operation
 -->
 
   </refsect1>
+  <refsect1>
+    <title>STATISTICS DATA</title>
+
+    <para>
+      The statistics data collected by the <command>b10-xfrin</command>
+      daemon for <quote>Xfrin</quote> include:
+    </para>
+
+    <variablelist>
+
+      <varlistentry>
+        <term>zones</term>
+        <listitem><simpara>
+          A directory name of per-zone statistics
+          </simpara>
+          <variablelist>
+
+            <varlistentry>
+              <term><replaceable>zonename</replaceable></term>
+              <listitem><simpara>
+                An actual zone name or special zone name
+                <quote>_SERVER_</quote> representing the entire server.
+                Zone classes (e.g. IN, CH, and HS) are mixed and counted so
+                far. But these will be distinguished in future release.
+                </simpara>
+                <variablelist>
+
+                  <varlistentry>
+                    <term>soaoutv4</term>
+                    <listitem><simpara>
+                      Number of IPv4 SOA queries sent from Xfrin
+                    </simpara></listitem>
+                  </varlistentry>
+
+                  <varlistentry>
+                    <term>soaoutv6</term>
+                    <listitem><simpara>
+                      Number of IPv6 SOA queries sent from Xfrin
+                    </simpara></listitem>
+                  </varlistentry>
+
+                  <varlistentry>
+                    <term>axfrreqv4</term>
+                    <listitem><simpara>
+                      Number of IPv4 AXFR requests sent from Xfrin
+                    </simpara></listitem>
+                  </varlistentry>
+
+                  <varlistentry>
+                    <term>axfrreqv6</term>
+                    <listitem><simpara>
+                      Number of IPv6 AXFR requests sent from Xfrin
+                    </simpara></listitem>
+                  </varlistentry>
+
+                  <varlistentry>
+                    <term>ixfrreqv4</term>
+                    <listitem><simpara>
+                      Number of IPv4 IXFR requests sent from Xfrin
+                    </simpara></listitem>
+                  </varlistentry>
+
+                  <varlistentry>
+                    <term>ixfrreqv6</term>
+                    <listitem><simpara>
+                      Number of IPv6 IXFR requests sent from Xfrin
+                    </simpara></listitem>
+                  </varlistentry>
+
+                  <varlistentry>
+                    <term>xfrsuccess</term>
+                    <listitem><simpara>
+                      Number of zone transfer requests succeeded.
+                      These include the case where the zone turns
+                      out to be the latest as a result of an
+                      initial SOA query (and there is actually no
+                      AXFR or IXFR transaction).
+                    </simpara></listitem>
+                  </varlistentry>
+
+                  <varlistentry>
+                    <term>xfrfail</term>
+                    <listitem><simpara>
+                      Number of zone transfer requests failed
+                    </simpara></listitem>
+                  </varlistentry>
+
+                  <varlistentry>
+                    <term>last_axfr_duration</term>
+                    <listitem><simpara>
+                      Duration in seconds of the last successful AXFR.  0.0
+                      means no successful AXFR done or means a successful AXFR
+                      done in less than a microsecond.  If an AXFR is aborted
+                      due to some failure, this duration won't be updated.
+                    </simpara></listitem>
+                  </varlistentry>
+
+                  <varlistentry>
+                    <term>last_ixfr_duration</term>
+                    <listitem><simpara>
+                      Duration in seconds of the last successful IXFR.  0.0
+                      means no successful IXFR done or means a successful IXFR
+                      done in less than a microsecond.  If an IXFR is aborted
+                      due to some failure, this duration won't be updated.
+                    </simpara></listitem>
+                  </varlistentry>
+
+                </variablelist>
+              </listitem>
+            </varlistentry><!-- end of zonename -->
+
+          </variablelist>
+        </listitem>
+      </varlistentry><!-- end of zones -->
+
+    </variablelist>
+
+    <para>
+      In per-zone counters the special zone name <quote>_SERVER_</quote>
+      exists.
+      It doesn't mean a specific zone. It represents the entire server
+      and the counter value of this special zone is the total of the
+      same counter for all zones.
+    </para>
+
+  </refsect1>
 
 <!--
   <refsect1>

src/bin/xfrin/tests/xfrin_test.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2009-2011  Internet Systems Consortium.
+# Copyright (C) 2009-2013  Internet Systems Consortium.
 #
 # Permission to use, copy, modify, and distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
@@ -19,6 +19,7 @@ import shutil
 import socket
 import sys
 import io
+from datetime import datetime
 from isc.testutils.tsigctx_mock import MockTSIGContext
 from isc.testutils.ccsession_mock import MockModuleCCSession
 from isc.testutils.rrset_utils import *
@@ -717,7 +718,7 @@ class TestXfrinConnection(unittest.TestCase):
         self.sock_map = {}
         self.conn = MockXfrinConnection(self.sock_map, TEST_ZONE_NAME,
                                         TEST_RRCLASS, None, threading.Event(),
-                                        TEST_MASTER_IPV4_ADDRINFO)
+                                        self._master_addrinfo)
         self.conn.init_socket()
         self.soa_response_params = {
             'questions': [example_soa_question],
@@ -749,6 +750,10 @@ class TestXfrinConnection(unittest.TestCase):
             os.remove(TEST_DB_FILE)
         xfrin.check_zone = self.__orig_check_zone
 
+    @property
+    def _master_addrinfo(self):
+        return TEST_MASTER_IPV4_ADDRINFO
+
     def __check_zone(self, name, rrclass, rrsets, callbacks):
         '''
         A mock function used instead of dns.check_zone.
@@ -1065,6 +1070,20 @@ class TestAXFR(TestXfrinConnection):
         self.assertRaises(XfrinProtocolError,
                           self.conn._handle_xfrin_responses)
 
+    def test_ipver_str(self):
+        addrs = (((socket.AF_INET, socket.SOCK_STREAM), 'v4'),
+                 ((socket.AF_INET6, socket.SOCK_STREAM), 'v6'),
+                 ((socket.AF_UNIX, socket.SOCK_STREAM), None))
+        for (info, ver) in addrs:
+            c = MockXfrinConnection({}, TEST_ZONE_NAME, RRClass.CH, None,
+                                    threading.Event(), info)
+            c.init_socket()
+            if ver is not None:
+                self.assertEqual(ver, c._get_ipver_str())
+            else:
+                self.assertRaises(ValueError, c._get_ipver_str)
+            c.close()
+
     def test_soacheck(self):
         # we need to defer the creation until we know the QID, which is
         # determined in _check_soa_serial(), so we use response_generator.
@@ -2104,6 +2123,187 @@ class TestXFRSessionWithSQLite3(TestXfrinConnection):
         self.assertFalse(self.record_exist(Name('dns01.example.com'),
                                            RRType.A))
 
+class TestStatisticsXfrinConn(TestXfrinConnection):
+    '''Test class based on TestXfrinConnection and including paramters
+    and methods related to statistics tests'''
+    def setUp(self):
+        super().setUp()
+        # clear all statistics counters before each test
+        self.conn._counters.clear_all()
+        # fake datetime
+        self.__orig_datetime = isc.statistics.counters.datetime
+        self.__orig_start_timer = isc.statistics.counters._start_timer
+        time1 = datetime(2000, 1, 1, 0, 0, 0, 0)
+        time2 = datetime(2000, 1, 1, 0, 0, 0, 1)
+        class FakeDateTime:
+            @classmethod
+            def now(cls): return time2
+        isc.statistics.counters.datetime = FakeDateTime
+        isc.statistics.counters._start_timer = lambda : time1
+        delta = time2 - time1
+        self._const_sec = round(delta.days * 86400 + delta.seconds +
+                                delta.microseconds * 1E-6, 6)
+        # List of statistics counter names and expected initial values
+        self.__name_to_counter = (('axfrreqv4', 0),
+                                 ('axfrreqv6', 0),
+                                 ('ixfrreqv4', 0),
+                                 ('ixfrreqv6', 0),
+                                 ('last_axfr_duration', 0.0),
+                                 ('last_ixfr_duration', 0.0),
+                                 ('soaoutv4', 0),
+                                 ('soaoutv6', 0),
+                                 ('xfrfail', 0),
+                                 ('xfrsuccess', 0))
+        self.__zones = 'zones'
+
+    def tearDown(self):
+        super().tearDown()
+        isc.statistics.counters.datetime = self.__orig_datetime
+        isc.statistics.counters._start_timer = self.__orig_start_timer
+
+    @property
+    def _ipver(self):
+        return 'v4'
+
+    def _check_init_statistics(self):
+        '''checks exception being raised if not incremented statistics
+        counter gotten'''
+        for (name, exp) in self.__name_to_counter:
+            self.assertRaises(isc.cc.data.DataNotFoundError,
+                              self.conn._counters.get, self.__zones,
+                              TEST_ZONE_NAME_STR, name)
+
+    def _check_updated_statistics(self, overwrite):
+        '''checks getting expect values after updating the pairs of
+        statistics counter name and value on to the "overwrite"
+        dictionary'''
+        name2count = dict(self.__name_to_counter)
+        name2count.update(overwrite)
+        for (name, exp) in name2count.items():
+            act = self.conn._counters.get(self.__zones,
+                                          TEST_ZONE_NAME_STR,
+                                          name)
+            msg = '%s is expected %s but actually %s' % (name, exp, act)
+            self.assertEqual(exp, act, msg=msg)
+
+class TestStatisticsXfrinAXFRv4(TestStatisticsXfrinConn):
+    '''Xfrin AXFR tests for IPv4 to check statistics counters'''
+    def test_soaout(self):
+        '''tests that an soaoutv4 or soaoutv6 counter is incremented
+        when an soa query succeeds'''
+        self.conn.response_generator = self._create_soa_response_data
+        self._check_init_statistics()
+        self.assertEqual(self.conn._check_soa_serial(), XFRIN_OK)
+        self._check_updated_statistics({'soaout' + self._ipver: 1})
+
+    def test_axfrreq_xfrsuccess_last_axfr_duration(self):
+        '''tests that axfrreqv4 or axfrreqv6 and xfrsuccess counters
+        and last_axfr_duration timer are incremented when xfr succeeds'''
+        self.conn.response_generator = self._create_normal_response_data
+        self._check_init_statistics()
+        self.assertEqual(self.conn.do_xfrin(False), XFRIN_OK)
+        self._check_updated_statistics({'axfrreq' + self._ipver: 1,
+                                        'xfrsuccess': 1,
+                                        'last_axfr_duration': self._const_sec})
+
+    def test_axfrreq_xfrsuccess_last_axfr_duration2(self):
+        '''tests that axfrreqv4 or axfrreqv6 and xfrsuccess counters
+        and last_axfr_duration timer are incremented when raising
+        XfrinZoneUptodate. The exception is treated as success.'''
+        def exception_raiser():
+            raise XfrinZoneUptodate()
+        self.conn._handle_xfrin_responses = exception_raiser
+        self._check_init_statistics()
+        self.assertEqual(self.conn.do_xfrin(False), XFRIN_OK)
+        self._check_updated_statistics({'axfrreq' + self._ipver: 1,
+                                        'xfrsuccess': 1,
+                                        'last_axfr_duration':
+                                            self._const_sec})
+
+    def test_axfrreq_xfrfail(self):
+        '''tests that axfrreqv4 or axfrreqv6 and xfrfail counters are
+        incremented even if some failure exceptions are expected to be
+        raised inside do_xfrin(): XfrinZoneError, XfrinProtocolError,
+        XfrinException, and Exception'''
+        self._check_init_statistics()
+        count = 0
+        for ex in [XfrinZoneError, XfrinProtocolError, XfrinException,
+                   Exception]:
+            def exception_raiser():
+                raise ex()
+            self.conn._handle_xfrin_responses = exception_raiser
+            self.assertEqual(self.conn.do_xfrin(False), XFRIN_FAIL)
+            count += 1
+            self._check_updated_statistics({'axfrreq' + self._ipver: count,
+                                            'xfrfail': count})
+
+class TestStatisticsXfrinIXFRv4(TestStatisticsXfrinConn):
+    '''Xfrin IXFR tests for IPv4 to check statistics counters'''
+    def test_ixfrreq_xfrsuccess_last_ixfr_duration(self):
+        '''tests that ixfrreqv4 or ixfrreqv6 and xfrsuccess counters
+        and last_ixfr_duration timer are incremented when xfr succeeds'''
+        def create_ixfr_response():
+            self.conn.reply_data = self.conn.create_response_data(
+                questions=[Question(TEST_ZONE_NAME, TEST_RRCLASS,
+                                    RRType.IXFR)],
+                answers=[soa_rrset, begin_soa_rrset, soa_rrset, soa_rrset])
+        self.conn.response_generator = create_ixfr_response
+        self._check_init_statistics()
+        self.assertEqual(XFRIN_OK, self.conn.do_xfrin(False, RRType.IXFR))
+        self._check_updated_statistics({'ixfrreq' + self._ipver: 1,
+                                        'xfrsuccess': 1,
+                                        'last_ixfr_duration':
+                                            self._const_sec})
+
+    def test_ixfrreq_xfrsuccess_last_ixfr_duration2(self):
+        '''tests that ixfrreqv4 or ixfrreqv6 and xfrsuccess counters
+        and last_ixfr_duration timer are incremented when raising
+        XfrinZoneUptodate. The exception is treated as success.'''
+        def exception_raiser():
+            raise XfrinZoneUptodate()
+        self.conn._handle_xfrin_responses = exception_raiser
+        self._check_init_statistics()
+        self.assertEqual(self.conn.do_xfrin(False, RRType.IXFR), XFRIN_OK)
+        self._check_updated_statistics({'ixfrreq' + self._ipver: 1,
+                                        'xfrsuccess': 1,
+                                        'last_ixfr_duration':
+                                            self._const_sec})
+
+    def test_ixfrreq_xfrfail(self):
+        '''tests that ixfrreqv4 or ixfrreqv6 and xfrfail counters are
+        incremented even if some failure exceptions are expected to be
+        raised inside do_xfrin(): XfrinZoneError, XfrinProtocolError,
+        XfrinException, and Exception'''
+        self._check_init_statistics()
+        count = 0
+        for ex in [XfrinZoneError, XfrinProtocolError, XfrinException,
+                   Exception]:
+            def exception_raiser():
+                raise ex()
+            self.conn._handle_xfrin_responses = exception_raiser
+            self.assertEqual(self.conn.do_xfrin(False, RRType.IXFR), XFRIN_FAIL)
+            count += 1
+            self._check_updated_statistics({'ixfrreq' + self._ipver: count,
+                                            'xfrfail': count})
+
+class TestStatisticsXfrinAXFRv6(TestStatisticsXfrinAXFRv4):
+    '''Same tests as TestStatisticsXfrinAXFRv4 for IPv6'''
+    @property
+    def _master_addrinfo(self):
+        return TEST_MASTER_IPV6_ADDRINFO
+    @property
+    def _ipver(self):
+        return 'v6'
+
+class TestStatisticsIXFRv6(TestStatisticsXfrinIXFRv4):
+    '''Same tests as TestStatisticsXfrinIXFRv4 for IPv6'''
+    @property
+    def _master_addrinfo(self):
+        return TEST_MASTER_IPV6_ADDRINFO
+    @property
+    def _ipver(self):
+        return 'v6'
+
 class TestXfrinRecorder(unittest.TestCase):
     def setUp(self):
         self.recorder = XfrinRecorder()
@@ -2512,6 +2712,14 @@ class TestXfrin(unittest.TestCase):
         self.assertEqual(self.xfr.config_handler({'transfers_in': 3})['result'][0], 0)
         self.assertEqual(self.xfr._max_transfers_in, 3)
 
+    def test_command_handler_getstats(self):
+        module_spec = isc.config.module_spec_from_file(
+            xfrin.SPECFILE_LOCATION)
+        ans = isc.config.parse_answer(
+            self.xfr.command_handler("getstats", None))
+        self.assertEqual(0, ans[0])
+        self.assertTrue(module_spec.validate_statistics(False, ans[1]))
+
     def _check_zones_config(self, config_given):
         if 'transfers_in' in config_given:
             self.assertEqual(config_given['transfers_in'],

src/bin/xfrin/xfrin.py.in

@@ -1,6 +1,6 @@
 #!@PYTHON@
 
-# Copyright (C) 2009-2011  Internet Systems Consortium.
+# Copyright (C) 2009-2013  Internet Systems Consortium.
 #
 # Permission to use, copy, modify, and distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
@@ -28,6 +28,7 @@ import time
 from functools import reduce
 from optparse import OptionParser, OptionValueError
 from isc.config.ccsession import *
+from isc.statistics import Counters
 from isc.notify import notify_out
 import isc.util.process
 from isc.datasrc import DataSourceClient, ZoneFinder
@@ -612,6 +613,7 @@ class XfrinConnection(asyncore.dispatcher):
         # keep a record of this specific transfer to log on success
         # (time, rr/s, etc)
         self._transfer_stats = XfrinTransferStats()
+        self._counters = Counters(SPECFILE_LOCATION)
 
     def init_socket(self):
         '''Initialize the underlyig socket.
@@ -891,6 +893,19 @@ class XfrinConnection(asyncore.dispatcher):
         # All okay, return it
         return soa
 
+    def _get_ipver_str(self):
+        """Returns a 'v4' or 'v6' string representing a IP version
+        depending on the socket family. This is for an internal use
+        only (except for tests). This is supported only for IP sockets.
+        It raises a ValueError exception on other address families.
+
+        """
+        if self.socket.family == socket.AF_INET:
+            return 'v4'
+        elif self.socket.family == socket.AF_INET6:
+            return 'v6'
+        raise ValueError("Invalid address family. "
+                         "This is supported only for IP sockets")
 
     def _check_soa_serial(self):
         '''Send SOA query and compare the local and remote serials.
@@ -902,6 +917,9 @@ class XfrinConnection(asyncore.dispatcher):
         '''
 
         self._send_query(RRType.SOA)
+        # count soaoutv4 or soaoutv6 requests
+        self._counters.inc('zones', self._zone_name.to_text(),
+                           'soaout' + self._get_ipver_str())
         data_len = self._get_request_response(2)
         msg_len = socket.htons(struct.unpack('H', data_len)[0])
         soa_response = self._get_request_response(msg_len)
@@ -931,9 +949,7 @@ class XfrinConnection(asyncore.dispatcher):
         try:
             ret = XFRIN_OK
             self._request_type = request_type
-            # Right now RRType.[IA]XFR().to_text() is 'TYPExxx', so we need
-            # to hardcode here.
-            req_str = 'IXFR' if request_type == RRType.IXFR else 'AXFR'
+            req_str = request_type.to_text()
             if check_soa:
                 self._check_soa_serial()
                 self.close()
@@ -941,7 +957,16 @@ class XfrinConnection(asyncore.dispatcher):
                 if not self.connect_to_master():
                     raise XfrinException('Unable to reconnect to master')
 
+            # start statistics timer
+            # Note: If the timer for the zone is already started but
+            # not yet stopped due to some error, the last start time
+            # is overwritten at this point.
+            self._counters.start_timer('zones', self._zone_name.to_text(),
+                                       'last_' + req_str.lower() + '_duration')
             logger.info(XFRIN_XFR_TRANSFER_STARTED, req_str, self.zone_str())
+            # An AXFR or IXFR is being requested.
+            self._counters.inc('zones', self._zone_name.to_text(),
+                               req_str.lower() + 'req' + self._get_ipver_str())
             self._send_query(self._request_type)
             self.__state = XfrinInitialSOA()
             self._handle_xfrin_responses()
@@ -968,7 +993,6 @@ class XfrinConnection(asyncore.dispatcher):
                             "%.3f" % self._transfer_stats.get_running_time(),
                             "%.f" % self._transfer_stats.get_bytes_per_second()
                            )
-
         except XfrinZoneUptodate:
             # Eventually we'll probably have to treat this case as a trigger
             # of trying another primary server, etc, but for now we treat it
@@ -1004,11 +1028,21 @@ class XfrinConnection(asyncore.dispatcher):
                          self.zone_str(), str(e))
             ret = XFRIN_FAIL
         finally:
+            # A xfrsuccess or xfrfail counter is incremented depending on
+            # the result.
+            result = {XFRIN_OK: 'xfrsuccess', XFRIN_FAIL: 'xfrfail'}[ret]
+            self._counters.inc('zones', self._zone_name.to_text(), result)
+            # The started statistics timer is finally stopped only in
+            # a successful case.
+            if ret == XFRIN_OK:
+                self._counters.stop_timer('zones',
+                                          self._zone_name.to_text(),
+                                          'last_' + req_str.lower() +
+                                          '_duration')
             # Make sure any remaining transaction in the diff is closed
             # (if not yet - possible in case of xfr-level exception) as soon
             # as possible
             self._diff = None
-
         return ret
 
     def _check_response_header(self, msg):
@@ -1339,6 +1373,7 @@ class Xfrin:
         self._cc_setup()
         self.recorder = XfrinRecorder()
         self._shutdown_event = threading.Event()
+        self._counters = Counters(SPECFILE_LOCATION)
 
     def _cc_setup(self):
         '''This method is used only as part of initialization, but is
@@ -1484,6 +1519,7 @@ class Xfrin:
             th.join()
 
     def command_handler(self, command, args):
+        logger.debug(DBG_XFRIN_TRACE, XFRIN_RECEIVED_COMMAND, command)
         answer = create_answer(0)
         try:
             if command == 'shutdown':
@@ -1552,6 +1588,14 @@ class Xfrin:
                                        (False if command == 'retransfer' else True))
                 answer = create_answer(ret[0], ret[1])
 
+            # return statistics data to the stats daemon
+            elif command == "getstats":
+                # The log level is here set to debug in order to avoid
+                # that a log becomes too verbose. Because the
+                # b10-stats daemon is periodically asking to the
+                # b10-xfrin daemon.
+                answer = create_answer(0, self._counters.get_statistics())
+
             else:
                 answer = create_answer(1, 'unknown command: ' + command)
         except XfrinException as err:

src/bin/xfrin/xfrin.spec

@@ -94,6 +94,119 @@
           }
         ]
       }
+    ],
+    "statistics": [
+      {
+        "item_name": "zones",
+        "item_type": "named_set",
+        "item_optional": false,
+        "item_default": {
+          "_SERVER_" : {
+	    "soaoutv4": 0,
+	    "soaoutv6": 0,
+	    "axfrreqv4": 0,
+	    "axfrreqv6": 0,
+	    "ixfrreqv4": 0,
+	    "ixfrreqv6": 0,
+	    "xfrsuccess": 0,
+	    "xfrfail": 0,
+	    "last_ixfr_duration": 0.0,
+	    "last_axfr_duration": 0.0
+          }
+        },
+        "item_title": "Zone names",
+        "item_description": "A directory name of per-zone statistics",
+        "named_set_item_spec": {
+          "item_name": "zonename",
+          "item_type": "map",
+          "item_optional": false,
+          "item_default": {},
+          "item_title": "Zone name",
+          "item_description": "An actual zone name or special zone name _SERVER_ representing the entire server. Zone classes (e.g. IN, CH, and HS) are mixed and counted so far. But these will be distinguished in future release.",
+          "map_item_spec": [
+            {
+              "item_name": "soaoutv4",
+              "item_type": "integer",
+              "item_optional": false,
+              "item_default": 0,
+              "item_title": "SOAOutv4",
+              "item_description": "Number of IPv4 SOA queries sent from Xfrin"
+            },
+            {
+              "item_name": "soaoutv6",
+              "item_type": "integer",
+              "item_optional": false,
+              "item_default": 0,
+              "item_title": "SOAOutv6",
+              "item_description": "Number of IPv6 SOA queries sent from Xfrin"
+            },
+            {
+              "item_name": "axfrreqv4",
+              "item_type": "integer",
+              "item_optional": false,
+              "item_default": 0,
+              "item_title": "AXFRReqv4",
+              "item_description": "Number of IPv4 AXFR requests sent from Xfrin"
+            },
+            {
+              "item_name": "axfrreqv6",
+              "item_type": "integer",
+              "item_optional": false,
+              "item_default": 0,
+              "item_title": "AXFRReqv6",
+              "item_description": "Number of IPv6 AXFR requests sent from Xfrin"
+            },
+            {
+              "item_name": "ixfrreqv4",
+              "item_type": "integer",
+              "item_optional": false,
+              "item_default": 0,
+              "item_title": "IXFRReqv4",
+              "item_description": "Number of IPv4 IXFR requests sent from Xfrin"
+            },
+            {
+              "item_name": "ixfrreqv6",
+              "item_type": "integer",
+              "item_optional": false,
+              "item_default": 0,
+              "item_title": "IXFRReqv6",
+              "item_description": "Number of IPv6 IXFR requests sent from Xfrin"
+            },
+            {
+              "item_name": "xfrsuccess",
+              "item_type": "integer",
+              "item_optional": false,
+              "item_default": 0,
+              "item_title": "XfrSuccess",
+              "item_description": "Number of zone transfer requests succeeded. These include the case where the zone turns out to be the latest as a result of an initial SOA query (and there is actually no AXFR or IXFR transaction)."
+            },
+            {
+              "item_name": "xfrfail",
+              "item_type": "integer",
+              "item_optional": false,
+              "item_default": 0,
+              "item_title": "XfrFail",
+              "item_description": "Number of zone transfer requests failed"
+            },
+            {
+              "item_name": "last_axfr_duration",
+              "item_type": "real",
+              "item_optional": false,
+              "item_default": 0.0,
+              "item_title": "Last AXFR duration",
+              "item_description": "Duration in seconds of the last successful AXFR.  0.0 means no successful AXFR done or means a successful AXFR done in less than a microsecond.  If an AXFR is aborted due to some failure, this duration won't be updated."
+            },
+            {
+              "item_name": "last_ixfr_duration",
+              "item_type": "real",
+              "item_optional": false,
+              "item_default": 0.0,
+              "item_title": "Last IXFR duration",
+              "item_description": "Duration in seconds of the last successful IXFR.  0.0 means no successful IXFR done or means a successful IXFR done in less than a microsecond.  If an IXFR is aborted due to some failure, this duration won't be updated."
+            }
+          ]
+        }
+      }
     ]
   }
 }

src/bin/xfrin/xfrin_messages.mes

@@ -135,6 +135,9 @@ from does not match the master address in the Xfrin configuration. The notify
 is ignored. This may indicate that the configuration for the master is wrong,
 that a wrong machine is sending notifies, or that fake notifies are being sent.
 
+% XFRIN_RECEIVED_COMMAND received command: %1
+The xfrin daemon received a command on the command channel.
+
 % XFRIN_RETRANSFER_UNKNOWN_ZONE got notification to retransfer unknown zone %1
 There was an internal command to retransfer the given zone, but the
 zone is not known to the system. This may indicate that the configuration

src/lib/asiodns/asiodns_messages.mes

@@ -53,6 +53,78 @@ The asynchronous I/O code encountered an error when trying to send data to
 the specified address on the given protocol.  The number of the system
 error that caused the problem is given in the message.
 
+% ASIODNS_SYNC_UDP_CLOSE_FAIL failed to close a DNS/UDP socket: %1
+This is the same to ASIODNS_UDP_CLOSE_FAIL but happens on the
+"synchronous UDP server", mainly used for the authoritative DNS server
+daemon.
+
+% ASIODNS_TCP_ACCEPT_FAIL failed to accept TCP DNS connection: %1
+Accepting a TCP connection from a DNS client failed due to an error
+that could happen but should be rare.  The reason for the error is
+included in the log message.  The server still keeps accepting new
+connections, so unless it happens often it's probably okay to ignore
+this error.  If the shown error indicates something like "too many
+open files", it's probably because the run time environment is too
+restrictive on this limitation, so consider adjusing the limit using
+a tool such as ulimit.  If you see other types of errors too often,
+there may be something overlooked; please file a bug report in that case.
+
+% ASIODNS_TCP_CLEANUP_CLOSE_FAIL failed to close a DNS/TCP socket on port cleanup: %1
+A TCP DNS server tried to close a TCP socket (one created on accepting
+a new connection or is already unused) as a step of cleaning up the
+corresponding listening port, but it failed to do that.  This is
+generally an unexpected event and so is logged as an error.
+See also the description of ASIODNS_TCP_CLOSE_ACCEPTOR_FAIL.
+
+% ASIODNS_TCP_CLOSE_ACCEPTOR_FAIL failed to close listening TCP socket: %1
+A TCP DNS server tried to close a listening TCP socket (for accepting
+new connections) as a step of cleaning up the corresponding listening
+port (e.g., on server shutdown or updating port configuration), but it
+failed to do that.  This is generally an unexpected event and so is
+logged as an error.  See ASIODNS_TCP_CLOSE_FAIL on the implication of
+related system resources.
+
+% ASIODNS_TCP_CLOSE_FAIL failed to close DNS/TCP socket with a client: %1
+A TCP DNS server tried to close a TCP socket used to communicate with
+a client, but it failed to do that.  While closing a socket should
+normally be an error-free operation, there have been known cases where
+this happened with a "connection reset by peer" error.  This might be
+because of some odd client behavior, such as sending a TCP RST after
+establishing the connection and before the server closes the socket,
+but how exactly this could happen seems to be system dependent (i.e,
+it's not part of the standard socket API), so it's difficult to
+provide a general explanation.  In any case, it is believed that an
+error on closing a socket doesn't mean leaking system resources (the
+kernel should clean up any internal resource related to the socket,
+just reporting an error detected in the close call), but, again, it
+seems to be system dependent.  This message is logged at a debug level
+as it's known to happen and could be triggered by a remote node and it
+would be better to not be too verbose, but you might want to increase
+the log level and make sure there's no resource leak or other system
+level troubles when it's logged.
+
+% ASIODNS_TCP_GETREMOTE_FAIL failed to get remote address of a DNS TCP connection: %1
+A TCP DNS server tried to get the address and port of a remote client
+on a connected socket but failed.  It's expected to be rare but can
+still happen.  See also ASIODNS_TCP_READLEN_FAIL.
+
+% ASIODNS_TCP_READDATA_FAIL failed to get DNS data on a TCP socket: %1
+A TCP DNS server tried to read a DNS message (that follows a 2-byte
+length field) but failed.  It's expected to be rare but can still happen.
+See also ASIODNS_TCP_READLEN_FAIL.
+
+% ASIODNS_TCP_READLEN_FAIL failed to get DNS data length on a TCP socket: %1
+A TCP DNS server tried to get the length field of a DNS message (the first
+2 bytes of a new chunk of data) but failed.  This is generally expected to
+be rare but can still happen, e.g, due to an unexpected reset of the
+connection.  A specific reason for the failure is included in the log
+message.
+
+% ASIODNS_TCP_WRITE_FAIL failed to send DNS message over a TCP socket: %1
+A TCP DNS server tried to send a DNS message to a remote client but
+failed.  It's expected to be rare but can still happen.  See also
+ASIODNS_TCP_READLEN_FAIL.
+
 % ASIODNS_UDP_ASYNC_SEND_FAIL Error sending UDP packet to %1: %2
 The low-level ASIO library reported an error when trying to send a UDP
 packet in asynchronous UDP mode. This can be any error reported by
@@ -64,6 +136,23 @@ If you see a single occurrence of this message, it probably does not
 indicate any significant problem, but if it is logged often, it is probably
 a good idea to inspect your network traffic.
 
+% ASIODNS_UDP_CLOSE_FAIL failed to close a DNS/UDP socket: %1
+A UDP DNS server tried to close its UDP socket, but failed to do that.
+This is generally an unexpected event and so is logged as an error.
+
+% ASIODNS_UDP_RECEIVE_FAIL failed to receive UDP DNS packet: %1
+Receiving a UDP packet from a DNS client failed due to an error that
+could happen but should be very rare.  The server still keeps
+receiving UDP packets on this socket.  The reason for the error is
+included in the log message.  This log message is basically not
+expected to appear at all in practice; if it does, there may be some
+system level failure and other system logs may have to be checked.
+
+% ASIODNS_UDP_SYNC_RECEIVE_FAIL failed to receive UDP DNS packet: %1
+This is the same to ASIODNS_UDP_RECEIVE_FAIL but happens on the
+"synchronous UDP server", mainly used for the authoritative DNS server
+daemon.
+
 % ASIODNS_UDP_SYNC_SEND_FAIL Error sending UDP packet to %1: %2
 The low-level ASIO library reported an error when trying to send a UDP
 packet in synchronous UDP mode. See ASIODNS_UDP_ASYNC_SEND_FAIL for

src/lib/asiodns/dns_service.cc

@@ -58,9 +58,15 @@ public:
     template<class Ptr, class Server> void addServerFromFD(int fd, int af) {
         Ptr server(new Server(io_service_.get_io_service(), fd, af, checkin_,
                               lookup_, answer_));
-        server->setTCPRecvTimeout(tcp_recv_timeout_);
-        (*server)();
-        servers_.push_back(server);
+        startServer(server);
+    }
+
+    // SyncUDPServer has different constructor signature so it cannot be
+    // templated.
+    void addSyncUDPServerFromFD(int fd, int af) {
+        SyncUDPServerPtr server(new SyncUDPServer(io_service_.get_io_service(),
+                                                  fd, af, lookup_));
+        startServer(server);
     }
 
     void setTCPRecvTimeout(size_t timeout) {
@@ -72,6 +78,13 @@ public:
             (*it)->setTCPRecvTimeout(timeout);
         }
     }
+
+private:
+    void startServer(DNSServerPtr server) {
+        server->setTCPRecvTimeout(tcp_recv_timeout_);
+        (*server)();
+        servers_.push_back(server);
+    }
 };
 
 DNSService::DNSService(IOService& io_service, SimpleCallback* checkin,
@@ -95,8 +108,7 @@ void DNSService::addServerUDPFromFD(int fd, int af, ServerFlag options) {
                   << options);
     }
     if ((options & SERVER_SYNC_OK) != 0) {
-        impl_->addServerFromFD<DNSServiceImpl::SyncUDPServerPtr,
-            SyncUDPServer>(fd, af);
+        impl_->addSyncUDPServerFromFD(fd, af);
     } else {
         impl_->addServerFromFD<DNSServiceImpl::UDPServerPtr, UDPServer>(
             fd, af);

src/lib/asiodns/sync_udp_server.cc

@@ -39,18 +39,21 @@ namespace isc {
 namespace asiodns {
 
 SyncUDPServer::SyncUDPServer(asio::io_service& io_service, const int fd,
-                             const int af, asiolink::SimpleCallback* checkin,
-                             DNSLookup* lookup, DNSAnswer* answer) :
+                             const int af, DNSLookup* lookup) :
     output_buffer_(new isc::util::OutputBuffer(0)),
     query_(new isc::dns::Message(isc::dns::Message::PARSE)),
-    answer_(new isc::dns::Message(isc::dns::Message::RENDER)),
-    checkin_callback_(checkin), lookup_callback_(lookup),
-    answer_callback_(answer), stopped_(false)
+    udp_endpoint_(sender_), lookup_callback_(lookup),
+    resume_called_(false), done_(false), stopped_(false),
+    recv_callback_(boost::bind(&SyncUDPServer::handleRead, this, _1, _2))
 {
     if (af != AF_INET && af != AF_INET6) {
         isc_throw(InvalidParameter, "Address family must be either AF_INET "
                   "or AF_INET6, not " << af);
     }
+    if (!lookup) {
+        isc_throw(InvalidParameter, "null lookup callback given to "
+                  "SyncUDPServer");
+    }
     LOG_DEBUG(logger, DBGLVL_TRACE_BASIC, ASIODNS_FD_ADD_UDP).arg(fd);
     try {
         socket_.reset(new asio::ip::udp::socket(io_service));
@@ -61,59 +64,36 @@ SyncUDPServer::SyncUDPServer(asio::io_service& io_service, const int fd,
         // convert it
         isc_throw(IOError, exception.what());
     }
+    udp_socket_.reset(new UDPSocket<DummyIOCallback>(*socket_));
 }
 
 void
 SyncUDPServer::scheduleRead() {
-    socket_->async_receive_from(asio::buffer(data_, MAX_LENGTH), sender_,
-                                boost::bind(&SyncUDPServer::handleRead, this,
-                                            _1, _2));
+    socket_->async_receive_from(asio::mutable_buffers_1(data_, MAX_LENGTH),
+                                sender_, recv_callback_);
 }
 
 void
 SyncUDPServer::handleRead(const asio::error_code& ec, const size_t length) {
-    // Abort on fatal errors
     if (ec) {
         using namespace asio::error;
-        if (ec.value() != would_block && ec.value() != try_again &&
-            ec.value() != interrupted) {
+        const asio::error_code::value_type err_val = ec.value();
+
+        // See TCPServer::operator() for details on error handling.
+        if (err_val == operation_aborted || err_val == bad_descriptor) {
             return;
         }
+        if (err_val != would_block && err_val != try_again &&
+            err_val != interrupted) {
+            LOG_ERROR(logger, ASIODNS_UDP_SYNC_RECEIVE_FAIL).arg(ec.message());
+        }
     }
-    // Some kind of interrupt, spurious wakeup, or like that. Just try reading
-    // again.
     if (ec || length == 0) {
         scheduleRead();
         return;
     }
     // OK, we have a real packet of data. Let's dig into it!
 
-    // XXX: This is taken (and ported) from UDPSocket class. What the hell does
-    // it really mean?
-
-    // The UDP socket class has been extended with asynchronous functions
-    // and takes as a template parameter a completion callback class.  As
-    // UDPServer does not use these extended functions (only those defined
-    // in the IOSocket base class) - but needs a UDPSocket to get hold of
-    // the underlying Boost UDP socket - DummyIOCallback is used.  This
-    // provides the appropriate operator() but is otherwise functionless.
-    UDPSocket<DummyIOCallback> socket(*socket_);
-    UDPEndpoint endpoint(sender_);
-    IOMessage message(data_, length, socket, endpoint);
-    if (checkin_callback_ != NULL) {
-        (*checkin_callback_)(message);
-        if (stopped_) {
-            return;
-        }
-    }
-
-    // If we don't have a DNS Lookup provider, there's no point in
-    // continuing; we exit the coroutine permanently.
-    if (lookup_callback_ == NULL) {
-        scheduleRead();
-        return;
-    }
-
     // Make sure the buffers are fresh.  Note that we don't touch query_
     // because it's supposed to be cleared in lookup_callback_.  We should
     // eventually even remove this member variable (and remove it from
@@ -121,13 +101,13 @@ SyncUDPServer::handleRead(const asio::error_code& ec, const size_t length) {
     // implementation should be careful that it's the responsibility of
     // the callback implementation.  See also #2239).
     output_buffer_->clear();
-    answer_->clear(isc::dns::Message::RENDER);
 
     // Mark that we don't have an answer yet.
     done_ = false;
     resume_called_ = false;
 
     // Call the actual lookup
+    const IOMessage message(data_, length, *udp_socket_, udp_endpoint_);
     (*lookup_callback_)(message, query_, answer_, output_buffer_, this);
 
     if (!resume_called_) {
@@ -135,27 +115,14 @@ SyncUDPServer::handleRead(const asio::error_code& ec, const size_t length) {
                   "No resume called from the lookup callback");
     }
 
-    if (stopped_) {
-        return;
-    }
-
     if (done_) {
         // Good, there's an answer.
-        // Call the answer callback to render it.
-        (*answer_callback_)(message, query_, answer_, output_buffer_);
-
-        if (stopped_) {
-            return;
-        }
-
-        asio::error_code ec;
-        socket_->send_to(asio::buffer(output_buffer_->getData(),
-                                      output_buffer_->getLength()),
-                         sender_, 0, ec);
-        if (ec) {
+        socket_->send_to(asio::const_buffers_1(output_buffer_->getData(),
+                                               output_buffer_->getLength()),
+                         sender_, 0, ec_);
+        if (ec_) {
             LOG_ERROR(logger, ASIODNS_UDP_SYNC_SEND_FAIL).
-                      arg(sender_.address().to_string()).
-                      arg(ec.message());
+                      arg(sender_.address().to_string()).arg(ec_.message());
         }
     }
 
@@ -181,13 +148,13 @@ SyncUDPServer::stop() {
     /// for it won't be scheduled by io service not matter it is
     /// submit to io service before or after close call. And we will
     /// get bad_descriptor error.
-    socket_->close();
+    socket_->close(ec_);
     stopped_ = true;
+    if (ec_) {
+        LOG_ERROR(logger, ASIODNS_SYNC_UDP_CLOSE_FAIL).arg(ec_.message());
+    }
 }
 
-/// Post this coroutine on the ASIO service queue so that it will
-/// resume processing where it left off.  The 'done' parameter indicates
-/// whether there is an answer to return to the client.
 void
 SyncUDPServer::resume(const bool done) {
     resume_called_ = true;

src/lib/asiodns/sync_udp_server.h

@@ -25,10 +25,14 @@
 
 #include <dns/message.h>
 #include <asiolink/simple_callback.h>
+#include <asiolink/dummy_io_cb.h>
+#include <asiolink/udp_socket.h>
 #include <util/buffer.h>
 #include <exceptions/exceptions.h>
 
+#include <boost/function.hpp>
 #include <boost/noncopyable.hpp>
+#include <boost/scoped_ptr.hpp>
 
 #include <stdint.h>
 
@@ -39,29 +43,39 @@ namespace asiodns {
 ///
 /// That means, the lookup handler must provide the answer right away.
 /// This allows for implementation with less overhead, compared with
-/// the UDPClass.
+/// the \c UDPServer class.
 class SyncUDPServer : public DNSServer, public boost::noncopyable {
 public:
     /// \brief Constructor
+    ///
+    /// Due to the nature of this server, it's meaningless if the lookup
+    /// callback is NULL.  So the constructor explicitly rejects that case
+    /// with an exception.  Likewise, it doesn't take "checkin" or "answer"
+    /// callbacks.  In fact, calling "checkin" from receive callback does not
+    /// make sense for any of the DNSServer variants (see Trac #2935);
+    /// "answer" callback is simply unnecessary for this class because a
+    /// complete answer is built in the lookup callback (it's the user's
+    /// responsibility to guarantee that condition).
+    ///
     /// \param io_service the asio::io_service to work with
     /// \param fd the file descriptor of opened UDP socket
     /// \param af address family, either AF_INET or AF_INET6
-    /// \param checkin the callbackprovider for non-DNS events
-    /// \param lookup the callbackprovider for DNS lookup events
-    /// \param answer the callbackprovider for DNS answer events
+    /// \param lookup the callbackprovider for DNS lookup events (must not be
+    ///        NULL)
+    ///
     /// \throw isc::InvalidParameter if af is neither AF_INET nor AF_INET6
+    /// \throw isc::InvalidParameter lookup is NULL
     /// \throw isc::asiolink::IOError when a low-level error happens, like the
     ///     fd is not a valid descriptor.
     SyncUDPServer(asio::io_service& io_service, const int fd, const int af,
-                  isc::asiolink::SimpleCallback* checkin = NULL,
-                  DNSLookup* lookup = NULL, DNSAnswer* answer = NULL);
+                  DNSLookup* lookup);
 
     /// \brief Start the SyncUDPServer.
     ///
     /// This is the function operator to keep interface with other server
     /// classes. They need that because they're coroutines.
     virtual void operator()(asio::error_code ec = asio::error_code(),
-                    size_t length = 0);
+                            size_t length = 0);
 
     /// \brief Calls the lookup callback
     virtual void asyncLookup() {
@@ -114,22 +128,52 @@ private:
     // If it was OK to have just a buffer, not the wrapper class,
     // we could reuse the data_
     isc::util::OutputBufferPtr output_buffer_;
-    // Objects to hold the query message and the answer
+    // Objects to hold the query message and the answer.  The latter isn't
+    // used and only defined as a placeholder as the callback signature
+    // requires it.
     isc::dns::MessagePtr query_, answer_;
     // The socket used for the communication
     std::auto_ptr<asio::ip::udp::socket> socket_;
+    // Wrapper of socket_ in the form of asiolink::IOSocket.
+    // "DummyIOCallback" is not necessary for this class, but using the
+    // template is the easiest way to create a UDP instance of IOSocket.
+    boost::scoped_ptr<asiolink::UDPSocket<asiolink::DummyIOCallback> >
+    udp_socket_;
     // Place the socket puts the sender of a packet when it is received
     asio::ip::udp::endpoint sender_;
-    // Callbacks
-    const asiolink::SimpleCallback* checkin_callback_;
+    // Wrapper of sender_ in the form of asiolink::IOEndpoint.  It's set to
+    // refer to sender_ on initialization, and keeps the reference throughout
+    // this server class.
+    asiolink::UDPEndpoint udp_endpoint_;
+    // Callback
     const DNSLookup* lookup_callback_;
-    const DNSAnswer* answer_callback_;
     // Answers from the lookup callback (not sent directly, but signalled
     // through resume()
     bool resume_called_, done_;
     // This turns true when the server stops. Allows for not sending the
     // answer after we closed the socket.
     bool stopped_;
+    // Placeholder for error code object.  It will be passed to ASIO library
+    // to have it set in case of error.
+    asio::error_code ec_;
+    // The callback functor for internal asynchronous read event.  This is
+    // stateless (and it will be copied in the ASIO library anyway), so
+    // can be const.
+    // SunStudio doesn't like a boost::function object to be passed, so
+    // we use the wrapper class as a workaround.
+    class CallbackWrapper {
+    public:
+        CallbackWrapper(boost::function<void(const asio::error_code&, size_t)>
+                        callback) :
+            callback_(callback)
+        {}
+        void operator()(const asio::error_code& error, size_t len) {
+            callback_(error, len);
+        }
+    private:
+        boost::function<void(const asio::error_code&, size_t)> callback_;
+    };
+    const CallbackWrapper recv_callback_;
 
     // Auxiliary functions
 
@@ -144,3 +188,7 @@ private:
 } // namespace asiodns
 } // namespace isc
 #endif // SYNC_UDP_SERVER_H
+
+// Local Variables:
+// mode: c++
+// End:

src/lib/asiodns/tcp_server.cc

@@ -14,13 +14,6 @@
 
 #include <config.h>
 
-#include <unistd.h>             // for some IPC/network system calls
-#include <netinet/in.h>
-#include <sys/socket.h>
-#include <errno.h>
-
-#include <boost/shared_array.hpp>
-
 #include <log/dummylog.h>
 
 #include <util/buffer.h>
@@ -32,6 +25,14 @@
 #include <asiodns/tcp_server.h>
 #include <asiodns/logger.h>
 
+#include <boost/shared_array.hpp>
+
+#include <cassert>
+#include <unistd.h>             // for some IPC/network system calls
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <errno.h>
+
 using namespace asio;
 using asio::ip::udp;
 using asio::ip::tcp;
@@ -100,41 +101,58 @@ TCPServer::operator()(asio::error_code ec, size_t length) {
 
     CORO_REENTER (this) {
         do {
-            /// Create a socket to listen for connections
+            /// Create a socket to listen for connections (no-throw operation)
             socket_.reset(new tcp::socket(acceptor_->get_io_service()));
 
             /// Wait for new connections. In the event of non-fatal error,
             /// try again
             do {
                 CORO_YIELD acceptor_->async_accept(*socket_, *this);
-
-                // Abort on fatal errors
-                // TODO: Log error?
                 if (ec) {
                     using namespace asio::error;
-                    if (ec.value() != would_block && ec.value() != try_again &&
-                        ec.value() != connection_aborted &&
-                        ec.value() != interrupted) {
+                    const error_code::value_type err_val = ec.value();
+                    // The following two cases can happen when this server is
+                    // stopped: operation_aborted in case it's stopped after
+                    // starting accept().  bad_descriptor in case it's stopped
+                    // even before starting.  In these cases we should simply
+                    // stop handling events.
+                    if (err_val == operation_aborted ||
+                        err_val == bad_descriptor) {
                         return;
                     }
+                    // Other errors should generally be temporary and we should
+                    // keep waiting for new connections.  We log errors that
+                    // should really be rare and would only be caused by an
+                    // internal erroneous condition (not an odd remote
+                    // behavior).
+                    if (err_val != would_block && err_val != try_again &&
+                        err_val != connection_aborted &&
+                        err_val != interrupted) {
+                        LOG_ERROR(logger, ASIODNS_TCP_ACCEPT_FAIL).
+                            arg(ec.message());
+                    }
                 }
             } while (ec);
 
             /// Fork the coroutine by creating a copy of this one and
             /// scheduling it on the ASIO service queue.  The parent
-            /// will continue listening for DNS connections while the
+            /// will continue listening for DNS connections while the child
             /// handles the one that has just arrived.
             CORO_FORK io_.post(TCPServer(*this));
         } while (is_parent());
 
+        // From this point, we'll simply return on error, which will
+        // immediately trigger destroying this object, cleaning up all
+        // resources including any open sockets.
+
         /// Instantiate the data buffer that will be used by the
         /// asynchronous read call.
         data_.reset(new char[MAX_LENGTH]);
 
         /// Start a timer to drop the connection if it is idle
         if (*tcp_recv_timeout_ > 0) {
-            timeout_.reset(new asio::deadline_timer(io_));
-            timeout_->expires_from_now(
+            timeout_.reset(new asio::deadline_timer(io_)); // shouldn't throw
+            timeout_->expires_from_now( // consider any exception fatal.
                 boost::posix_time::milliseconds(*tcp_recv_timeout_));
             timeout_->async_wait(boost::bind(&do_timeout, boost::ref(*socket_),
                                  asio::placeholders::error));
@@ -144,29 +162,22 @@ TCPServer::operator()(asio::error_code ec, size_t length) {
         CORO_YIELD async_read(*socket_, asio::buffer(data_.get(),
                               TCP_MESSAGE_LENGTHSIZE), *this);
         if (ec) {
-            socket_->close();
-            CORO_YIELD return;
+            LOG_DEBUG(logger, DBGLVL_TRACE_BASIC, ASIODNS_TCP_READLEN_FAIL).
+                arg(ec.message());
+            return;
         }
 
         /// Now read the message itself. (This is done in a different scope
         /// to allow inline variable declarations.)
         CORO_YIELD {
             InputBuffer dnsbuffer(data_.get(), length);
-            uint16_t msglen = dnsbuffer.readUint16();
+            const uint16_t msglen = dnsbuffer.readUint16();
             async_read(*socket_, asio::buffer(data_.get(), msglen), *this);
         }
-
         if (ec) {
-            socket_->close();
-            CORO_YIELD return;
-        }
-
-        // Due to possible timeouts and other bad behaviour, after the
-        // timely reads are done, there is a chance the socket has
-        // been closed already. So before we move on to the actual
-        // processing, check that, and stop if so.
-        if (!socket_->is_open()) {
-            CORO_YIELD return;
+            LOG_DEBUG(logger, DBGLVL_TRACE_BASIC, ASIODNS_TCP_READDATA_FAIL).
+                arg(ec.message());
+            return;
         }
 
         // Create an \c IOMessage object to store the query.
@@ -174,7 +185,12 @@ TCPServer::operator()(asio::error_code ec, size_t length) {
         // (XXX: It would be good to write a factory function
         // that would quickly generate an IOMessage object without
         // all these calls to "new".)
-        peer_.reset(new TCPEndpoint(socket_->remote_endpoint()));
+        peer_.reset(new TCPEndpoint(socket_->remote_endpoint(ec)));
+        if (ec) {
+            LOG_DEBUG(logger, DBGLVL_TRACE_BASIC, ASIODNS_TCP_GETREMOTE_FAIL).
+                arg(ec.message());
+            return;
+        }
 
         // The TCP socket class has been extended with asynchronous functions
         // and takes as a template parameter a completion callback class.  As
@@ -183,7 +199,8 @@ TCPServer::operator()(asio::error_code ec, size_t length) {
         // the underlying Boost TCP socket - DummyIOCallback is used.  This
         // provides the appropriate operator() but is otherwise functionless.
         iosock_.reset(new TCPSocket<DummyIOCallback>(*socket_));
-        io_message_.reset(new IOMessage(data_.get(), length, *iosock_, *peer_));
+        io_message_.reset(new IOMessage(data_.get(), length, *iosock_,
+                                        *peer_));
 
         // Perform any necessary operations prior to processing the incoming
         // packet (e.g., checking for queued configuration messages).
@@ -198,8 +215,7 @@ TCPServer::operator()(asio::error_code ec, size_t length) {
         // If we don't have a DNS Lookup provider, there's no point in
         // continuing; we exit the coroutine permanently.
         if (lookup_callback_ == NULL) {
-            socket_->close();
-            CORO_YIELD return;
+            return;
         }
 
         // Reset or instantiate objects that will be needed by the
@@ -210,25 +226,24 @@ TCPServer::operator()(asio::error_code ec, size_t length) {
 
         // Schedule a DNS lookup, and yield.  When the lookup is
         // finished, the coroutine will resume immediately after
-        // this point.
+        // this point.  On resume, this method should be called with its
+        // default parameter values (because of the signature of post()'s
+        // handler), so ec shouldn't indicate any error.
         CORO_YIELD io_.post(AsyncLookup<TCPServer>(*this));
+        assert(!ec);
 
         // The 'done_' flag indicates whether we have an answer
         // to send back.  If not, exit the coroutine permanently.
         if (!done_) {
             // TODO: should we keep the connection open for a short time
             // to see if new requests come in?
-            socket_->close();
-            CORO_YIELD return;
+            return;
         }
 
-        if (ec) {
-            CORO_YIELD return;
-        }
         // Call the DNS answer provider to render the answer into
         // wire format
-        (*answer_callback_)(*io_message_, query_message_,
-                            answer_message_, respbuf_);
+        (*answer_callback_)(*io_message_, query_message_, answer_message_,
+                            respbuf_);
 
         // Set up the response, beginning with two length bytes.
         lenbuf.writeUint16(respbuf_->getLength());
@@ -240,13 +255,22 @@ TCPServer::operator()(asio::error_code ec, size_t length) {
         // (though we have nothing further to do, so the coroutine
         // will simply exit at that time).
         CORO_YIELD async_write(*socket_, bufs, *this);
+        if (ec) {
+            LOG_DEBUG(logger, DBGLVL_TRACE_BASIC, ASIODNS_TCP_WRITE_FAIL).
+                arg(ec.message());
+        }
 
-        // All done, cancel the timeout timer
+        // All done, cancel the timeout timer. if it throws, consider it fatal.
         timeout_->cancel();
 
         // TODO: should we keep the connection open for a short time
         // to see if new requests come in?
-        socket_->close();
+        socket_->close(ec);
+        if (ec) {
+            // close() should be unlikely to fail, but we've seen it fail once,
+            // so we log the event (at the lowest level of debug).
+            LOG_DEBUG(logger, 0, ASIODNS_TCP_CLOSE_FAIL).arg(ec.message());
+        }
     }
 }
 
@@ -259,14 +283,23 @@ TCPServer::asyncLookup() {
 }
 
 void TCPServer::stop() {
+    asio::error_code ec;
+
     /// we use close instead of cancel, with the same reason
     /// with udp server stop, refer to the udp server code
 
-    acceptor_->close();
+    acceptor_->close(ec);
+    if (ec) {
+        LOG_ERROR(logger, ASIODNS_TCP_CLOSE_ACCEPTOR_FAIL).arg(ec.message());
+    }
+
     // User may stop the server even when it hasn't started to
-    // run, in that that socket_ is empty
+    // run, in that case socket_ is empty
     if (socket_) {
-        socket_->close();
+        socket_->close(ec);
+        if (ec) {
+            LOG_ERROR(logger, ASIODNS_TCP_CLEANUP_CLOSE_FAIL).arg(ec.message());
+        }
     }
 }
 /// Post this coroutine on the ASIO service queue so that it will
@@ -275,6 +308,10 @@ void TCPServer::stop() {
 void
 TCPServer::resume(const bool done) {
     done_ = done;
+
+    // post() can throw due to memory allocation failure, but as like other
+    // cases of the entire BIND 10 implementation, we consider it fatal and
+    // let the exception be propagated.
     io_.post(*this);
 }
 

src/lib/asiodns/tests/dns_server_unittest.cc

@@ -117,7 +117,7 @@ public:
     DummyLookup() :
         allow_resume_(true)
     { }
-    void operator()(const IOMessage& io_message,
+    virtual void operator()(const IOMessage& io_message,
             isc::dns::MessagePtr message,
             isc::dns::MessagePtr answer_message,
             isc::util::OutputBufferPtr buffer,
@@ -147,6 +147,24 @@ class SimpleAnswer : public DNSAnswer, public ServerStopper {
 
 };
 
+/// \brief Mixture of DummyLookup and SimpleAnswer: build the answer in the
+/// lookup callback.  Used with SyncUDPServer.
+class SyncDummyLookup : public DummyLookup {
+public:
+    virtual void operator()(const IOMessage& io_message,
+                            isc::dns::MessagePtr message,
+                            isc::dns::MessagePtr answer_message,
+                            isc::util::OutputBufferPtr buffer,
+                            DNSServer* server) const
+    {
+        buffer->writeData(io_message.getData(), io_message.getDataSize());
+        stopServer();
+        if (allow_resume_) {
+            server->resume(true);
+        }
+    }
+};
+
 // \brief simple client, send one string to server and wait for response
 //  in case, server stopped and client can't get response, there is a timer wait
 //  for specified seconds (the value is just a estimate since server process logic is quite
@@ -353,6 +371,7 @@ class DNSServerTestBase : public::testing::Test {
             server_address_(ip::address::from_string(server_ip)),
             checker_(new DummyChecker()),
             lookup_(new DummyLookup()),
+            sync_lookup_(new SyncDummyLookup()),
             answer_(new SimpleAnswer()),
             udp_client_(new UDPClient(service,
                                       ip::udp::endpoint(server_address_,
@@ -375,6 +394,7 @@ class DNSServerTestBase : public::testing::Test {
             }
             delete checker_;
             delete lookup_;
+            delete sync_lookup_;
             delete answer_;
             delete udp_server_;
             delete udp_client_;
@@ -421,7 +441,8 @@ class DNSServerTestBase : public::testing::Test {
         asio::io_service service;
         const ip::address server_address_;
         DummyChecker* const checker_;
-        DummyLookup*  const lookup_;
+        DummyLookup* lookup_;     // we need to replace it in some cases
+        SyncDummyLookup*  const sync_lookup_;
         SimpleAnswer* const answer_;
         UDPClient*    const udp_client_;
         TCPClient*    const tcp_client_;
@@ -482,19 +503,34 @@ private:
 protected:
     // Using SetUp here so we can ASSERT_*
     void SetUp() {
-        const int fdUDP(getFd(SOCK_DGRAM));
-        ASSERT_NE(-1, fdUDP) << strerror(errno);
-        this->udp_server_ = new UDPServerClass(this->service, fdUDP, AF_INET6,
-                                               this->checker_, this->lookup_,
-                                               this->answer_);
-        const int fdTCP(getFd(SOCK_STREAM));
-        ASSERT_NE(-1, fdTCP) << strerror(errno);
-        this->tcp_server_ = new TCPServer(this->service, fdTCP, AF_INET6,
+        const int fd_udp(getFd(SOCK_DGRAM));
+        ASSERT_NE(-1, fd_udp) << strerror(errno);
+        this->udp_server_ = createServer(fd_udp, AF_INET6);
+        const int fd_tcp(getFd(SOCK_STREAM));
+        ASSERT_NE(-1, fd_tcp) << strerror(errno);
+        this->tcp_server_ = new TCPServer(this->service, fd_tcp, AF_INET6,
                                           this->checker_, this->lookup_,
                                           this->answer_);
     }
+
+    // A helper factory of the tested UDP server class: allow customization
+    // by template specialization.
+    UDPServerClass* createServer(int fd, int af) {
+        return (new UDPServerClass(this->service, fd, af,
+                                   this->checker_, this->lookup_,
+                                   this->answer_));
+    }
 };
 
+// Specialization for SyncUDPServer.  It needs to use SyncDummyLookup.
+template<>
+SyncUDPServer*
+FdInit<SyncUDPServer>::createServer(int fd, int af) {
+    delete this->lookup_;
+    this->lookup_ = new SyncDummyLookup;
+    return (new SyncUDPServer(this->service, fd, af, this->lookup_));
+}
+
 // This makes it the template as gtest wants it.
 template<class Parent>
 class DNSServerTest : public Parent { };
@@ -503,6 +539,11 @@ typedef ::testing::Types<FdInit<UDPServer>, FdInit<SyncUDPServer> >
     ServerTypes;
 TYPED_TEST_CASE(DNSServerTest, ServerTypes);
 
+// Some tests work only for SyncUDPServer, some others work only for
+// (non Sync)UDPServer.  We specialize these tests.
+typedef FdInit<UDPServer> AsyncServerTest;
+typedef FdInit<SyncUDPServer> SyncServerTest;
+
 typedef ::testing::Types<UDPServer, SyncUDPServer> UDPServerTypes;
 TYPED_TEST_CASE(DNSServerTestBase, UDPServerTypes);
 
@@ -513,7 +554,7 @@ asio::io_service* DNSServerTestBase<UDPServerClass>::current_service(NULL);
 
 // Test whether server stopped successfully after client get response
 // client will send query and start to wait for response, once client
-// get response, udp server will be stopped, the io service won't quit
+// get response, UDP server will be stopped, the io service won't quit
 // if udp server doesn't stop successfully.
 TYPED_TEST(DNSServerTest, stopUDPServerAfterOneQuery) {
     this->testStopServerByStopper(this->udp_server_, this->udp_client_,
@@ -532,8 +573,10 @@ TYPED_TEST(DNSServerTest, stopUDPServerBeforeItStartServing) {
 }
 
 
-// Test whether udp server stopped successfully during message check
-TYPED_TEST(DNSServerTest, stopUDPServerDuringMessageCheck) {
+// Test whether udp server stopped successfully during message check.
+// This only works for non-sync server; SyncUDPServer doesn't use checkin
+// callback.
+TEST_F(AsyncServerTest, stopUDPServerDuringMessageCheck) {
     this->testStopServerByStopper(this->udp_server_, this->udp_client_,
                                   this->checker_);
     EXPECT_EQ(std::string(""), this->udp_client_->getReceivedData());
@@ -548,12 +591,13 @@ TYPED_TEST(DNSServerTest, stopUDPServerDuringQueryLookup) {
     EXPECT_TRUE(this->serverStopSucceed());
 }
 
-// Test whether udp server stopped successfully during composing answer
-TYPED_TEST(DNSServerTest, stopUDPServerDuringPrepareAnswer) {
-    this->testStopServerByStopper(this->udp_server_, this->udp_client_,
-                                  this->answer_);
-    EXPECT_EQ(std::string(""), this->udp_client_->getReceivedData());
-    EXPECT_TRUE(this->serverStopSucceed());
+// Test whether UDP server stopped successfully during composing answer.
+// Only works for (non-sync) server because SyncUDPServer doesn't use answer
+// callback.
+TEST_F(AsyncServerTest, stopUDPServerDuringPrepareAnswer) {
+    testStopServerByStopper(udp_server_, udp_client_, answer_);
+    EXPECT_EQ(std::string(""), udp_client_->getReceivedData());
+    EXPECT_TRUE(serverStopSucceed());
 }
 
 void
@@ -565,7 +609,7 @@ stopServerManyTimes(DNSServer *server, unsigned int times) {
 
 // Test whether udp server stop interface can be invoked several times without
 // throw any exception
-TYPED_TEST(DNSServerTest, stopUDPServeMoreThanOnce) {
+TYPED_TEST(DNSServerTest, stopUDPServerMoreThanOnce) {
     ASSERT_NO_THROW({
         boost::function<void()> stop_server_3_times
             = boost::bind(stopServerManyTimes, this->udp_server_, 3);
@@ -668,14 +712,15 @@ TYPED_TEST(DNSServerTest, stopTCPServeMoreThanOnce) {
 TYPED_TEST(DNSServerTestBase, invalidFamily) {
     // We abuse DNSServerTestBase for this test, as we don't need the
     // initialization.
-    EXPECT_THROW(TypeParam(this->service, 0, AF_UNIX, this->checker_,
-                           this->lookup_, this->answer_),
-                 isc::InvalidParameter);
     EXPECT_THROW(TCPServer(this->service, 0, AF_UNIX, this->checker_,
                            this->lookup_, this->answer_),
                  isc::InvalidParameter);
 }
 
+TYPED_TEST(DNSServerTest, invalidFamilyUDP) {
+    EXPECT_THROW(this->createServer(0, AF_UNIX), isc::InvalidParameter);
+}
+
 // It raises an exception when invalid address family is passed
 TYPED_TEST(DNSServerTestBase, invalidTCPFD) {
     // We abuse DNSServerTestBase for this test, as we don't need the
@@ -694,7 +739,7 @@ TYPED_TEST(DNSServerTestBase, invalidTCPFD) {
                  isc::asiolink::IOError);
 }
 
-TYPED_TEST(DNSServerTestBase, DISABLED_invalidUDPFD) {
+TYPED_TEST(DNSServerTest, DISABLED_invalidUDPFD) {
     /*
      FIXME: The UDP server doesn't fail reliably with an invalid FD.
      We need to find a way to trigger it reliably (it seems epoll
@@ -702,14 +747,9 @@ TYPED_TEST(DNSServerTestBase, DISABLED_invalidUDPFD) {
      not the others, maybe we could make it run this at least on epoll-based
      systems).
     */
-    EXPECT_THROW(TypeParam(this->service, -1, AF_INET, this->checker_,
-                           this->lookup_, this->answer_),
-                 isc::asiolink::IOError);
+    EXPECT_THROW(this->createServer(-1, AF_INET), isc::asiolink::IOError);
 }
 
-// A specialized test type for SyncUDPServer.
-typedef FdInit<SyncUDPServer> SyncServerTest;
-
 // Check it rejects some of the unsupported operations
 TEST_F(SyncServerTest, unsupportedOps) {
     EXPECT_THROW(udp_server_->clone(), isc::Unexpected);
@@ -723,4 +763,10 @@ TEST_F(SyncServerTest, mustResume) {
                  isc::Unexpected);
 }
 
+// SyncUDPServer doesn't allow NULL lookup callback.
+TEST_F(SyncServerTest, nullLookupCallback) {
+    EXPECT_THROW(SyncUDPServer(service, 0, AF_INET, NULL),
+                 isc::InvalidParameter);
+}
+
 }

src/lib/asiodns/udp_server.cc

@@ -82,8 +82,8 @@ struct UDPServer::Data {
          answer_callback_(answer)
     {
         if (af != AF_INET && af != AF_INET6) {
-            isc_throw(InvalidParameter, "Address family must be either AF_INET "
-                      "or AF_INET6, not " << af);
+            isc_throw(InvalidParameter, "Address family must be either AF_INET"
+                      " or AF_INET6, not " << af);
         }
         LOG_DEBUG(logger, DBGLVL_TRACE_BASIC, ASIODNS_FD_ADD_UDP).arg(fd);
         try {
@@ -212,14 +212,19 @@ UDPServer::operator()(asio::error_code ec, size_t length) {
                     buffer(data_->data_.get(), MAX_LENGTH), *data_->sender_,
                     *this);
 
-                // Abort on fatal errors
-                // TODO: add log
+                // See TCPServer::operator() for details on error handling.
                 if (ec) {
                     using namespace asio::error;
-                    if (ec.value() != would_block && ec.value() != try_again &&
-                        ec.value() != interrupted) {
+                    const error_code::value_type err_val = ec.value();
+                    if (err_val == operation_aborted ||
+                        err_val == bad_descriptor) {
                         return;
                     }
+                    if (err_val != would_block && err_val != try_again &&
+                        err_val != interrupted) {
+                        LOG_ERROR(logger, ASIODNS_UDP_RECEIVE_FAIL).
+                            arg(ec.message());
+                    }
                 }
 
             } while (ec || length == 0);
@@ -270,7 +275,7 @@ UDPServer::operator()(asio::error_code ec, size_t length) {
         // If we don't have a DNS Lookup provider, there's no point in
         // continuing; we exit the coroutine permanently.
         if (data_->lookup_callback_ == NULL) {
-            CORO_YIELD return;
+            return;
         }
 
         // Instantiate objects that will be needed by the
@@ -287,7 +292,7 @@ UDPServer::operator()(asio::error_code ec, size_t length) {
         // The 'done_' flag indicates whether we have an answer
         // to send back.  If not, exit the coroutine permanently.
         if (!data_->done_) {
-            CORO_YIELD return;
+            return;
         }
 
         // Call the DNS answer provider to render the answer into
@@ -322,6 +327,8 @@ UDPServer::asyncLookup() {
 /// Stop the UDPServer
 void
 UDPServer::stop() {
+    asio::error_code ec;
+
     /// Using close instead of cancel, because cancel
     /// will only cancel the asynchronized event already submitted
     /// to io service, the events post to io service after
@@ -330,7 +337,10 @@ UDPServer::stop() {
     /// for it won't be scheduled by io service not matter it is
     /// submit to io service before or after close call. And we will
     //  get bad_descriptor error.
-    data_->socket_->close();
+    data_->socket_->close(ec);
+    if (ec) {
+        LOG_ERROR(logger, ASIODNS_UDP_CLOSE_FAIL).arg(ec.message());
+    }
 }
 
 /// Post this coroutine on the ASIO service queue so that it will
@@ -339,7 +349,7 @@ UDPServer::stop() {
 void
 UDPServer::resume(const bool done) {
     data_->done_ = done;
-    data_->io_.post(*this);
+    data_->io_.post(*this);  // this can throw, but can be considered fatal.
 }
 
 } // namespace asiodns

src/lib/asiolink/io_service.cc

@@ -1,4 +1,4 @@
-// Copyright (C) 2011  Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2011,2013  Internet Systems Consortium, Inc. ("ISC")
 //
 // Permission to use, copy, modify, and/or distribute this software for any
 // purpose with or without fee is hereby granted, provided that the above
@@ -24,6 +24,23 @@
 namespace isc {
 namespace asiolink {
 
+namespace {
+// A trivial wrapper for boost::function.  SunStudio doesn't seem to be capable
+// of handling a boost::function object if directly passed to
+// io_service::post().
+class CallbackWrapper {
+public:
+    CallbackWrapper(const boost::function<void()>& callback) :
+        callback_(callback)
+    {}
+    void operator()() {
+        callback_();
+    }
+private:
+    boost::function<void()> callback_;
+};
+}
+
 class IOServiceImpl {
 private:
     IOServiceImpl(const IOService& source);
@@ -63,6 +80,10 @@ public:
     /// It will eventually be removed once the wrapper interface is
     /// generalized.
     asio::io_service& get_io_service() { return io_service_; };
+    void post(const boost::function<void ()>& callback) {
+        const CallbackWrapper wrapper(callback);
+        io_service_.post(wrapper);
+    }
 private:
     asio::io_service io_service_;
     asio::io_service::work work_;
@@ -96,5 +117,10 @@ IOService::get_io_service() {
     return (io_impl_->get_io_service());
 }
 
+void
+IOService::post(const boost::function<void ()>& callback) {
+    return (io_impl_->post(callback));
+}
+
 } // namespace asiolink
 } // namespace isc

src/lib/asiolink/io_service.h

@@ -1,4 +1,4 @@
-// Copyright (C) 2011  Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2011,2013  Internet Systems Consortium, Inc. ("ISC")
 //
 // Permission to use, copy, modify, and/or distribute this software for any
 // purpose with or without fee is hereby granted, provided that the above
@@ -15,6 +15,8 @@
 #ifndef ASIOLINK_IO_SERVICE_H
 #define ASIOLINK_IO_SERVICE_H 1
 
+#include <boost/function.hpp>
+
 namespace asio {
     class io_service;
 }
@@ -70,6 +72,17 @@ public:
     /// generalized.
     asio::io_service& get_io_service();
 
+    /// \brief Post a callback to the end of the queue.
+    ///
+    /// Requests the callback be called sometime later. It is not guaranteed
+    /// by the underlying asio, but it can reasonably be expected the callback
+    /// is put to the end of the callback queue. It is not called from within
+    /// this function.
+    ///
+    /// It may be used to implement "background" work, for example (doing stuff
+    /// by small bits that are called from time to time).
+    void post(const boost::function<void ()>& callback);
+
 private:
     IOServiceImpl* io_impl_;
 };

src/lib/asiolink/tests/Makefile.am

@@ -33,6 +33,7 @@ run_unittests_SOURCES += tcp_endpoint_unittest.cc
 run_unittests_SOURCES += tcp_socket_unittest.cc
 run_unittests_SOURCES += udp_endpoint_unittest.cc
 run_unittests_SOURCES += udp_socket_unittest.cc
+run_unittests_SOURCES += io_service_unittest.cc
 
 run_unittests_CPPFLAGS = $(AM_CPPFLAGS) $(GTEST_INCLUDES)
 

src/lib/asiolink/tests/io_service_unittest.cc

@@ -0,0 +1,48 @@
+// Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <asiolink/io_service.h>
+
+#include <gtest/gtest.h>
+#include <boost/bind.hpp>
+#include <vector>
+
+using namespace isc::asiolink;
+
+namespace {
+
+void
+postedEvent(std::vector<int>* destination, int value) {
+    destination->push_back(value);
+}
+
+// Check the posted events are called, in the same order they are posted.
+TEST(IOService, post) {
+    std::vector<int> called;
+    IOService service;
+    // Post two events
+    service.post(boost::bind(&postedEvent, &called, 1));
+    service.post(boost::bind(&postedEvent, &called, 2));
+    // They have not yet been called
+    EXPECT_TRUE(called.empty());
+    // Process two events
+    service.run_one();
+    service.run_one();
+    // Both events were called in the right order
+    ASSERT_EQ(2, called.size());
+    EXPECT_EQ(1, called[0]);
+    EXPECT_EQ(2, called[1]);
+}
+
+}

src/lib/bench/benchmark.h

@@ -1,4 +1,4 @@
-// Copyright (C) 2010  Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2010,2013  Internet Systems Consortium, Inc. ("ISC")
 //
 // Permission to use, copy, modify, and/or distribute this software for any
 // purpose with or without fee is hereby granted, provided that the above
@@ -93,7 +93,7 @@ namespace bench {
 ///        vector<int>::const_iterator end_key = keys_.end();
 ///        for (iter = keys_.begin(); iter != end_key; ++iter) {
 ///            data_.find(*iter);
-///        }        
+///        }
 ///        return (keys_.size());
 ///    }
 ///    const set<int>& data_;

src/lib/datasrc/Makefile.am

@@ -24,8 +24,7 @@ CLEANFILES += datasrc_config.h
 CLEANFILES += static.zone
 
 lib_LTLIBRARIES = libb10-datasrc.la
-libb10_datasrc_la_SOURCES = data_source.h
-libb10_datasrc_la_SOURCES += exceptions.h
+libb10_datasrc_la_SOURCES = exceptions.h
 libb10_datasrc_la_SOURCES += zone.h zone_finder.h zone_finder.cc
 libb10_datasrc_la_SOURCES += zone_finder_context.cc
 libb10_datasrc_la_SOURCES += zone_iterator.h
@@ -40,6 +39,9 @@ libb10_datasrc_la_SOURCES += master_loader_callbacks.cc
 libb10_datasrc_la_SOURCES += rrset_collection_base.h rrset_collection_base.cc
 libb10_datasrc_la_SOURCES += zone_loader.h zone_loader.cc
 libb10_datasrc_la_SOURCES += cache_config.h cache_config.cc
+libb10_datasrc_la_SOURCES += zone_table_accessor.h
+libb10_datasrc_la_SOURCES += zone_table_accessor_cache.h
+libb10_datasrc_la_SOURCES += zone_table_accessor_cache.cc
 nodist_libb10_datasrc_la_SOURCES = datasrc_messages.h datasrc_messages.cc
 libb10_datasrc_la_LDFLAGS = -no-undefined -version-info 1:0:1
 

src/lib/datasrc/cache_config.cc

@@ -15,10 +15,19 @@
 #include <datasrc/cache_config.h>
 #include <datasrc/client.h>
 #include <datasrc/memory/load_action.h>
+#include <datasrc/memory/zone_data_loader.h>
+
+#include <util/memory_segment.h>
+
 #include <dns/name.h>
+#include <dns/rrclass.h>
+
 #include <cc/data.h>
 #include <exceptions/exceptions.h>
 
+#include <boost/bind.hpp>
+
+#include <cassert>
 #include <map>
 #include <string>
 
@@ -37,7 +46,7 @@ getEnabledFromConf(const Element& conf) {
 
 std::string
 getSegmentTypeFromConf(const Element& conf) {
-    // If cache-zones is not explicitly configured, use the default type.
+    // If cache-type is not explicitly configured, use the default type.
     // (Ideally we should retrieve the default from the spec).
     if (!conf.contains("cache-type")) {
         return ("local");
@@ -108,6 +117,82 @@ CacheConfig::CacheConfig(const std::string& datasrc_type,
     }
 }
 
+namespace {
+
+// We would like to use boost::bind for this. However, the loadZoneData takes
+// a reference, while we have a shared pointer to the iterator -- and we need
+// to keep it alive as long as the ZoneWriter is alive. Therefore we can't
+// really just dereference it and pass it, since it would get destroyed once
+// the getCachedZoneWriter would end. This class holds the shared pointer
+// alive, otherwise is mostly simple.
+//
+// It might be doable with nested boost::bind, but it would probably look
+// more awkward and complicated than this.
+class IteratorLoader {
+public:
+    IteratorLoader(const dns::RRClass& rrclass, const dns::Name& name,
+                   const ZoneIteratorPtr& iterator) :
+        rrclass_(rrclass),
+        name_(name),
+        iterator_(iterator)
+    {}
+    memory::ZoneData* operator()(util::MemorySegment& segment) {
+        return (memory::loadZoneData(segment, rrclass_, name_, *iterator_));
+    }
+private:
+    const dns::RRClass rrclass_;
+    const dns::Name name_;
+    ZoneIteratorPtr iterator_;
+};
+
+// We can't use the loadZoneData function directly in boost::bind, since
+// it is overloaded and the compiler can't choose the correct version
+// reliably and fails. So we simply wrap it into an unique name.
+memory::ZoneData*
+loadZoneDataFromFile(util::MemorySegment& segment, const dns::RRClass& rrclass,
+                     const dns::Name& name, const std::string& filename)
+{
+    return (memory::loadZoneData(segment, rrclass, name, filename));
+}
+
+} // unnamed namespace
+
+memory::LoadAction
+CacheConfig::getLoadAction(const dns::RRClass& rrclass,
+                           const dns::Name& zone_name) const
+{
+    // First, check if the specified zone is configured to be cached.
+    Zones::const_iterator found = zone_config_.find(zone_name);
+    if (found == zone_config_.end()) {
+        return (memory::LoadAction());
+    }
+
+    if (!found->second.empty()) {
+        // This is "MasterFiles" data source.
+        return (boost::bind(loadZoneDataFromFile, _1, rrclass, zone_name,
+                            found->second));
+    }
+
+    // Otherwise there must be a "source" data source (ensured by constructor)
+    assert(datasrc_client_);
+
+    // If the specified zone name does not exist in our client of the source,
+    // DataSourceError is thrown, which is exactly the result what we
+    // want, so no need to handle it.
+    ZoneIteratorPtr iterator(datasrc_client_->getIterator(zone_name));
+    if (!iterator) {
+        // This shouldn't happen for a compliant implementation of
+        // DataSourceClient, but we'll protect ourselves from buggy
+        // implementations.
+        isc_throw(Unexpected, "getting LoadAction for " << zone_name
+                  << "/" << rrclass << " resulted in Null zone iterator");
+    }
+
+    // Wrap the iterator into the correct functor (which keeps it alive as
+    // long as it is needed).
+    return (IteratorLoader(rrclass, zone_name, iterator));
+}
+
 } // namespace internal
 } // namespace datasrc
 } // namespace isc

src/lib/datasrc/cache_config.h

@@ -17,7 +17,7 @@
 
 #include <exceptions/exceptions.h>
 
-#include <dns/name.h>
+#include <dns/dns_fwd.h>
 #include <cc/data.h>
 #include <datasrc/memory/load_action.h>
 
@@ -53,7 +53,6 @@ public:
 /// object that can be used for loading zones, regardless of the underlying
 /// data source properties, i.e., whether it's special "MasterFiles" type
 /// or other generic data sources.
-/// NOTE: this part will be done in #2834.
 ///
 /// This class is publicly defined so it can be tested directly, but
 /// it's essentially private to the \c ConfigurableClientList class.
@@ -144,12 +143,56 @@ public:
     /// \throw None
     const std::string& getSegmentType() const { return (segment_type_); }
 
-    /// \todo the following definition is tentative, mainly for tests.
-    /// In #2834 we'll (probably) extend it to be a custom iterator so
-    /// the caller can iterate over the whole set of zones, loading the
-    /// content in memory.
+    /// \brief Return a \c LoadAction functor to load zone data into memory.
+    ///
+    /// This method returns an appropriate \c LoadAction functor that can be
+    /// passed to a \c memory::ZoneWriter object to load data of the specified
+    /// zone into memory.  The source of the zone data differs depending on
+    /// the cache configuration (either a master file or another data source),
+    /// but this method hides the details and works as a unified interface
+    /// for the caller.
+    ///
+    /// If the specified zone is not configured to be cached, it returns an
+    /// empty functor (which can be evaluated to be \c false as a boolean).
+    /// It doesn't throw an exception in this case because the expected caller
+    /// of this method would handle such a case internally.
+    ///
+    /// \throw DataSourceError error happens in the underlying data source
+    /// storing the cache data.  Most commonly it's because the specified zone
+    /// doesn't exist there.
+    /// \throw Unexpected Unexpected error happens in the underlying data
+    /// source storing the cache data.  This shouldn't happen as long as the
+    /// data source implementation meets the public API requirement.
+    ///
+    /// \param rrclass The RR class of the zone
+    /// \param zone_name The origin name of the zone
+    /// \return A \c LoadAction functor to load zone data or an empty functor
+    /// (see above).
+    memory::LoadAction getLoadAction(const dns::RRClass& rrlcass,
+                                     const dns::Name& zone_name) const;
+
+    /// \brief Read only iterator type over configured cached zones.
+    ///
+    /// \note This initial version exposes the internal data structure (i.e.
+    /// map from name to string) through this public iterator type for
+    /// simplicity.  In terms of data encapsulation it's better to introduce
+    /// a custom iterator type that only goes through the conceptual list
+    /// of zone names, but due to the limitation of the expected user of this
+    /// class that would probably be premature generalization.  In future,
+    /// we might want to allow getting the list of zones directly from the
+    /// underlying data source.  If and when that happens we should introduce
+    /// a custom type.  In any case, the user of this class should only
+    /// use the typedef, not the original map iterator.  It should also
+    /// use this iterator as a forward iterator (datasource-based iterator
+    /// wouldn't be able to be bidirectional), and it shouldn't use the
+    /// value of the map entry (a string, specifying a path to master file
+    /// for MasterFiles data source).
     typedef std::map<dns::Name, std::string>::const_iterator ConstZoneIterator;
+
+    /// \brief Return the beginning of cached zones in the form of iterator.
     ConstZoneIterator begin() const { return (zone_config_.begin()); }
+
+    /// \brief Return the end of cached zones in the form of iterator.
     ConstZoneIterator end() const { return (zone_config_.end()); }
 
 private:
@@ -158,6 +201,9 @@ private:
     // client of underlying data source, will be NULL for MasterFile datasrc
     const DataSourceClient* datasrc_client_;
 
+    // Maps each of zones to be cached to a string.  For "MasterFiles" type
+    // of data source, the string is a path to the master zone file; for
+    // others it's an empty string.
     typedef std::map<dns::Name, std::string> Zones;
     Zones zone_config_;
 };

src/lib/datasrc/client_list.cc

@@ -31,6 +31,7 @@
 #include <set>
 #include <boost/foreach.hpp>
 #include <boost/bind.hpp>
+#include <boost/scoped_ptr.hpp>
 
 using namespace isc::data;
 using namespace isc::dns;
@@ -119,6 +120,9 @@ ConfigurableClientList::configure(const ConstElementPtr& config,
                 continue;
             }
 
+            // Build in-memory cache configuration, and create a set of
+            // related objects including the in-memory zone table for the
+            // cache.
             boost::shared_ptr<internal::CacheConfig> cache_conf(
                 new internal::CacheConfig(type, dsrc_pair.first, *dconf,
                                           allow_cache));
@@ -127,60 +131,38 @@ ConfigurableClientList::configure(const ConstElementPtr& config,
                                                       cache_conf, rrclass_,
                                                       name));
 
-            if (cache_conf->isEnabled()) {
-                // List the zones we are loading
-                vector<string> zones_origins;
-                if (type == "MasterFiles") {
-                    const map<string, ConstElementPtr>
-                        zones_files(paramConf->mapValue());
-                    for (map<string, ConstElementPtr>::const_iterator
-                         it(zones_files.begin()); it != zones_files.end();
-                         ++it) {
-                        zones_origins.push_back(it->first);
-                    }
-                } else {
-                    const ConstElementPtr zones(dconf->get("cache-zones"));
-                    for (size_t i(0); i < zones->size(); ++i) {
-                        zones_origins.push_back(zones->get(i)->stringValue());
-                    }
+            // If cache is disabled we are done for this data source.
+            // Otherwise load zones into the in-memory cache.
+            if (!cache_conf->isEnabled()) {
+                continue;
+            }
+            internal::CacheConfig::ConstZoneIterator end_of_zones =
+                cache_conf->end();
+            for (internal::CacheConfig::ConstZoneIterator zone_it =
+                     cache_conf->begin();
+                 zone_it != end_of_zones;
+                 ++zone_it)
+            {
+                const Name& zname = zone_it->first;
+                memory::LoadAction load_action;
+                try {
+                    load_action = cache_conf->getLoadAction(rrclass_, zname);
+                } catch (const DataSourceError&) {
+                    isc_throw(ConfigurationError, "Data source error for "
+                              "loading a zone (possibly non-existent) "
+                              << zname << "/" << rrclass_);
                 }
-
-                const shared_ptr<InMemoryClient>
-                    cache(new_data_sources.back().cache_);
-                const DataSourceClient* const
-                    client(new_data_sources.back().data_src_client_);
-
-                for (vector<string>::const_iterator it(zones_origins.begin());
-                     it != zones_origins.end(); ++it) {
-                    const Name origin(*it);
-                    if (type == "MasterFiles") {
-                        try {
-                            cache->load(origin,
-                                        paramConf->get(*it)->stringValue());
-                        } catch (const ZoneLoaderException& e) {
-                            LOG_ERROR(logger, DATASRC_LOAD_FROM_FILE_ERROR)
-                                .arg(origin).arg(e.what());
-                        }
-                    } else {
-                        ZoneIteratorPtr iterator;
-                        try {
-                            iterator = client->getIterator(origin);
-                        } catch (const DataSourceError&) {
-                            isc_throw(ConfigurationError, "Unable to "
-                                      "cache non-existent zone "
-                                      << origin);
-                        }
-                        if (!iterator) {
-                            isc_throw(isc::Unexpected, "Got NULL iterator "
-                                      "for zone " << origin);
-                        }
-                        try {
-                            cache->load(origin, *iterator);
-                        } catch (const ZoneLoaderException& e) {
-                            LOG_ERROR(logger, DATASRC_LOAD_FROM_ITERATOR_ERROR)
-                                .arg(origin).arg(e.what());
-                        }
-                    }
+                assert(load_action); // in this loop this should be always true
+                boost::scoped_ptr<memory::ZoneWriter> writer;
+                try {
+                    writer.reset(new_data_sources.back().ztable_segment_->
+                                 getZoneWriter(load_action, zname, rrclass_));
+                    writer->load();
+                    writer->install();
+                    writer->cleanup();
+                } catch (const ZoneLoaderException& e) {
+                    LOG_ERROR(logger, DATASRC_LOAD_ZONE_ERROR)
+                        .arg(zname).arg(rrclass_).arg(name).arg(e.what());
                 }
             }
         }
@@ -344,46 +326,6 @@ ConfigurableClientList::reload(const Name& name) {
     return (ZONE_SUCCESS);
 }
 
-namespace {
-
-// We would like to use boost::bind for this. However, the loadZoneData takes
-// a reference, while we have a shared pointer to the iterator -- and we need
-// to keep it alive as long as the ZoneWriter is alive. Therefore we can't
-// really just dereference it and pass it, since it would get destroyed once
-// the getCachedZoneWriter would end. This class holds the shared pointer
-// alive, otherwise is mostly simple.
-//
-// It might be doable with nested boost::bind, but it would probably look
-// more awkward and complicated than this.
-class IteratorLoader {
-public:
-    IteratorLoader(const RRClass& rrclass, const Name& name,
-                   const ZoneIteratorPtr& iterator) :
-        rrclass_(rrclass),
-        name_(name),
-        iterator_(iterator)
-    {}
-    memory::ZoneData* operator()(util::MemorySegment& segment) {
-        return (memory::loadZoneData(segment, rrclass_, name_, *iterator_));
-    }
-private:
-    const RRClass rrclass_;
-    const Name name_;
-    ZoneIteratorPtr iterator_;
-};
-
-// We can't use the loadZoneData function directly in boost::bind, since
-// it is overloaded and the compiler can't choose the correct version
-// reliably and fails. So we simply wrap it into an unique name.
-memory::ZoneData*
-loadZoneDataFromFile(util::MemorySegment& segment, const RRClass& rrclass,
-                     const Name& name, const string& filename)
-{
-    return (memory::loadZoneData(segment, rrclass, name, filename));
-}
-
-}
-
 ConfigurableClientList::ZoneWriterPair
 ConfigurableClientList::getCachedZoneWriter(const Name& name) {
     if (!allow_cache_) {
@@ -395,36 +337,15 @@ ConfigurableClientList::getCachedZoneWriter(const Name& name) {
     if (!result.finder) {
         return (ZoneWriterPair(ZONE_NOT_FOUND, ZoneWriterPtr()));
     }
-    // Try to get the in-memory cache for the zone. If there's none,
-    // we can't provide the result.
-    if (!result.info->cache_) {
+
+    // Then get the appropriate load action and create a zone writer.
+    // Note that getCacheConfig() must return non NULL in this module (only
+    // tests could set it to a bogus value).
+    const memory::LoadAction load_action =
+        result.info->getCacheConfig()->getLoadAction(rrclass_, name);
+    if (!load_action) {
         return (ZoneWriterPair(ZONE_NOT_CACHED, ZoneWriterPtr()));
     }
-    memory::LoadAction load_action;
-    DataSourceClient* client(result.info->data_src_client_);
-    if (client != NULL) {
-        // Now finally provide the writer.
-        // If it does not exist in client,
-        // DataSourceError is thrown, which is exactly the result what we
-        // want, so no need to handle it.
-        ZoneIteratorPtr iterator(client->getIterator(name));
-        if (!iterator) {
-            isc_throw(isc::Unexpected, "Null iterator from " << name);
-        }
-        // And wrap the iterator into the correct functor (which
-        // keeps it alive as long as it is needed).
-        load_action = IteratorLoader(rrclass_, name, iterator);
-    } else {
-        // The MasterFiles special case
-        const string filename(result.info->cache_->getFileName(name));
-        if (filename.empty()) {
-            isc_throw(isc::Unexpected, "Confused about missing both filename "
-                      "and data source");
-        }
-        // boost::bind is enough here.
-        load_action = boost::bind(loadZoneDataFromFile, _1, rrclass_, name,
-                                  filename);
-    }
     return (ZoneWriterPair(ZONE_SUCCESS,
                            ZoneWriterPtr(
                                result.info->ztable_segment_->

src/lib/datasrc/client_list.h

@@ -342,8 +342,10 @@ public:
     /// \brief Result of the reload() method.
     enum ReloadResult {
         CACHE_DISABLED,     ///< The cache is not enabled in this list.
-        ZONE_NOT_CACHED,    ///< Zone is served directly, not from cache.
-        ZONE_NOT_FOUND,     ///< Zone does not exist or not cached.
+        ZONE_NOT_CACHED,    ///< Zone is served directly, not from cache
+                            ///  (including the case cache is disabled for
+                            ///  the specific data source).
+        ZONE_NOT_FOUND,     ///< Zone does not exist in this list.
         ZONE_SUCCESS        ///< The zone was successfully reloaded or
                             ///  the writer provided.
     };
@@ -418,6 +420,10 @@ public:
         boost::shared_ptr<memory::InMemoryClient> cache_;
         boost::shared_ptr<memory::ZoneTableSegment> ztable_segment_;
         std::string name_;
+
+        const internal::CacheConfig* getCacheConfig() const {
+            return (cache_conf_.get());
+        }
     private:
         // this is kept private for now.  When it needs to be accessed,
         // we'll add a read-only getter method.

src/lib/datasrc/data_source.h

@@ -1,68 +0,0 @@
-// Copyright (C) 2009  Internet Systems Consortium, Inc. ("ISC")
-//
-// Permission to use, copy, modify, and/or distribute this software for any
-// purpose with or without fee is hereby granted, provided that the above
-// copyright notice and this permission notice appear in all copies.
-//
-// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
-// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
-// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
-// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
-// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
-// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-// PERFORMANCE OF THIS SOFTWARE.
-
-#ifndef DATA_SOURCE_H
-#define DATA_SOURCE_H
-
-#include <stdint.h>
-
-#include <vector>
-
-#include <boost/shared_ptr.hpp>
-
-#include <exceptions/exceptions.h>
-
-#include <dns/name.h>
-#include <dns/rrclass.h>
-#include <cc/data.h>
-
-namespace isc {
-
-namespace dns {
-class Name;
-class RRType;
-class RRset;
-class RRsetList;
-}
-
-namespace datasrc {
-
-/// This exception represents Backend-independent errors relating to
-/// data source operations.
-class DataSourceError : public Exception {
-public:
-    DataSourceError(const char* file, size_t line, const char* what) :
-        isc::Exception(file, line, what) {}
-};
-
-/// \brief No such serial number when obtaining difference iterator
-///
-/// Thrown if either the zone/start serial number or zone/end serial number
-/// combination does not exist in the differences table.  (Note that this
-/// includes the case where the differences table contains no records related
-/// to that zone.)
-class NoSuchSerial : public DataSourceError {
-public:
-    NoSuchSerial(const char* file, size_t line, const char* what) :
-        DataSourceError(file, line, what) {}
-};
-
-}
-}
-
-#endif
-
-// Local Variables:
-// mode: c++
-// End:

src/lib/datasrc/database.cc

@@ -17,7 +17,7 @@
 #include <vector>
 
 #include <datasrc/database.h>
-#include <datasrc/data_source.h>
+#include <datasrc/exceptions.h>
 #include <datasrc/zone_iterator.h>
 #include <datasrc/rrset_collection_base.h>
 
@@ -30,7 +30,7 @@
 #include <dns/rdataclass.h>
 #include <dns/nsec3hash.h>
 
-#include <datasrc/data_source.h>
+#include <datasrc/exceptions.h>
 #include <datasrc/logger.h>
 
 #include <boost/foreach.hpp>
@@ -1707,15 +1707,16 @@ DatabaseUpdater::deleteRRset(const AbstractRRset& rrset) {
             LOG_DEBUG(logger, DBG_TRACE_DETAILED, DATASRC_DATABASE_DELETEDIFF).
                 arg(cvtr.getName()).arg(cvtr.getType()).arg(rdata_txt);
         }
-        const string params[Accessor::DEL_PARAM_COUNT] =
-            { nsec3_type ? cvtr.getNSEC3Name() : cvtr.getName(),
-              cvtr.getType(), rdata_txt,
-              nsec3_type ? cvtr.getNSEC3Name() : cvtr.getRevName() };
         if (nsec3_type) {
+            const string params[Accessor::DEL_NSEC3_PARAM_COUNT] =
+                { cvtr.getNSEC3Name(), cvtr.getType(), rdata_txt };
             accessor_->deleteNSEC3RecordInZone(params);
             LOG_DEBUG(logger, DBG_TRACE_DETAILED, DATASRC_DATABASE_DELETENSEC3).
                 arg(cvtr.getNSEC3Name()).arg(rdata_txt);
         } else {
+            const string params[Accessor::DEL_PARAM_COUNT] =
+                { cvtr.getName(), cvtr.getType(), rdata_txt,
+                  cvtr.getRevName() };
             accessor_->deleteRecordInZone(params);
             LOG_DEBUG(logger, DBG_TRACE_DETAILED, DATASRC_DATABASE_DELETERR).
                 arg(cvtr.getName()).arg(cvtr.getType()).arg(rdata_txt);

src/lib/datasrc/database.h

@@ -24,7 +24,7 @@
 #include <dns/rrset.h>
 #include <dns/rrtype.h>
 
-#include <datasrc/data_source.h>
+#include <datasrc/exceptions.h>
 #include <datasrc/client.h>
 #include <datasrc/zone.h>
 #include <datasrc/logger.h>
@@ -116,14 +116,13 @@ public:
         ADD_NSEC3_COLUMN_COUNT = 4 ///< Number of columns
     };
 
-    /// \brief Definitions of the fields to be passed to deleteRecordInZone()
-    /// and deleteNSEC3RecordInZone()
+    /// \brief Definitions of the fields to be passed to deleteRecordInZone().
     ///
     /// Each derived implementation of deleteRecordInZone() should expect
     /// the "params" array to be filled with the values as described in this
     /// enumeration, in this order.
     ///
-    /// DEL_RNAME is included in case the reversed from is more convenient
+    /// DEL_RNAME is included in case the reversed form is more convenient
     /// for the underlying implementation to identify the record to be
     /// deleted (reversed names are generally easier to sort, which may help
     /// perform the search faster).  It's up to the underlying implementation
@@ -132,16 +131,29 @@ public:
     /// in that sense redundant.  But both are provided so the underlying
     /// implementation doesn't have to deal with DNS level concepts.
     enum DeleteRecordParams {
-        DEL_NAME = 0, ///< The owner name of the record (a domain name)
-                      ///< or the hash label for deleteNSEC3RecordInZone()
+        DEL_NAME = 0, ///< The owner name of the record (a domain name).
         DEL_TYPE = 1, ///< The RRType of the record (A/NS/TXT etc.)
         DEL_RDATA = 2, ///< Full text representation of the record's RDATA
         DEL_RNAME = 3, ///< As DEL_NAME, but with the labels of domain name
-                       ///< in reverse order (eg. org.example.). With NSEC3,
-                       ///< it is the same as DEL_NAME.
+                       ///< in reverse order (eg. org.example.).
         DEL_PARAM_COUNT = 4 ///< Number of parameters
     };
 
+    /// \brief Definitions of the fields to be passed to
+    /// deleteNSEC3RecordInZone().
+    ///
+    /// Each derived implementation of deleteNSEC3RecordInZone() should expect
+    /// the "params" array to be filled with the values as described in this
+    /// enumeration, in this order.
+    enum DeleteNSEC3RecordParams {
+        DEL_NSEC3_HASH = 0, ///< The hash (1st) label of the owren name,
+                            ///< excluding the dot character.
+        DEL_NSEC3_TYPE = 1, ///< The type of RR. Either RRSIG or NSEC3.
+        DEL_NSEC3_RDATA = 2, ///< Full text representation of the record's
+                             ///<  RDATA. Must match the one in the database.
+        DEL_NSEC3_PARAM_COUNT = 3 ///< Number of parameters.
+    };
+
     /// \brief Operation mode when adding a record diff.
     ///
     /// This is used as the "operation" parameter value of addRecordDiff().
@@ -588,11 +600,8 @@ public:
     /// \c addRecordToZone() and \c addNSEC3RecordToZone(), and the same
     /// notes apply to this method.
     ///
-    /// This method uses the same set of parameters to specify the record
-    /// to be deleted as \c deleteRecordInZone(), but the \c DEL_NAME column
-    /// is expected to only store the hash label of the owner name.
-    /// This is the same as \c ADD_NSEC3_HASH column for
-    /// \c addNSEC3RecordToZone().
+    /// This method uses the \c DeleteNSEC3RecordParams enum to specify the
+    /// values.
     ///
     /// \exception DataSourceError Invalid call without starting a transaction,
     /// or other internal database error.
@@ -602,7 +611,7 @@ public:
     /// \param params An array of strings that defines a record to be deleted
     /// from the NSEC3 namespace of the zone.
     virtual void deleteNSEC3RecordInZone(
-        const std::string (&params)[DEL_PARAM_COUNT]) = 0;
+        const std::string (&params)[DEL_NSEC3_PARAM_COUNT]) = 0;
 
     /// \brief Start a general transaction.
     ///

src/lib/datasrc/datasrc_messages.mes

@@ -354,15 +354,12 @@ Therefore, the entire data source will not be available for this process. If
 this is a problem, you should configure the zones of that data source to some
 database backend (sqlite3, for example) and use it from there.
 
-% DATASRC_LOAD_FROM_FILE_ERROR Error loading zone %1: %2
-An error was found in the zone data when it was being loaded from a
-file. The zone was not loaded. The specific error is shown in the
-message, and should be addressed.
-
-% DATASRC_LOAD_FROM_ITERATOR_ERROR Error loading zone %1: %2
-An error was found in the zone data when it was being loaded from
-another data source. The zone was not loaded. The specific error is
-shown in the message, and should be addressed.
+% DATASRC_LOAD_ZONE_ERROR Error loading zone %1/%2 on data source %3: %4
+During data source configuration, an error was found in the zone data
+when it was being loaded in to memory on the shown data source.  This
+particular zone was not loaded, but data source configuration
+continues, possibly loading other zones into memory. The specific
+error is shown in the message, and should be addressed.
 
 % DATASRC_MASTER_LOAD_ERROR %1:%2: Zone '%3/%4' contains error: %5
 There's an error in the given master file. The zone won't be loaded for

src/lib/datasrc/exceptions.h

@@ -20,6 +20,26 @@
 namespace isc {
 namespace datasrc {
 
+/// This exception represents Backend-independent errors relating to
+/// data source operations.
+class DataSourceError : public Exception {
+public:
+    DataSourceError(const char* file, size_t line, const char* what) :
+        isc::Exception(file, line, what) {}
+};
+
+/// \brief No such serial number when obtaining difference iterator
+///
+/// Thrown if either the zone/start serial number or zone/end serial number
+/// combination does not exist in the differences table.  (Note that this
+/// includes the case where the differences table contains no records related
+/// to that zone.)
+class NoSuchSerial : public DataSourceError {
+public:
+    NoSuchSerial(const char* file, size_t line, const char* what) :
+        DataSourceError(file, line, what) {}
+};
+
 /// Base class for a number of exceptions that are thrown while working
 /// with zones.
 struct ZoneException : public Exception {

src/lib/datasrc/factory.cc

@@ -14,7 +14,7 @@
 
 #include "factory.h"
 
-#include "data_source.h"
+#include "exceptions.h"
 #include "database.h"
 #include "sqlite3_accessor.h"
 

src/lib/datasrc/factory.h

@@ -15,7 +15,7 @@
 #ifndef DATA_SOURCE_FACTORY_H
 #define DATA_SOURCE_FACTORY_H 1
 
-#include <datasrc/data_source.h>
+#include <datasrc/exceptions.h>
 #include <datasrc/client.h>
 
 #include <cc/data.h>

src/lib/datasrc/memory/memory_client.cc

@@ -18,15 +18,11 @@
 #include <datasrc/memory/logger.h>
 #include <datasrc/memory/zone_data.h>
 #include <datasrc/memory/rdataset.h>
-#include <datasrc/memory/segment_object_holder.h>
 #include <datasrc/memory/treenode_rrset.h>
 #include <datasrc/memory/zone_finder.h>
-#include <datasrc/memory/zone_data_loader.h>
 #include <datasrc/memory/zone_table_segment.h>
 
-#include <util/memory_segment_local.h>
-
-#include <datasrc/data_source.h>
+#include <datasrc/exceptions.h>
 #include <datasrc/factory.h>
 #include <datasrc/result.h>
 
@@ -34,12 +30,8 @@
 #include <dns/rdataclass.h>
 #include <dns/rrclass.h>
 
-#include <algorithm>
 #include <utility>
-#include <cctype>
-#include <cassert>
 
-using namespace std;
 using namespace isc::dns;
 using namespace isc::dns::rdata;
 using namespace isc::datasrc::memory;
@@ -49,86 +41,14 @@ namespace isc {
 namespace datasrc {
 namespace memory {
 
-using detail::SegmentObjectHolder;
 using boost::shared_ptr;
 
-namespace { // unnamed namespace
-
-// A helper internal class used by the memory client, used for deleting
-// filenames stored in an internal tree.
-class FileNameDeleter {
-public:
-    FileNameDeleter() {}
-
-    void operator()(std::string* filename) const {
-        delete filename;
-    }
-};
-
-} // end of unnamed namespace
-
 InMemoryClient::InMemoryClient(shared_ptr<ZoneTableSegment> ztable_segment,
                                RRClass rrclass) :
     ztable_segment_(ztable_segment),
-    rrclass_(rrclass),
-    zone_count_(0),
-    file_name_tree_(FileNameTree::create(
-        ztable_segment_->getMemorySegment(), false))
+    rrclass_(rrclass)
 {}
 
-InMemoryClient::~InMemoryClient() {
-    MemorySegment& mem_sgmt = ztable_segment_->getMemorySegment();
-    FileNameDeleter deleter;
-    FileNameTree::destroy(mem_sgmt, file_name_tree_, deleter);
-}
-
-result::Result
-InMemoryClient::loadInternal(const isc::dns::Name& zone_name,
-                             const std::string& filename,
-                             ZoneData* zone_data)
-{
-    MemorySegment& mem_sgmt = ztable_segment_->getMemorySegment();
-    SegmentObjectHolder<ZoneData, RRClass> holder(
-        mem_sgmt, zone_data, rrclass_);
-
-    LOG_DEBUG(logger, DBG_TRACE_BASIC, DATASRC_MEMORY_MEM_ADD_ZONE).
-        arg(zone_name).arg(rrclass_);
-
-    // Set the filename in file_name_tree_ now, so that getFileName()
-    // can use it (during zone reloading).
-    FileNameNode* node(NULL);
-    switch (file_name_tree_->insert(mem_sgmt, zone_name, &node)) {
-    case FileNameTree::SUCCESS:
-    case FileNameTree::ALREADYEXISTS:
-        // These are OK
-        break;
-    default:
-        // Can Not Happen
-        assert(false);
-    }
-    // node must point to a valid node now
-    assert(node != NULL);
-
-    const std::string* tstr = node->setData(new std::string(filename));
-    delete tstr;
-
-    ZoneTable* zone_table = ztable_segment_->getHeader().getTable();
-    const ZoneTable::AddResult result(zone_table->addZone(mem_sgmt, rrclass_,
-                                                          zone_name,
-                                                          holder.release()));
-    if (result.code == result::SUCCESS) {
-        // Only increment the zone count if the zone doesn't already
-        // exist.
-        ++zone_count_;
-    }
-    // Destroy the old instance of the zone if there was any
-    if (result.zone_data != NULL) {
-        ZoneData::destroy(mem_sgmt, result.zone_data, rrclass_);
-    }
-
-    return (result.code);
-}
-
 RRClass
 InMemoryClient::getClass() const {
     return (rrclass_);
@@ -136,7 +56,8 @@ InMemoryClient::getClass() const {
 
 unsigned int
 InMemoryClient::getZoneCount() const {
-    return (zone_count_);
+    const ZoneTable* zone_table = ztable_segment_->getHeader().getTable();
+    return (zone_table->getZoneCount());
 }
 
 isc::datasrc::DataSourceClient::FindResult
@@ -162,39 +83,6 @@ InMemoryClient::findZoneData(const isc::dns::Name& zone_name) {
     return (result.zone_data);
 }
 
-result::Result
-InMemoryClient::load(const isc::dns::Name& zone_name,
-                     const std::string& filename)
-{
-    LOG_DEBUG(logger, DBG_TRACE_BASIC, DATASRC_MEMORY_MEM_LOAD).arg(zone_name).
-        arg(filename);
-
-    MemorySegment& mem_sgmt = ztable_segment_->getMemorySegment();
-    ZoneData* zone_data = loadZoneData(mem_sgmt, rrclass_, zone_name,
-                                       filename);
-    return (loadInternal(zone_name, filename, zone_data));
-}
-
-result::Result
-InMemoryClient::load(const isc::dns::Name& zone_name, ZoneIterator& iterator) {
-    MemorySegment& mem_sgmt = ztable_segment_->getMemorySegment();
-    ZoneData* zone_data = loadZoneData(mem_sgmt, rrclass_, zone_name,
-                                       iterator);
-    return (loadInternal(zone_name, string(), zone_data));
-}
-
-const std::string
-InMemoryClient::getFileName(const isc::dns::Name& zone_name) const {
-    const FileNameNode* node(NULL);
-    const FileNameTree::Result result = file_name_tree_->find(zone_name,
-                                                              &node);
-    if (result == FileNameTree::EXACTMATCH) {
-        return (*node->getData());
-    } else {
-        return (std::string());
-    }
-}
-
 namespace {
 
 class MemoryIterator : public ZoneIterator {
@@ -369,7 +257,7 @@ InMemoryClient::getUpdater(const isc::dns::Name&, bool, bool) const {
     isc_throw(isc::NotImplemented, "Update attempt on in memory data source");
 }
 
-pair<ZoneJournalReader::Result, ZoneJournalReaderPtr>
+std::pair<ZoneJournalReader::Result, ZoneJournalReaderPtr>
 InMemoryClient::getJournalReader(const isc::dns::Name&, uint32_t,
                                  uint32_t) const
 {

src/lib/datasrc/memory/memory_client.h

@@ -55,7 +55,7 @@ class ZoneTableSegment;
 class InMemoryClient : public DataSourceClient {
 public:
     ///
-    /// \name Constructors and Destructor.
+    /// \name Constructor.
     ///
     //@{
 
@@ -66,9 +66,6 @@ public:
     /// It never throws an exception otherwise.
     InMemoryClient(boost::shared_ptr<ZoneTableSegment> ztable_segment,
                    isc::dns::RRClass rrclass);
-
-    /// The destructor.
-    ~InMemoryClient();
     //@}
 
     /// \brief Returns the class of the data source client.
@@ -81,68 +78,6 @@ public:
     /// \return The number of zones stored in the client.
     virtual unsigned int getZoneCount() const;
 
-    /// \brief Load zone from masterfile.
-    ///
-    /// This loads data from masterfile specified by filename. It replaces
-    /// current content. The masterfile parsing ability is kind of limited,
-    /// see isc::dns::masterLoad.
-    ///
-    /// This throws isc::dns::MasterLoadError or AddError if there are
-    /// problems with loading (missing file, malformed data, unexpected
-    /// zone, etc. - see isc::dns::masterLoad for details).
-    ///
-    /// In case of internal problems, NullRRset or AssertError could
-    /// be thrown, but they should not be expected. Exceptions caused by
-    /// allocation may be thrown as well.
-    ///
-    /// If anything is thrown, the previous content is preserved (so it can
-    /// be used to update the data, but if user makes a typo, the old one
-    /// is kept).
-    ///
-    /// \param filename The master file to load.
-    ///
-    /// \todo We may need to split it to some kind of build and commit/abort.
-    ///     This will probably be needed when a better implementation of
-    ///     configuration reloading is written.
-    result::Result load(const isc::dns::Name& zone_name,
-                        const std::string& filename);
-
-    /// \brief Load zone from another data source.
-    ///
-    /// This is similar to the other version, but zone's RRsets are provided
-    /// by an iterator of another data source.  On successful load, the
-    /// internal filename will be cleared.
-    ///
-    /// This implementation assumes the iterator produces combined RRsets,
-    /// that is, there should exactly one RRset for the same owner name and
-    /// RR type.  This means the caller is expected to create the iterator
-    /// with \c separate_rrs being \c false.  This implementation also assumes
-    /// RRsets of different names are not mixed; so if the iterator produces
-    /// an RRset of a different name than that of the previous RRset, that
-    /// previous name must never appear in the subsequent sequence of RRsets.
-    /// Note that the iterator API does not ensure this.  If the underlying
-    /// implementation does not follow it, load() will fail.  Note, however,
-    /// that this whole interface is tentative.  in-memory zone loading will
-    /// have to be revisited fundamentally, and at that point this restriction
-    /// probably won't matter.
-    result::Result load(const isc::dns::Name& zone_name,
-                        ZoneIterator& iterator);
-
-    /// Return the master file name of the zone
-    ///
-    /// This method returns the name of the zone's master file to be loaded.
-    /// The returned string will be an empty unless the data source client has
-    /// successfully loaded the \c zone_name zone from a file before.
-    ///
-    /// This method should normally not throw an exception.  But the creation
-    /// of the return string may involve a resource allocation, and if it
-    /// fails, the corresponding standard exception will be thrown.
-    ///
-    /// \return The name of the zone file corresponding to the zone, or
-    /// an empty string if the client hasn't loaded the \c zone_name
-    /// zone from a file before.
-    const std::string getFileName(const isc::dns::Name& zone_name) const;
-
     /// Returns a \c ZoneFinder result that best matches the given name.
     ///
     /// This derived version of the method never throws an exception.
@@ -180,20 +115,8 @@ public:
                      uint32_t end_serial) const;
 
 private:
-    // Some type aliases
-    typedef DomainTree<std::string> FileNameTree;
-    typedef DomainTreeNode<std::string> FileNameNode;
-
-    // Common process for zone load. Registers filename internally and
-    // adds the ZoneData to the ZoneTable.
-    result::Result loadInternal(const isc::dns::Name& zone_name,
-                                const std::string& filename,
-                                ZoneData* zone_data);
-
     boost::shared_ptr<ZoneTableSegment> ztable_segment_;
     const isc::dns::RRClass rrclass_;
-    unsigned int zone_count_;
-    FileNameTree* file_name_tree_;
 };
 
 } // namespace memory

src/lib/datasrc/memory/memory_messages.mes

@@ -126,7 +126,11 @@ RRset is split into multiple locations is not supported yet.
 Debug information. A zone object for this zone is being searched for in the
 in-memory data source.
 
-% DATASRC_MEMORY_MEM_LOAD loading zone '%1' from file '%2'
+% DATASRC_MEMORY_MEM_LOAD_FROM_DATASRC loading zone '%1/%2' from other data source
+Debug information. The content of another  data source is being loaded
+into the memory.
+
+% DATASRC_MEMORY_MEM_LOAD_FROM_FILE loading zone '%1/%2' from file '%3'
 Debug information. The content of master file is being loaded into the memory.
 
 % DATASRC_MEMORY_MEM_NO_NSEC3PARAM NSEC3PARAM is missing for NSEC3-signed zone %1/%2

src/lib/datasrc/memory/rdataset.cc

@@ -28,7 +28,6 @@
 #include <stdint.h>
 #include <algorithm>
 #include <cstring>
-#include <typeinfo>             // for bad_cast
 #include <new>                  // for the placement new
 
 using namespace isc::dns;
@@ -41,13 +40,12 @@ namespace memory {
 namespace {
 RRType
 getCoveredType(const Rdata& rdata) {
-    try {
-        const generic::RRSIG& rrsig_rdata =
-            dynamic_cast<const generic::RRSIG&>(rdata);
-        return (rrsig_rdata.typeCovered());
-    } catch (const std::bad_cast&) {
+    const generic::RRSIG* rrsig_rdata =
+        dynamic_cast<const generic::RRSIG*>(&rdata);
+    if (!rrsig_rdata) {
         isc_throw(BadValue, "Non RRSIG is given where it's expected");
     }
+    return (rrsig_rdata->typeCovered());
 }
 
 // A helper for lowestTTL: restore RRTTL object from wire-format 32-bit data.

src/lib/datasrc/memory/zone_data_loader.cc

@@ -253,6 +253,9 @@ loadZoneData(util::MemorySegment& mem_sgmt,
              const isc::dns::Name& zone_name,
              const std::string& zone_file)
 {
+    LOG_DEBUG(logger, DBG_TRACE_BASIC, DATASRC_MEMORY_MEM_LOAD_FROM_FILE).
+        arg(zone_name).arg(rrclass).arg(zone_file);
+
      return (loadZoneDataInternal(mem_sgmt, rrclass, zone_name,
                                  boost::bind(masterLoaderWrapper,
                                              zone_file.c_str(),
@@ -266,6 +269,9 @@ loadZoneData(util::MemorySegment& mem_sgmt,
              const isc::dns::Name& zone_name,
              ZoneIterator& iterator)
 {
+    LOG_DEBUG(logger, DBG_TRACE_BASIC, DATASRC_MEMORY_MEM_LOAD_FROM_DATASRC).
+        arg(zone_name).arg(rrclass);
+
     return (loadZoneDataInternal(mem_sgmt, rrclass, zone_name,
                                  boost::bind(generateRRsetFromIterator,
                                              &iterator, _1)));

src/lib/datasrc/memory/zone_finder.cc

@@ -18,7 +18,7 @@
 #include <datasrc/memory/rdata_serialization.h>
 
 #include <datasrc/zone_finder.h>
-#include <datasrc/data_source.h>
+#include <datasrc/exceptions.h>
 #include <dns/labelsequence.h>
 #include <dns/name.h>
 #include <dns/rrset.h>

src/lib/datasrc/memory/zone_table.cc

@@ -12,14 +12,15 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-#include <util/memory_segment.h>
-
-#include <dns/name.h>
-
-#include <datasrc/memory/zone_data.h>
 #include <datasrc/memory/zone_table.h>
+#include <datasrc/memory/zone_data.h>
 #include <datasrc/memory/domaintree.h>
 #include <datasrc/memory/segment_object_holder.h>
+#include <datasrc/memory/logger.h>
+
+#include <util/memory_segment.h>
+
+#include <dns/name.h>
 
 #include <boost/function.hpp>
 #include <boost/bind.hpp>
@@ -70,6 +71,9 @@ ZoneTable::AddResult
 ZoneTable::addZone(util::MemorySegment& mem_sgmt, RRClass zone_class,
                    const Name& zone_name, ZoneData* content)
 {
+    LOG_DEBUG(logger, DBG_TRACE_BASIC, DATASRC_MEMORY_MEM_ADD_ZONE).
+        arg(zone_name).arg(rrclass_);
+
     if (content == NULL) {
         isc_throw(isc::BadValue, "Zone content must not be NULL");
     }
@@ -94,6 +98,7 @@ ZoneTable::addZone(util::MemorySegment& mem_sgmt, RRClass zone_class,
     if (old != NULL) {
         return (AddResult(result::EXIST, old));
     } else {
+        ++zone_count_;
         return (AddResult(result::SUCCESS, NULL));
     }
 }

src/lib/datasrc/memory/zone_table.h

@@ -104,6 +104,7 @@ private:
     /// It never throws an exception otherwise.
     ZoneTable(const dns::RRClass& rrclass, ZoneTableTree* zones) :
         rrclass_(rrclass),
+        zone_count_(0),
         zones_(zones)
     {}
 
@@ -139,6 +140,11 @@ public:
     /// is undefined if this condition isn't met).
     static void destroy(util::MemorySegment& mem_sgmt, ZoneTable* ztable);
 
+    /// \brief Return the number of zones contained in the zone table.
+    ///
+    /// \throw None.
+    size_t getZoneCount() const { return (zone_count_); }
+
     /// Add a new zone to the \c ZoneTable.
     ///
     /// This method adds a given zone data to the internal table.
@@ -187,6 +193,7 @@ public:
 
 private:
     const dns::RRClass rrclass_;
+    size_t zone_count_;
     boost::interprocess::offset_ptr<ZoneTableTree> zones_;
 };
 }

src/lib/datasrc/sqlite3_accessor.cc

@@ -25,7 +25,7 @@
 #include <datasrc/sqlite3_accessor.h>
 #include <datasrc/sqlite3_datasrc_messages.h>
 #include <datasrc/logger.h>
-#include <datasrc/data_source.h>
+#include <datasrc/exceptions.h>
 #include <datasrc/factory.h>
 #include <datasrc/database.h>
 #include <util/filename.h>
@@ -1311,20 +1311,14 @@ SQLite3Accessor::deleteRecordInZone(const string (&params)[DEL_PARAM_COUNT]) {
 
 void
 SQLite3Accessor::deleteNSEC3RecordInZone(
-    const string (&params)[DEL_PARAM_COUNT])
+    const string (&params)[DEL_NSEC3_PARAM_COUNT])
 {
     if (!dbparameters_->updating_zone) {
         isc_throw(DataSourceError, "deleting NSEC3-related record in SQLite3 "
                   "data source without transaction");
     }
-    const size_t SQLITE3_DEL_PARAM_COUNT = DEL_PARAM_COUNT - 1;
-    const string sqlite3_params[SQLITE3_DEL_PARAM_COUNT] = {
-        params[DEL_NAME],
-        params[DEL_TYPE],
-        params[DEL_RDATA]
-    };
-    doUpdate<const string (&)[SQLITE3_DEL_PARAM_COUNT]>(
-        *dbparameters_, DEL_NSEC3_RECORD, sqlite3_params,
+    doUpdate<const string (&)[DEL_NSEC3_PARAM_COUNT]>(
+        *dbparameters_, DEL_NSEC3_RECORD, params,
         "delete NSEC3 record from zone");
 }
 

src/lib/datasrc/sqlite3_accessor.h

@@ -17,7 +17,7 @@
 #define DATASRC_SQLITE3_ACCESSOR_H
 
 #include <datasrc/database.h>
-#include <datasrc/data_source.h>
+#include <datasrc/exceptions.h>
 
 #include <exceptions/exceptions.h>
 
@@ -230,7 +230,7 @@ public:
         const std::string (&params)[DEL_PARAM_COUNT]);
 
     virtual void deleteNSEC3RecordInZone(
-        const std::string (&params)[DEL_PARAM_COUNT]);
+        const std::string (&params)[DEL_NSEC3_PARAM_COUNT]);
 
     /// This derived version of the method prepares an SQLite3 statement
     /// for adding the diff first time it's called, and if it fails throws

src/lib/datasrc/tests/Makefile.am

@@ -60,6 +60,7 @@ run_unittests_SOURCES += client_list_unittest.cc
 run_unittests_SOURCES += master_loader_callbacks_test.cc
 run_unittests_SOURCES += zone_loader_unittest.cc
 run_unittests_SOURCES += cache_config_unittest.cc
+run_unittests_SOURCES += zone_table_accessor_unittest.cc
 
 # We need the actual module implementation in the tests (they are not part
 # of libdatasrc)

src/lib/datasrc/tests/cache_config_unittest.cc

@@ -13,10 +13,15 @@
 // PERFORMANCE OF THIS SOFTWARE.
 
 #include <datasrc/cache_config.h>
+#include <datasrc/exceptions.h>
+#include <datasrc/memory/load_action.h>
+#include <datasrc/memory/zone_data.h>
 #include <datasrc/tests/mock_client.h>
 
 #include <cc/data.h>
+#include <util/memory_segment_local.h>
 #include <dns/name.h>
+#include <dns/rrclass.h>
 
 #include <gtest/gtest.h>
 
@@ -28,12 +33,15 @@ using namespace isc::dns;
 using isc::datasrc::unittest::MockDataSourceClient;
 using isc::datasrc::internal::CacheConfig;
 using isc::datasrc::internal::CacheConfigError;
+using isc::datasrc::memory::LoadAction;
+using isc::datasrc::memory::ZoneData;
 
 namespace {
 
 const char* zones[] = {
     "example.org.",
     "example.com.",
+    "null.org",                 // test for bad iterator case
     NULL
 };
 
@@ -50,9 +58,14 @@ protected:
                                        " \"cache-zones\": [\".\"]}"))
     {}
 
+    virtual void TearDown() {
+        EXPECT_TRUE(msgmt_.allMemoryDeallocated());
+    }
+
     MockDataSourceClient mock_client_;
     const ConstElementPtr master_config_; // valid config for MasterFiles
     const ConstElementPtr mock_config_; // valid config for MasterFiles
+    isc::util::MemorySegmentLocal msgmt_;
 };
 
 size_t
@@ -140,6 +153,28 @@ TEST_F(CacheConfigTest, badConstructMasterFiles) {
                  isc::InvalidParameter);
 }
 
+TEST_F(CacheConfigTest, getLoadActionWithMasterFiles) {
+    uint8_t labels_buf[LabelSequence::MAX_SERIALIZED_LENGTH];
+
+    const CacheConfig cache_conf("MasterFiles", 0, *master_config_, true);
+
+    // Check getLoadAction.  Since it returns a mere functor, we can only
+    // check the behavior by actually calling it.  For the purpose of this
+    // test, it should suffice if we confirm the call succeeds and shows
+    // some reasonably valid behavior (we'll check the origin name for that).
+    LoadAction action = cache_conf.getLoadAction(RRClass::IN(),
+                                                 Name::ROOT_NAME());
+    ZoneData* zone_data = action(msgmt_);
+    ASSERT_TRUE(zone_data);
+    EXPECT_EQ(".", zone_data->getOriginNode()->
+              getAbsoluteLabels(labels_buf).toText());
+    ZoneData::destroy(msgmt_, zone_data, RRClass::IN());
+
+    // If the specified zone name is not configured to be cached,
+    // getLoadAction returns empty (false) functor.
+    EXPECT_FALSE(cache_conf.getLoadAction(RRClass::IN(), Name("example.com")));
+}
+
 TEST_F(CacheConfigTest, constructWithMock) {
     // Performing equivalent set of tests as constructMasterFiles
 
@@ -219,6 +254,40 @@ TEST_F(CacheConfigTest, badConstructWithMock) {
                  isc::InvalidParameter);
 }
 
+TEST_F(CacheConfigTest, getLoadActionWithMock) {
+    uint8_t labels_buf[LabelSequence::MAX_SERIALIZED_LENGTH];
+
+    // Similar to MasterFiles counterpart, but using underlying source
+    // data source.
+
+    // Note: there's a mismatch between this configuration and the actual
+    // mock data source content: example.net doesn't exist in the data source.
+    const ConstElementPtr config(Element::fromJSON(
+                                     "{\"cache-enable\": true,"
+                                     " \"cache-zones\": [\"example.org\","
+                                     " \"example.net\", \"null.org\"]}"));
+    const CacheConfig cache_conf("mock", &mock_client_, *config, true);
+    LoadAction action = cache_conf.getLoadAction(RRClass::IN(),
+                                                 Name("example.org"));
+    ZoneData* zone_data = action(msgmt_);
+    ASSERT_TRUE(zone_data);
+    EXPECT_EQ("example.org.", zone_data->getOriginNode()->
+              getAbsoluteLabels(labels_buf).toText());
+    ZoneData::destroy(msgmt_, zone_data, RRClass::IN());
+
+    // Zone not configured for the cache
+    EXPECT_FALSE(cache_conf.getLoadAction(RRClass::IN(), Name("example.com")));
+
+    // Zone configured for the cache but doesn't exist in the underling data
+    // source.
+    EXPECT_THROW(cache_conf.getLoadAction(RRClass::IN(), Name("example.net")),
+                 DataSourceError);
+
+    // buggy data source client: it returns a null pointer from getIterator.
+    EXPECT_THROW(cache_conf.getLoadAction(RRClass::IN(), Name("null.org")),
+                 isc::Unexpected);
+}
+
 TEST_F(CacheConfigTest, getSegmentType) {
     // Default type
     EXPECT_EQ("local",

src/lib/datasrc/tests/client_list_unittest.cc

@@ -14,8 +14,9 @@
 
 #include <datasrc/client_list.h>
 #include <datasrc/client.h>
+#include <datasrc/cache_config.h>
 #include <datasrc/zone_iterator.h>
-#include <datasrc/data_source.h>
+#include <datasrc/exceptions.h>
 #include <datasrc/memory/memory_client.h>
 #include <datasrc/memory/zone_table_segment.h>
 #include <datasrc/memory/zone_finder.h>
@@ -133,29 +134,50 @@ public:
     }
 
     // Install a "fake" cached zone using a temporary underlying data source
-    // client.
-    void prepareCache(size_t index, const Name& zone) {
-        // Prepare the temporary data source client
-        const char* zones[2];
-        const std::string zonename_txt = zone.toText();
-        zones[0] = zonename_txt.c_str();
-        zones[1] = NULL;
-        MockDataSourceClient mock_client(zones);
+    // client.  If 'enabled' is set to false, emulate a disabled cache, in
+    // which case there will be no data in memory.
+    void prepareCache(size_t index, const Name& zone, bool enabled = true) {
+        ConfigurableClientList::DataSourceInfo& dsrc_info =
+                list_->getDataSources()[index];
+        MockDataSourceClient* mock_client =
+            static_cast<MockDataSourceClient*>(dsrc_info.data_src_client_);
+
         // Disable some default features of the mock to distinguish the
         // temporary case from normal case.
-        mock_client.disableA();
-        mock_client.disableBadIterator();
-
-        // Create cache from the temporary data source, and push it to the
-        // client list.
-        const shared_ptr<InMemoryClient> cache(
-            new InMemoryClient(ztable_segment_, rrclass_));
-        cache->load(zone, *mock_client.getIterator(zone, false));
+        mock_client->disableA();
+        mock_client->disableBadIterator();
+
+        // Build new cache config to load the specified zone, and replace
+        // the data source info with the new config.
+        ConstElementPtr cache_conf_elem =
+            Element::fromJSON("{\"type\": \"mock\","
+                              " \"cache-enable\": " +
+                              string(enabled ? "true," : "false,") +
+                              " \"cache-zones\": "
+                              "   [\"" + zone.toText() + "\"]}");
+        boost::shared_ptr<internal::CacheConfig> cache_conf(
+            new internal::CacheConfig("mock", mock_client, *cache_conf_elem,
+                                      true));
+        dsrc_info = ConfigurableClientList::DataSourceInfo(
+            dsrc_info.data_src_client_,
+            dsrc_info.container_,
+            cache_conf, rrclass_, dsrc_info.name_);
+
+        // Load the data into the zone table.
+        if (enabled) {
+            boost::scoped_ptr<memory::ZoneWriter> writer(
+                dsrc_info.ztable_segment_->getZoneWriter(
+                    cache_conf->getLoadAction(rrclass_, zone),
+                    zone, rrclass_));
+            writer->load();
+            writer->install();
+            writer->cleanup(); // not absolutely necessary, but just in case
+        }
 
-        ConfigurableClientList::DataSourceInfo& dsrc_info =
-                list_->getDataSources()[index];
-        dsrc_info.cache_ = cache;
-        dsrc_info.ztable_segment_ = ztable_segment_;
+        // On completion of load revert to the previous state of underlying
+        // data source.
+        mock_client->enableA();
+        mock_client->enableBadIterator();
     }
     // Check the positive result is as we expect it.
     void positiveResult(const ClientList::FindResult& result,
@@ -874,7 +896,7 @@ TYPED_TEST(ReloadTest, reloadSuccess) {
 }
 
 // The cache is not enabled. The load should be rejected.
-TYPED_TEST(ReloadTest, reloadNotEnabled) {
+TYPED_TEST(ReloadTest, reloadNotAllowed) {
     this->list_->configure(this->config_elem_zones_, false);
     const Name name("example.org");
     // We put the cache in even when not enabled. This won't confuse the thing.
@@ -893,6 +915,17 @@ TYPED_TEST(ReloadTest, reloadNotEnabled) {
                        RRType::A())->code);
 }
 
+// Similar to the previous case, but the cache is disabled in config.
+TYPED_TEST(ReloadTest, reloadNotEnabled) {
+    this->list_->configure(this->config_elem_zones_, true);
+    const Name name("example.org");
+    // We put the cache, actually disabling it.
+    this->prepareCache(0, name, false);
+    // In this case we cannot really look up due to the limitation of
+    // the mock implementation.  We only check reload fails.
+    EXPECT_EQ(ConfigurableClientList::ZONE_NOT_CACHED, this->doReload(name));
+}
+
 // Test several cases when the zone does not exist
 TYPED_TEST(ReloadTest, reloadNoSuchZone) {
     this->list_->configure(this->config_elem_zones_, true);
@@ -926,7 +959,7 @@ TYPED_TEST(ReloadTest, reloadNoSuchZone) {
 // Check we gracefuly throw an exception when a zone disappeared in
 // the underlying data source when we want to reload it
 TYPED_TEST(ReloadTest, reloadZoneGone) {
-    this->list_->configure(this->config_elem_, true);
+    this->list_->configure(this->config_elem_zones_, true);
     const Name name("example.org");
     // We put in a cache for non-existent zone. This emulates being loaded
     // and then the zone disappearing. We prefill the cache, so we can check
@@ -936,6 +969,10 @@ TYPED_TEST(ReloadTest, reloadZoneGone) {
     EXPECT_EQ(ZoneFinder::SUCCESS,
               this->list_->find(name).finder_->find(name,
                                                     RRType::SOA())->code);
+    // Remove the zone from the data source.
+    static_cast<MockDataSourceClient*>(
+        this->list_->getDataSources()[0].data_src_client_)->eraseZone(name);
+
     // The zone is not there, so abort the reload.
     EXPECT_THROW(this->doReload(name), DataSourceError);
     // The (cached) zone is not hurt.

src/lib/datasrc/tests/database_unittest.cc

@@ -25,7 +25,7 @@
 #include <datasrc/database.h>
 #include <datasrc/zone.h>
 #include <datasrc/zone_finder.h>
-#include <datasrc/data_source.h>
+#include <datasrc/exceptions.h>
 #include <datasrc/zone_iterator.h>
 
 #include <testutils/dnsmessage_test.h>
@@ -167,7 +167,8 @@ public:
     virtual void addNSEC3RecordToZone(const string (&)[ADD_NSEC3_COLUMN_COUNT])
     {}
     virtual void deleteRecordInZone(const string (&)[DEL_PARAM_COUNT]) {}
-    virtual void deleteNSEC3RecordInZone(const string (&)[DEL_PARAM_COUNT]) {}
+    virtual void deleteNSEC3RecordInZone(const string
+                                         (&)[DEL_NSEC3_PARAM_COUNT]) {}
     virtual void addRecordDiff(int, uint32_t, DiffOperation,
                                const std::string (&)[DIFF_PARAM_COUNT]) {}
 
@@ -634,9 +635,8 @@ private:
     };
 
     // Common subroutine for deleteRecordinZone and deleteNSEC3RecordInZone.
-    void deleteRecord(Domains& domains,
-                      const string (&params)[DEL_PARAM_COUNT])
-    {
+    template<size_t param_count>
+    void deleteRecord(Domains& domains, const string (&params)[param_count]) {
         vector<vector<string> >& records =
             domains[params[DatabaseAccessor::DEL_NAME]];
         records.erase(remove_if(records.begin(), records.end(),
@@ -655,7 +655,7 @@ public:
     }
 
     virtual void deleteNSEC3RecordInZone(
-        const string (&params)[DEL_PARAM_COUNT])
+        const string (&params)[DEL_NSEC3_PARAM_COUNT])
     {
         deleteRecord(*update_nsec3_namespace_, params);
     }

src/lib/datasrc/tests/factory_unittest.cc

@@ -16,7 +16,7 @@
 
 #include <datasrc/datasrc_config.h>
 #include <datasrc/factory.h>
-#include <datasrc/data_source.h>
+#include <datasrc/exceptions.h>
 #include <datasrc/sqlite3_accessor.h>
 
 #include <dns/rrclass.h>

src/lib/datasrc/tests/memory/Makefile.am

@@ -21,6 +21,7 @@ if HAVE_GTEST
 TESTS += run_unittests
 
 run_unittests_SOURCES = run_unittests.cc
+run_unittests_SOURCES += zone_loader_util.h zone_loader_util.cc
 run_unittests_SOURCES += rdata_serialization_unittest.cc
 run_unittests_SOURCES += rdataset_unittest.cc
 run_unittests_SOURCES += domaintree_unittest.cc

src/lib/datasrc/tests/memory/memory_client_unittest.cc

@@ -12,6 +12,8 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
+#include <datasrc/tests/memory/zone_loader_util.h>
+
 #include <exceptions/exceptions.h>
 
 #include <util/memory_segment_local.h>
@@ -26,7 +28,7 @@
 #include <dns/masterload.h>
 
 #include <datasrc/result.h>
-#include <datasrc/data_source.h>
+#include <datasrc/exceptions.h>
 #include <datasrc/memory/zone_data.h>
 #include <datasrc/memory/zone_table.h>
 #include <datasrc/memory/zone_data_updater.h>
@@ -42,6 +44,7 @@
 
 #include <boost/lexical_cast.hpp>
 #include <boost/shared_ptr.hpp>
+#include <boost/scoped_ptr.hpp>
 
 #include <new>                  // for bad_alloc
 
@@ -53,6 +56,7 @@ using namespace isc::datasrc::memory;
 using namespace isc::testutils;
 using boost::shared_ptr;
 using std::vector;
+using isc::datasrc::memory::test::loadZoneIntoTable;
 
 namespace {
 
@@ -169,26 +173,22 @@ protected:
                              zclass_, mem_sgmt_)),
                          client_(new InMemoryClient(ztable_segment_, zclass_))
     {}
-    ~MemoryClientTest() {
-        delete client_;
-    }
     void TearDown() {
-        delete client_;
-        client_ = NULL;
+        client_.reset();
         ztable_segment_.reset();
         EXPECT_TRUE(mem_sgmt_.allMemoryDeallocated()); // catch any leak here.
     }
     const RRClass zclass_;
     test::MemorySegmentTest mem_sgmt_;
     shared_ptr<ZoneTableSegment> ztable_segment_;
-    InMemoryClient* client_;
+    boost::scoped_ptr<InMemoryClient> client_;
 };
 
 TEST_F(MemoryClientTest, loadRRsetDoesntMatchOrigin) {
     // Attempting to load example.org to example.com zone should result
     // in an exception.
-    EXPECT_THROW(client_->load(Name("example.com"),
-                               TEST_DATA_DIR "/example.org-empty.zone"),
+    EXPECT_THROW(loadZoneData(mem_sgmt_, zclass_, Name("example.com"),
+                              TEST_DATA_DIR "/example.org-empty.zone"),
                  ZoneLoaderException);
 }
 
@@ -196,8 +196,8 @@ TEST_F(MemoryClientTest, loadErrorsInParsingZoneMustNotLeak1) {
     // Attempting to load broken example.org zone should result in an
     // exception. This should not leak ZoneData and other such
     // allocations.
-    EXPECT_THROW(client_->load(Name("example.org"),
-                               TEST_DATA_DIR "/example.org-broken1.zone"),
+    EXPECT_THROW(loadZoneData(mem_sgmt_, zclass_, Name("example.org"),
+                              TEST_DATA_DIR "/example.org-broken1.zone"),
                  ZoneLoaderException);
     // Teardown checks for memory segment leaks
 }
@@ -206,50 +206,45 @@ TEST_F(MemoryClientTest, loadErrorsInParsingZoneMustNotLeak2) {
     // Attempting to load broken example.org zone should result in an
     // exception. This should not leak ZoneData and other such
     // allocations.
-    EXPECT_THROW(client_->load(Name("example.org"),
-                               TEST_DATA_DIR "/example.org-broken2.zone"),
+    EXPECT_THROW(loadZoneData(mem_sgmt_, zclass_, Name("example.org"),
+                              TEST_DATA_DIR "/example.org-broken2.zone"),
                  ZoneLoaderException);
     // Teardown checks for memory segment leaks
 }
 
 TEST_F(MemoryClientTest, loadNonExistentZoneFile) {
-    EXPECT_THROW(client_->load(Name("example.org"),
-                               TEST_DATA_DIR "/somerandomfilename"),
+    EXPECT_THROW(loadZoneData(mem_sgmt_, zclass_, Name("example.org"),
+                              TEST_DATA_DIR "/somerandomfilename"),
                  ZoneLoaderException);
     // Teardown checks for memory segment leaks
 }
 
 TEST_F(MemoryClientTest, loadEmptyZoneFileThrows) {
     // When an empty zone file is loaded, the origin doesn't even have
-    // an SOA RR. This condition should be avoided, and hence load()
-    // should throw when an empty zone is loaded.
-
-    EXPECT_EQ(0, client_->getZoneCount());
-
-    EXPECT_THROW(client_->load(Name("."),
-                               TEST_DATA_DIR "/empty.zone"),
+    // an SOA RR. This condition should be avoided, and hence it results in
+    // an exception.
+    EXPECT_THROW(loadZoneData(mem_sgmt_, zclass_, Name("."),
+                              TEST_DATA_DIR "/empty.zone"),
                  ZoneValidationError);
-
-    EXPECT_EQ(0, client_->getZoneCount());
-
     // Teardown checks for memory segment leaks
 }
 
 TEST_F(MemoryClientTest, load) {
     // This is a simple load check for a "full" and correct zone that
     // should not result in any exceptions.
-    client_->load(Name("example.org"),
-                  TEST_DATA_DIR "/example.org.zone");
-    const ZoneData* zone_data =
-        client_->findZoneData(Name("example.org"));
+    ZoneData* zone_data = loadZoneData(mem_sgmt_, zclass_,
+                                       Name("example.org"),
+                                       TEST_DATA_DIR
+                                       "/example.org.zone");
     ASSERT_NE(static_cast<const ZoneData*>(NULL), zone_data);
     EXPECT_FALSE(zone_data->isSigned());
     EXPECT_FALSE(zone_data->isNSEC3Signed());
+    ZoneData::destroy(mem_sgmt_, zone_data, zclass_);
 }
 
 TEST_F(MemoryClientTest, loadFromIterator) {
-    client_->load(Name("example.org"),
-                  *MockIterator::makeIterator(rrset_data));
+    loadZoneIntoTable(*ztable_segment_, Name("example.org"), zclass_,
+                      *MockIterator::makeIterator(rrset_data));
 
     ZoneIteratorPtr iterator(client_->getIterator(Name("example.org")));
 
@@ -281,21 +276,23 @@ TEST_F(MemoryClientTest, loadFromIterator) {
     // Iterating past the end should result in an exception
     EXPECT_THROW(iterator->getNextRRset(), isc::Unexpected);
 
+    // NOTE: The rest of the tests is not actually about InMemoryClient
+
     // Loading the zone with an iterator separating RRs of the same
     // RRset should not fail. It is acceptable to load RRs of the same
     // type again.
-    client_->load(Name("example.org"),
-                  *MockIterator::makeIterator(
-                      rrset_data_separated));
+    loadZoneIntoTable(*ztable_segment_, Name("example.org"), zclass_,
+                      *MockIterator::makeIterator(rrset_data_separated));
 
     // Similar to the previous case, but with separated RRSIGs.
-    client_->load(Name("example.org"),
-                  *MockIterator::makeIterator(
-                      rrset_data_sigseparated));
+    loadZoneIntoTable(*ztable_segment_, Name("example.org"), zclass_,
+                      *MockIterator::makeIterator(rrset_data_sigseparated));
 
     // Emulating bogus iterator implementation that passes empty RRSIGs.
-    EXPECT_THROW(client_->load(Name("example.org"),
-                               *MockIterator::makeIterator(rrset_data, true)),
+    EXPECT_THROW(loadZoneIntoTable(*ztable_segment_, Name("example.org"),
+                                   zclass_,
+                                   *MockIterator::makeIterator(rrset_data,
+                                                               true)),
                  isc::Unexpected);
 }
 
@@ -316,16 +313,16 @@ TEST_F(MemoryClientTest, loadMemoryAllocationFailures) {
             // fail (due to MemorySegmentTest throwing) and we check for
             // leaks when this happens.
             InMemoryClient client2(ztable_segment, zclass_);
-            client2.load(Name("example.org"),
-                         TEST_DATA_DIR "/example.org.zone");
+            loadZoneIntoTable(*ztable_segment, Name("example.org"), zclass_,
+                              TEST_DATA_DIR "/example.org.zone");
         }, std::bad_alloc);
     }
     // Teardown checks for memory segment leaks
 }
 
 TEST_F(MemoryClientTest, loadNSEC3Signed) {
-    client_->load(Name("example.org"),
-                  TEST_DATA_DIR "/example.org-nsec3-signed.zone");
+    loadZoneIntoTable(*ztable_segment_, Name("example.org"), zclass_,
+                      TEST_DATA_DIR "/example.org-nsec3-signed.zone");
     const ZoneData* zone_data =
         client_->findZoneData(Name("example.org"));
     ASSERT_NE(static_cast<const ZoneData*>(NULL), zone_data);
@@ -336,8 +333,8 @@ TEST_F(MemoryClientTest, loadNSEC3Signed) {
 TEST_F(MemoryClientTest, loadNSEC3EmptySalt) {
     // Load NSEC3 with empty ("-") salt. This should not throw or crash
     // or anything.
-    client_->load(Name("example.org"),
-                  TEST_DATA_DIR "/example.org-nsec3-empty-salt.zone");
+    loadZoneIntoTable(*ztable_segment_, Name("example.org"), zclass_,
+                      TEST_DATA_DIR "/example.org-nsec3-empty-salt.zone");
     const ZoneData* zone_data =
         client_->findZoneData(Name("example.org"));
     ASSERT_NE(static_cast<const ZoneData*>(NULL), zone_data);
@@ -346,8 +343,8 @@ TEST_F(MemoryClientTest, loadNSEC3EmptySalt) {
 }
 
 TEST_F(MemoryClientTest, loadNSEC3SignedNoParam) {
-    client_->load(Name("example.org"),
-                  TEST_DATA_DIR "/example.org-nsec3-signed-no-param.zone");
+    loadZoneIntoTable(*ztable_segment_, Name("example.org"), zclass_,
+                      TEST_DATA_DIR "/example.org-nsec3-signed-no-param.zone");
     const ZoneData* zone_data =
         client_->findZoneData(Name("example.org"));
     ASSERT_NE(static_cast<const ZoneData*>(NULL), zone_data);
@@ -360,14 +357,14 @@ TEST_F(MemoryClientTest, loadReloadZone) {
     // doesn't increase.
     EXPECT_EQ(0, client_->getZoneCount());
 
-    client_->load(Name("example.org"),
-                  TEST_DATA_DIR "/example.org-empty.zone");
+    loadZoneIntoTable(*ztable_segment_, Name("example.org"), zclass_,
+                      TEST_DATA_DIR "/example.org-empty.zone");
     EXPECT_EQ(1, client_->getZoneCount());
 
     // Reload zone with same data
 
-    client_->load(Name("example.org"),
-                  client_->getFileName(Name("example.org")));
+    loadZoneIntoTable(*ztable_segment_, Name("example.org"), zclass_,
+                      TEST_DATA_DIR "/example.org-empty.zone");
     EXPECT_EQ(1, client_->getZoneCount());
 
     const ZoneData* zone_data =
@@ -396,8 +393,8 @@ TEST_F(MemoryClientTest, loadReloadZone) {
 
     // Reload zone with different data
 
-    client_->load(Name("example.org"),
-                  TEST_DATA_DIR "/example.org-rrsigs.zone");
+    loadZoneIntoTable(*ztable_segment_, Name("example.org"), zclass_,
+                      TEST_DATA_DIR "/example.org-rrsigs.zone");
     EXPECT_EQ(1, client_->getZoneCount());
 
     zone_data = client_->findZoneData(Name("example.org"));
@@ -441,15 +438,14 @@ TEST_F(MemoryClientTest, loadReloadZone) {
 TEST_F(MemoryClientTest, loadDuplicateType) {
     // This should not result in any exceptions (multiple records of the
     // same name, type are present, one after another in sequence).
-    client_->load(Name("example.org"),
-                  TEST_DATA_DIR "/example.org-duplicate-type.zone");
+    loadZoneIntoTable(*ztable_segment_, Name("example.org"), zclass_,
+                      TEST_DATA_DIR "/example.org-duplicate-type.zone");
 
     // This should not result in any exceptions (multiple records of the
     // same name, type are present, but not one after another in
     // sequence).
-    client_->load(Name("example.org"),
-                  TEST_DATA_DIR
-                  "/example.org-duplicate-type-bad.zone");
+    loadZoneIntoTable(*ztable_segment_, Name("example.org"), zclass_,
+                      TEST_DATA_DIR "/example.org-duplicate-type-bad.zone");
 
     const ZoneData* zone_data =
         client_->findZoneData(Name("example.org"));
@@ -479,104 +475,116 @@ TEST_F(MemoryClientTest, loadDuplicateType) {
 
 TEST_F(MemoryClientTest, loadMultipleCNAMEThrows) {
     // Multiple CNAME RRs should throw.
-    EXPECT_THROW(client_->load(Name("example.org"),
-                               TEST_DATA_DIR
-                               "/example.org-multiple-cname.zone"),
+    EXPECT_THROW(loadZoneIntoTable(*ztable_segment_, Name("example.org"),
+                                   zclass_,
+                                   TEST_DATA_DIR
+                                   "/example.org-multiple-cname.zone"),
                  ZoneDataUpdater::AddError);
     // Teardown checks for memory segment leaks
 }
 
 TEST_F(MemoryClientTest, loadMultipleDNAMEThrows) {
     // Multiple DNAME RRs should throw.
-    EXPECT_THROW(client_->load(Name("example.org"),
-                               TEST_DATA_DIR
-                               "/example.org-multiple-dname.zone"),
+    EXPECT_THROW(loadZoneIntoTable(*ztable_segment_, Name("example.org"),
+                                   zclass_,
+                                   TEST_DATA_DIR
+                                   "/example.org-multiple-dname.zone"),
                  ZoneDataUpdater::AddError);
     // Teardown checks for memory segment leaks
 }
 
 TEST_F(MemoryClientTest, loadMultipleNSEC3Throws) {
     // Multiple NSEC3 RRs should throw.
-    EXPECT_THROW(client_->load(Name("example.org"),
-                               TEST_DATA_DIR
-                               "/example.org-multiple-nsec3.zone"),
+    EXPECT_THROW(loadZoneIntoTable(*ztable_segment_, Name("example.org"),
+                                   zclass_,
+                                   TEST_DATA_DIR
+                                   "/example.org-multiple-nsec3.zone"),
                  ZoneDataUpdater::AddError);
     // Teardown checks for memory segment leaks
 }
 
 TEST_F(MemoryClientTest, loadMultipleNSEC3PARAMThrows) {
     // Multiple NSEC3PARAM RRs should throw.
-    EXPECT_THROW(client_->load(Name("example.org"),
-                               TEST_DATA_DIR
-                               "/example.org-multiple-nsec3param.zone"),
+    EXPECT_THROW(loadZoneIntoTable(*ztable_segment_, Name("example.org"),
+                                   zclass_,
+                                   TEST_DATA_DIR
+                                   "/example.org-multiple-nsec3param.zone"),
                  ZoneDataUpdater::AddError);
     // Teardown checks for memory segment leaks
 }
 
 TEST_F(MemoryClientTest, loadOutOfZoneThrows) {
     // Out of zone names should throw.
-    EXPECT_THROW(client_->load(Name("example.org"),
-                               TEST_DATA_DIR
-                               "/example.org-out-of-zone.zone"),
+    EXPECT_THROW(loadZoneIntoTable(*ztable_segment_, Name("example.org"),
+                                   zclass_,
+                                   TEST_DATA_DIR
+                                   "/example.org-out-of-zone.zone"),
                  ZoneLoaderException);
     // Teardown checks for memory segment leaks
 }
 
 TEST_F(MemoryClientTest, loadWildcardNSThrows) {
     // Wildcard NS names should throw
-    EXPECT_THROW(client_->load(Name("example.org"),
-                               TEST_DATA_DIR
-                               "/example.org-wildcard-ns.zone"),
+    EXPECT_THROW(loadZoneIntoTable(*ztable_segment_, Name("example.org"),
+                                   zclass_,
+                                   TEST_DATA_DIR
+                                   "/example.org-wildcard-ns.zone"),
                  ZoneDataUpdater::AddError);
     // Teardown checks for memory segment leaks
 }
 
 TEST_F(MemoryClientTest, loadWildcardDNAMEThrows) {
     // Wildcard DNAME names should throw
-    EXPECT_THROW(client_->load(Name("example.org"),
-                               TEST_DATA_DIR
-                               "/example.org-wildcard-dname.zone"),
+    EXPECT_THROW(loadZoneIntoTable(*ztable_segment_, Name("example.org"),
+                                   zclass_,
+                                   TEST_DATA_DIR
+                                   "/example.org-wildcard-dname.zone"),
                  ZoneDataUpdater::AddError);
     // Teardown checks for memory segment leaks
 }
 
 TEST_F(MemoryClientTest, loadWildcardNSEC3Throws) {
     // Wildcard NSEC3 names should throw
-    EXPECT_THROW(client_->load(Name("example.org"),
-                               TEST_DATA_DIR
-                               "/example.org-wildcard-nsec3.zone"),
+    EXPECT_THROW(loadZoneIntoTable(*ztable_segment_, Name("example.org"),
+                                   zclass_,
+                                   TEST_DATA_DIR
+                                   "/example.org-wildcard-nsec3.zone"),
                  ZoneDataUpdater::AddError);
     // Teardown checks for memory segment leaks
 }
 
 TEST_F(MemoryClientTest, loadNSEC3WithFewerLabelsThrows) {
     // NSEC3 names with labels != (origin_labels + 1) should throw
-    EXPECT_THROW(client_->load(Name("example.org"),
-                               TEST_DATA_DIR
-                               "/example.org-nsec3-fewer-labels.zone"),
+    EXPECT_THROW(loadZoneIntoTable(*ztable_segment_, Name("example.org"),
+                                   zclass_,
+                                   TEST_DATA_DIR
+                                   "/example.org-nsec3-fewer-labels.zone"),
                  ZoneDataUpdater::AddError);
     // Teardown checks for memory segment leaks
 }
 
 TEST_F(MemoryClientTest, loadNSEC3WithMoreLabelsThrows) {
     // NSEC3 names with labels != (origin_labels + 1) should throw
-    EXPECT_THROW(client_->load(Name("example.org"),
-                               TEST_DATA_DIR
-                               "/example.org-nsec3-more-labels.zone"),
+    EXPECT_THROW(loadZoneIntoTable(*ztable_segment_, Name("example.org"),
+                                   zclass_,
+                                   TEST_DATA_DIR
+                                   "/example.org-nsec3-more-labels.zone"),
                  ZoneDataUpdater::AddError);
     // Teardown checks for memory segment leaks
 }
 
 TEST_F(MemoryClientTest, loadCNAMEAndNotNSECThrows) {
     // CNAME and not NSEC should throw
-    EXPECT_THROW(client_->load(Name("example.org"),
-                               TEST_DATA_DIR
-                               "/example.org-cname-and-not-nsec-1.zone"),
+    EXPECT_THROW(loadZoneIntoTable(*ztable_segment_, Name("example.org"),
+                                   zclass_,
+                                   TEST_DATA_DIR
+                                   "/example.org-cname-and-not-nsec-1.zone"),
                  ZoneDataUpdater::AddError);
 
-    EXPECT_THROW(client_->load(Name("example.org"),
-                               TEST_DATA_DIR
-                               "/example.org-cname-and-not-nsec-2.zone"),
+    EXPECT_THROW(loadZoneIntoTable(*ztable_segment_, Name("example.org"),
+                                   zclass_,
+                                   TEST_DATA_DIR
+                                   "/example.org-cname-and-not-nsec-2.zone"),
                  ZoneDataUpdater::AddError);
 
     // Teardown checks for memory segment leaks
@@ -584,41 +592,41 @@ TEST_F(MemoryClientTest, loadCNAMEAndNotNSECThrows) {
 
 TEST_F(MemoryClientTest, loadDNAMEAndNSApex1) {
     // DNAME + NS (apex) is OK
-    client_->load(Name("example.org"),
-                  TEST_DATA_DIR
-                  "/example.org-dname-ns-apex-1.zone");
+    loadZoneIntoTable(*ztable_segment_, Name("example.org"), zclass_,
+                      TEST_DATA_DIR "/example.org-dname-ns-apex-1.zone");
     // Teardown checks for memory segment leaks
 }
 
 TEST_F(MemoryClientTest, loadDNAMEAndNSApex2) {
     // DNAME + NS (apex) is OK (reverse order)
-    client_->load(Name("example.org"),
-                  TEST_DATA_DIR
-                  "/example.org-dname-ns-apex-2.zone");
+    loadZoneIntoTable(*ztable_segment_, Name("example.org"), zclass_,
+                      TEST_DATA_DIR "/example.org-dname-ns-apex-2.zone");
     // Teardown checks for memory segment leaks
 }
 
 TEST_F(MemoryClientTest, loadDNAMEAndNSNonApex1) {
     // DNAME + NS (non-apex) must throw
-    EXPECT_THROW(client_->load(Name("example.org"),
-                               TEST_DATA_DIR
-                               "/example.org-dname-ns-nonapex-1.zone"),
+    EXPECT_THROW(loadZoneIntoTable(*ztable_segment_, Name("example.org"),
+                                   zclass_,
+                                   TEST_DATA_DIR
+                                   "/example.org-dname-ns-nonapex-1.zone"),
                  ZoneDataUpdater::AddError);
     // Teardown checks for memory segment leaks
 }
 
 TEST_F(MemoryClientTest, loadDNAMEAndNSNonApex2) {
     // DNAME + NS (non-apex) must throw (reverse order)
-    EXPECT_THROW(client_->load(Name("example.org"),
-                               TEST_DATA_DIR
-                               "/example.org-dname-ns-nonapex-2.zone"),
+    EXPECT_THROW(loadZoneIntoTable(*ztable_segment_, Name("example.org"),
+                                   zclass_,
+                                   TEST_DATA_DIR
+                                   "/example.org-dname-ns-nonapex-2.zone"),
                  ZoneDataUpdater::AddError);
     // Teardown checks for memory segment leaks
 }
 
 TEST_F(MemoryClientTest, loadRRSIGs) {
-    client_->load(Name("example.org"),
-                  TEST_DATA_DIR "/example.org-rrsigs.zone");
+    loadZoneIntoTable(*ztable_segment_, Name("example.org"), zclass_,
+                      TEST_DATA_DIR "/example.org-rrsigs.zone");
     EXPECT_EQ(1, client_->getZoneCount());
 }
 
@@ -642,37 +650,31 @@ TEST_F(MemoryClientTest, loadRRSIGsRdataMixedCoveredTypes) {
 
     rrsets_vec.push_back(rrset);
 
-    EXPECT_THROW(
-        client_->load(Name("example.org"),
-                      *MockVectorIterator::makeIterator(rrsets_vec)),
-        ZoneDataUpdater::AddError);
+    EXPECT_THROW(loadZoneIntoTable(*ztable_segment_, Name("example.org"),
+                                   zclass_,
+                                   *MockVectorIterator::makeIterator(
+                                       rrsets_vec)),
+                 ZoneDataUpdater::AddError);
     // Teardown checks for memory segment leaks
 }
 
 TEST_F(MemoryClientTest, getZoneCount) {
     EXPECT_EQ(0, client_->getZoneCount());
-    client_->load(Name("example.org"), TEST_DATA_DIR "/example.org-empty.zone");
+    loadZoneIntoTable(*ztable_segment_, Name("example.org"), zclass_,
+                      TEST_DATA_DIR "/example.org-empty.zone");
+    // We've updated the zone table already in the client, so the count
+    // should also be incremented indirectly.
     EXPECT_EQ(1, client_->getZoneCount());
 }
 
-TEST_F(MemoryClientTest, getFileNameForNonExistentZone) {
-    // Zone "example.org." doesn't exist
-    EXPECT_TRUE(client_->getFileName(Name("example.org.")).empty());
-}
-
-TEST_F(MemoryClientTest, getFileName) {
-    client_->load(Name("example.org"), TEST_DATA_DIR "/example.org-empty.zone");
-    EXPECT_EQ(TEST_DATA_DIR "/example.org-empty.zone",
-              client_->getFileName(Name("example.org")));
-}
-
 TEST_F(MemoryClientTest, getIteratorForNonExistentZone) {
     // Zone "." doesn't exist
     EXPECT_THROW(client_->getIterator(Name(".")), DataSourceError);
 }
 
 TEST_F(MemoryClientTest, getIterator) {
-    client_->load(Name("example.org"), TEST_DATA_DIR "/example.org-empty.zone");
+    loadZoneIntoTable(*ztable_segment_, Name("example.org"), zclass_,
+                      TEST_DATA_DIR "/example.org-empty.zone");
     ZoneIteratorPtr iterator(client_->getIterator(Name("example.org")));
 
     // First we have the SOA
@@ -693,8 +695,8 @@ TEST_F(MemoryClientTest, getIterator) {
 }
 
 TEST_F(MemoryClientTest, getIteratorSeparateRRs) {
-    client_->load(Name("example.org"),
-                  TEST_DATA_DIR "/example.org-multiple.zone");
+    loadZoneIntoTable(*ztable_segment_, Name("example.org"), zclass_,
+                      TEST_DATA_DIR "/example.org-multiple.zone");
 
     // separate_rrs = false
     ZoneIteratorPtr iterator(client_->getIterator(Name("example.org")));
@@ -746,8 +748,8 @@ TEST_F(MemoryClientTest, getIteratorSeparateRRs) {
 
 // Test we get RRSIGs and NSEC3s too for iterating with separate RRs
 TEST_F(MemoryClientTest, getIteratorSeparateSigned) {
-    client_->load(Name("example.org"),
-                       TEST_DATA_DIR "/example.org-nsec3-signed.zone");
+    loadZoneIntoTable(*ztable_segment_, Name("example.org"), zclass_,
+                      TEST_DATA_DIR "/example.org-nsec3-signed.zone");
     ZoneIteratorPtr iterator(client_->getIterator(Name("example.org"), true));
     bool seen_rrsig = false, seen_nsec3 = false;
     for (ConstRRsetPtr rrset = iterator->getNextRRset();
@@ -764,7 +766,8 @@ TEST_F(MemoryClientTest, getIteratorSeparateSigned) {
 }
 
 TEST_F(MemoryClientTest, getIteratorGetSOAThrowsNotImplemented) {
-    client_->load(Name("example.org"), TEST_DATA_DIR "/example.org-empty.zone");
+    loadZoneIntoTable(*ztable_segment_, Name("example.org"), zclass_,
+                      TEST_DATA_DIR "/example.org-empty.zone");
     ZoneIteratorPtr iterator(client_->getIterator(Name("example.org")));
 
     // This method is not implemented.
@@ -780,16 +783,17 @@ TEST_F(MemoryClientTest, addEmptyRRsetThrows) {
     rrsets_vec.push_back(RRsetPtr(new RRset(Name("example.org"), zclass_,
                                             RRType::A(), RRTTL(3600))));
 
-    EXPECT_THROW(
-        client_->load(Name("example.org"),
-                      *MockVectorIterator::makeIterator(rrsets_vec)),
-        ZoneDataUpdater::AddError);
+    EXPECT_THROW(loadZoneIntoTable(*ztable_segment_, Name("example.org"),
+                                   zclass_,
+                                   *MockVectorIterator::makeIterator(
+                                       rrsets_vec)),
+                 ZoneDataUpdater::AddError);
     // Teardown checks for memory segment leaks
 }
 
 TEST_F(MemoryClientTest, findZoneData) {
-    client_->load(Name("example.org"),
-                  TEST_DATA_DIR "/example.org-rrsigs.zone");
+    loadZoneIntoTable(*ztable_segment_, Name("example.org"), zclass_,
+                      TEST_DATA_DIR "/example.org-rrsigs.zone");
 
     const ZoneData* zone_data = client_->findZoneData(Name("example.com"));
     EXPECT_EQ(static_cast<const ZoneData*>(NULL), zone_data);

src/lib/datasrc/tests/memory/zone_finder_unittest.cc

@@ -12,8 +12,9 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-#include "memory_segment_test.h"
-#include "zone_table_segment_test.h"
+#include <datasrc/tests/memory/memory_segment_test.h>
+#include <datasrc/tests/memory/zone_table_segment_test.h>
+#include <datasrc/tests/memory/zone_loader_util.h>
 
 // NOTE: this faked_nsec3 inclusion (and all related code below)
 // was ported during #2109 for the convenience of implementing #2218
@@ -21,14 +22,14 @@
 // In #2219 the original is expected to be removed, and this file should
 // probably be moved here (and any leftover code not handled in #2218 should
 // be cleaned up)
-#include "../../tests/faked_nsec3.h"
+#include <datasrc/tests/faked_nsec3.h>
 
 #include <datasrc/memory/zone_finder.h>
 #include <datasrc/memory/zone_data_updater.h>
 #include <datasrc/memory/rdata_serialization.h>
 #include <datasrc/memory/zone_table_segment.h>
 #include <datasrc/memory/memory_client.h>
-#include <datasrc/data_source.h>
+#include <datasrc/exceptions.h>
 #include <datasrc/client.h>
 #include <testutils/dnsmessage_test.h>
 
@@ -1610,12 +1611,13 @@ TEST_F(InMemoryZoneFinderTest, findOrphanRRSIG) {
 // \brief testcase for #2504 (Problem in inmem NSEC denial of existence
 // handling)
 TEST_F(InMemoryZoneFinderTest, NSECNonExistentTest) {
+    const Name name("example.com.");
     shared_ptr<ZoneTableSegment> ztable_segment(
          new ZoneTableSegmentTest(class_, mem_sgmt_));
+    loadZoneIntoTable(*ztable_segment, name, class_,
+                      TEST_DATA_DIR "/2504-test.zone");
     InMemoryClient client(ztable_segment, class_);
-    Name name("example.com.");
 
-    client.load(name, TEST_DATA_DIR "/2504-test.zone");
     DataSourceClient::FindResult result(client.findZone(name));
 
     // Check for a non-existing name
@@ -1771,16 +1773,17 @@ TEST_F(InMemoryZoneFinderNSEC3Test, findNSEC3MissingOrigin) {
      DefaultNSEC3HashCreator creator;
      setNSEC3HashCreator(&creator);
 
+     const Name name("example.com.");
      shared_ptr<ZoneTableSegment> ztable_segment(
           new ZoneTableSegmentTest(class_, mem_sgmt_));
+     loadZoneIntoTable(*ztable_segment, name, class_,
+                       TEST_DATA_DIR "/2503-test.zone");
      InMemoryClient client(ztable_segment, class_);
-     Name name("example.com.");
 
-     client.load(name, TEST_DATA_DIR "/2503-test.zone");
      DataSourceClient::FindResult result(client.findZone(name));
 
      // Check for a non-existing name
-     Name search_name("nonexist.example.com.");
+     const Name search_name("nonexist.example.com.");
      ZoneFinder::FindNSEC3Result find_result(
           result.zone_finder->findNSEC3(search_name, true));
      // findNSEC3() must have completed (not throw or assert). Because

src/lib/datasrc/tests/memory/zone_loader_util.cc

@@ -0,0 +1,93 @@
+// Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <datasrc/tests/memory/zone_loader_util.h>
+
+#include <datasrc/zone_iterator.h>
+#include <datasrc/cache_config.h>
+#include <datasrc/memory/zone_table_segment.h>
+#include <datasrc/memory/zone_data_loader.h>
+#include <datasrc/memory/zone_writer.h>
+
+#include <dns/dns_fwd.h>
+
+#include <cc/data.h>
+
+#include <boost/bind.hpp>
+#include <boost/scoped_ptr.hpp>
+
+#include <string>
+
+namespace isc {
+namespace datasrc {
+namespace memory {
+namespace test {
+
+void
+loadZoneIntoTable(ZoneTableSegment& zt_sgmt, const dns::Name& zname,
+                  const dns::RRClass& zclass, const std::string& zone_file)
+{
+    const isc::datasrc::internal::CacheConfig cache_conf(
+        "MasterFiles", NULL, *data::Element::fromJSON(
+            "{\"cache-enable\": true,"
+            " \"params\": {\"" + zname.toText() + "\": \"" + zone_file +
+            "\"}}"), true);
+    boost::scoped_ptr<memory::ZoneWriter> writer(
+        zt_sgmt.getZoneWriter(cache_conf.getLoadAction(zclass, zname),
+                              zname, zclass));
+    writer->load();
+    writer->install();
+    writer->cleanup();
+}
+
+namespace {
+// borrowed from CacheConfig's internal
+class IteratorLoader {
+public:
+    IteratorLoader(const dns::RRClass& rrclass, const dns::Name& name,
+                   ZoneIterator& iterator) :
+        rrclass_(rrclass),
+        name_(name),
+        iterator_(iterator)
+    {}
+    memory::ZoneData* operator()(util::MemorySegment& segment) {
+        return (memory::loadZoneData(segment, rrclass_, name_, iterator_));
+    }
+private:
+    const dns::RRClass rrclass_;
+    const dns::Name name_;
+    ZoneIterator& iterator_;
+};
+}
+
+void
+loadZoneIntoTable(ZoneTableSegment& zt_sgmt, const dns::Name& zname,
+                  const dns::RRClass& zclass, ZoneIterator& iterator)
+{
+    boost::scoped_ptr<memory::ZoneWriter> writer(
+        zt_sgmt.getZoneWriter(IteratorLoader(zclass, zname, iterator),
+                              zname, zclass));
+    writer->load();
+    writer->install();
+    writer->cleanup();
+}
+
+} // namespace test
+} // namespace memory
+} // namespace datasrc
+} // namespace isc
+
+// Local Variables:
+// mode: c++
+// End:

src/lib/datasrc/tests/memory/zone_loader_util.h

@@ -0,0 +1,57 @@
+// Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#ifndef DATASRC_MEMORY_TEST_ZONE_LOADER_UTIL_H
+#define DATASRC_MEMORY_TEST_ZONE_LOADER_UTIL_H 1
+
+#include <datasrc/memory/zone_table_segment.h>
+#include <datasrc/memory/zone_data_loader.h>
+
+#include <dns/dns_fwd.h>
+
+#include <string>
+
+namespace isc {
+namespace datasrc {
+namespace memory {
+namespace test {
+
+/// \brief A shortcut utility to load a specified zone into ZoneTableSegment
+/// from a file.
+///
+/// This function does nothing special, simply provides a shortcut for commonly
+/// used pattern that would be used in tests with a ZoneTableSegment loading
+/// a zone from file into it.
+void
+loadZoneIntoTable(ZoneTableSegment& zt_sgmt, const dns::Name& zname,
+                  const dns::RRClass& zclass, const std::string& zone_file);
+
+/// \brief A shortcut utility to load a specified zone into ZoneTableSegment
+/// from a zone iterator.
+///
+/// This is similar to the other version, but use a zone iterator as the
+/// source of the zone data.
+void
+loadZoneIntoTable(ZoneTableSegment& zt_sgmt, const dns::Name& zname,
+                  const dns::RRClass& zclass, ZoneIterator& iterator);
+} // namespace test
+} // namespace memory
+} // namespace datasrc
+} // namespace isc
+
+#endif // DATASRC_MEMORY_TEST_ZONE_LOADER_UTIL_H
+
+// Local Variables:
+// mode: c++
+// End:

src/lib/datasrc/tests/memory/zone_table_unittest.cc

@@ -70,9 +70,13 @@ TEST_F(ZoneTableTest, create) {
 }
 
 TEST_F(ZoneTableTest, addZone) {
+    // By default there's no zone contained.
+    EXPECT_EQ(0, zone_table->getZoneCount());
+
     // It doesn't accept empty (NULL) zones
     EXPECT_THROW(zone_table->addZone(mem_sgmt_, zclass_, zname1, NULL),
                  isc::BadValue);
+    EXPECT_EQ(0, zone_table->getZoneCount()); // count is still 0
 
     SegmentObjectHolder<ZoneData, RRClass> holder1(
         mem_sgmt_, ZoneData::create(mem_sgmt_, zname1), zclass_);
@@ -85,6 +89,7 @@ TEST_F(ZoneTableTest, addZone) {
     EXPECT_EQ(static_cast<const ZoneData*>(NULL), result1.zone_data);
     // It got released by it
     EXPECT_EQ(static_cast<const ZoneData*>(NULL), holder1.get());
+    EXPECT_EQ(1, zone_table->getZoneCount()); // count is now incremented
 
     // Duplicate add doesn't replace the existing data.
     SegmentObjectHolder<ZoneData, RRClass> holder2(
@@ -99,6 +104,7 @@ TEST_F(ZoneTableTest, addZone) {
     EXPECT_EQ(static_cast<const ZoneData*>(NULL), holder2.get());
     // We need to release the old one manually
     ZoneData::destroy(mem_sgmt_, result2.zone_data, zclass_);
+    EXPECT_EQ(1, zone_table->getZoneCount()); // count doesn't change.
 
     SegmentObjectHolder<ZoneData, RRClass> holder3(
         mem_sgmt_, ZoneData::create(mem_sgmt_, Name("EXAMPLE.COM")),
@@ -115,11 +121,13 @@ TEST_F(ZoneTableTest, addZone) {
     EXPECT_EQ(result::SUCCESS,
               zone_table->addZone(mem_sgmt_, zclass_, zname2,
                                   holder4.release()).code);
+    EXPECT_EQ(2, zone_table->getZoneCount());
     SegmentObjectHolder<ZoneData, RRClass> holder5(
         mem_sgmt_, ZoneData::create(mem_sgmt_, zname3), zclass_);
     EXPECT_EQ(result::SUCCESS,
               zone_table->addZone(mem_sgmt_, zclass_, zname3,
                                   holder5.release()).code);
+    EXPECT_EQ(3, zone_table->getZoneCount());
 
     // Have the memory segment throw an exception in extending the internal
     // tree.  It still shouldn't cause memory leak (which would be detected

src/lib/datasrc/tests/mock_client.cc

@@ -16,7 +16,7 @@
 #include <datasrc/client.h>
 #include <datasrc/result.h>
 #include <datasrc/zone_iterator.h>
-#include <datasrc/data_source.h>
+#include <datasrc/exceptions.h>
 
 #include <dns/name.h>
 #include <dns/rrclass.h>

src/lib/datasrc/tests/mock_client.h

@@ -52,7 +52,12 @@ public:
     }
     virtual ZoneIteratorPtr getIterator(const dns::Name& name, bool) const;
     void disableA() { have_a_ = false; }
+    void enableA() { have_a_ = true; }
     void disableBadIterator() { use_baditerator_ = false; }
+    void enableBadIterator() { use_baditerator_ = true; }
+    void eraseZone(const dns::Name& zone_name) {
+        zones.erase(zone_name);
+    }
     const std::string type_;
     const data::ConstElementPtr configuration_;
 

src/lib/datasrc/tests/sqlite3_accessor_unittest.cc

@@ -16,7 +16,7 @@
 
 #include <datasrc/sqlite3_accessor.h>
 
-#include <datasrc/data_source.h>
+#include <datasrc/exceptions.h>
 
 #include <dns/rrclass.h>
 
@@ -823,8 +823,7 @@ const char* const nsec3_sig_data[DatabaseAccessor::ADD_NSEC3_COLUMN_COUNT] = {
 const char* const nsec3_deleted_data[] = {
     // Delete parameters for nsec3_data
     apex_hash, nsec3_data[DatabaseAccessor::ADD_NSEC3_TYPE],
-    nsec3_data[DatabaseAccessor::ADD_NSEC3_RDATA],
-    apex_hash
+    nsec3_data[DatabaseAccessor::ADD_NSEC3_RDATA]
 };
 
 class SQLite3Update : public SQLite3AccessorTest {
@@ -854,6 +853,7 @@ protected:
     std::string add_columns[DatabaseAccessor::ADD_COLUMN_COUNT];
     std::string add_nsec3_columns[DatabaseAccessor::ADD_NSEC3_COLUMN_COUNT];
     std::string del_params[DatabaseAccessor::DEL_PARAM_COUNT];
+    std::string del_nsec3_params[DatabaseAccessor::DEL_NSEC3_PARAM_COUNT];
     std::string diff_params[DatabaseAccessor::DIFF_PARAM_COUNT];
 
     vector<const char* const*> expected_stored; // placeholder for checkRecords
@@ -1193,8 +1193,9 @@ TEST_F(SQLite3Update, deleteNSEC3Record) {
 
     // Delete it, and confirm that.
     copy(nsec3_deleted_data,
-         nsec3_deleted_data + DatabaseAccessor::DEL_PARAM_COUNT, del_params);
-    accessor->deleteNSEC3RecordInZone(del_params);
+         nsec3_deleted_data + DatabaseAccessor::DEL_NSEC3_PARAM_COUNT,
+         del_nsec3_params);
+    accessor->deleteNSEC3RecordInZone(del_nsec3_params);
     checkNSEC3Records(*accessor, zone_id, apex_hash, empty_stored);
 
     // Commit the change, and confirm the deleted data still isn't there.
@@ -1251,7 +1252,7 @@ TEST_F(SQLite3Update, invalidDelete) {
     EXPECT_THROW(accessor->deleteRecordInZone(del_params), DataSourceError);
 
     // Same for NSEC3.
-    EXPECT_THROW(accessor->deleteNSEC3RecordInZone(del_params),
+    EXPECT_THROW(accessor->deleteNSEC3RecordInZone(del_nsec3_params),
                  DataSourceError);
 }
 

src/lib/datasrc/tests/zone_finder_context_unittest.cc

@@ -18,9 +18,13 @@
 #include <dns/name.h>
 #include <dns/rrclass.h>
 
+#include <cc/data.h>
+
 #include <datasrc/zone_finder.h>
+#include <datasrc/cache_config.h>
 #include <datasrc/memory/memory_client.h>
 #include <datasrc/memory/zone_table_segment.h>
+#include <datasrc/memory/zone_writer.h>
 #include <datasrc/database.h>
 #include <datasrc/sqlite3_accessor.h>
 
@@ -32,6 +36,7 @@
 #include <boost/bind.hpp>
 #include <boost/foreach.hpp>
 #include <boost/shared_ptr.hpp>
+#include <boost/scoped_ptr.hpp>
 
 #include <fstream>
 #include <sstream>
@@ -39,11 +44,13 @@
 
 using namespace std;
 using boost::shared_ptr;
+using boost::scoped_ptr;
 
 using namespace isc::data;
 using namespace isc::util;
 using namespace isc::dns;
 using namespace isc::datasrc;
+using isc::data::Element;
 using isc::datasrc::memory::InMemoryClient;
 using isc::datasrc::memory::ZoneTableSegment;
 using namespace isc::testutils;
@@ -64,11 +71,22 @@ typedef DataSourceClientPtr (*ClientCreator)(RRClass, const Name&);
 // Creator for the in-memory client to be tested
 DataSourceClientPtr
 createInMemoryClient(RRClass zclass, const Name& zname) {
+    const internal::CacheConfig cache_conf(
+        "MasterFiles", NULL, *Element::fromJSON(
+            "{\"cache-enable\": true,"
+            " \"params\":"
+            "  {\"" + zname.toText() + "\": \"" +
+            string(TEST_ZONE_FILE) + "\"}}"), true);
     shared_ptr<ZoneTableSegment> ztable_segment(
-        ZoneTableSegment::create(zclass, "local"));
+        ZoneTableSegment::create(zclass, cache_conf.getSegmentType()));
+    scoped_ptr<memory::ZoneWriter> writer(
+        ztable_segment->getZoneWriter(cache_conf.getLoadAction(zclass, zname),
+                                      zname, zclass));
+    writer->load();
+    writer->install();
+    writer->cleanup();
     shared_ptr<InMemoryClient> client(new InMemoryClient(ztable_segment,
                                                          zclass));
-    client->load(zname, TEST_ZONE_FILE);
 
     return (client);
 }

src/lib/datasrc/tests/zone_loader_unittest.cc

@@ -13,11 +13,13 @@
 // PERFORMANCE OF THIS SOFTWARE.
 
 #include <datasrc/zone_loader.h>
-#include <datasrc/data_source.h>
+#include <datasrc/exceptions.h>
 #include <datasrc/rrset_collection_base.h>
+#include <datasrc/cache_config.h>
 
 #include <datasrc/memory/zone_table_segment.h>
 #include <datasrc/memory/memory_client.h>
+#include <datasrc/memory/zone_writer.h>
 
 #include <dns/rrclass.h>
 #include <dns/name.h>
@@ -26,6 +28,8 @@
 #include <util/memory_segment_local.h>
 #include <exceptions/exceptions.h>
 
+#include <cc/data.h>
+
 #include <gtest/gtest.h>
 
 #include <boost/shared_ptr.hpp>
@@ -37,6 +41,7 @@
 
 using namespace isc::dns;
 using namespace isc::datasrc;
+using isc::data::Element;
 using boost::shared_ptr;
 using std::string;
 using std::vector;
@@ -301,13 +306,28 @@ protected:
 
         // (re)configure zone table, then (re)construct the in-memory client
         // with it.
-        ztable_segment_.reset(memory::ZoneTableSegment::create(rrclass_,
-                                                               "local"));
-        source_client_.reset(new memory::InMemoryClient(ztable_segment_,
-                                                        rrclass_));
+        string param_data;
         if (filename) {
-            source_client_->load(zone, string(TEST_DATA_DIR) + "/" + filename);
+            param_data = "\"" + zone.toText() + "\": \"" +
+                string(TEST_DATA_DIR) + "/" + filename + "\"";
         }
+        const internal::CacheConfig cache_conf(
+            "MasterFiles", NULL, *Element::fromJSON(
+                "{\"cache-enable\": true,"
+                " \"params\": {" + param_data + "}}"), true);
+        ztable_segment_.reset(memory::ZoneTableSegment::create(
+                                  rrclass_, cache_conf.getSegmentType()));
+        if (filename) {
+            boost::scoped_ptr<memory::ZoneWriter> writer(
+                ztable_segment_->getZoneWriter(cache_conf.getLoadAction(
+                                                   rrclass_, zone),
+                                               zone, rrclass_));
+            writer->load();
+            writer->install();
+            writer->cleanup();
+        }
+        source_client_.reset(new memory::InMemoryClient(ztable_segment_,
+                                                        rrclass_));
     }
 private:
     const RRClass rrclass_;

src/lib/datasrc/tests/zone_table_accessor_unittest.cc

@@ -0,0 +1,112 @@
+// Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <datasrc/zone_table_accessor_cache.h>
+#include <datasrc/cache_config.h>
+#include <datasrc/tests/mock_client.h>
+
+#include <exceptions/exceptions.h>
+
+#include <cc/data.h>
+
+#include <dns/name.h>
+
+#include <gtest/gtest.h>
+
+using namespace isc::dns;
+using namespace isc::datasrc;
+using namespace isc::datasrc::internal;
+using isc::data::Element;
+using isc::datasrc::unittest::MockDataSourceClient;
+
+namespace {
+
+// This test checks the abstract ZoneTableAccessor interface using
+// ZoneTableAccessorCache instances, thereby testing the top level interface
+// and the derived class behavior.  If ZoneTableAccessorCache becomes more
+// complicated we may have to separate some test cases into dedicated test
+// fixture.
+class ZoneTableAccessorTest : public ::testing::Test {
+protected:
+    ZoneTableAccessorTest() :
+        // The paths of the zone files are dummy and don't even exist,
+        // but it doesn't matter in this test.
+        config_spec_(Element::fromJSON(
+                         "{\"cache-enable\": true,"
+                         " \"params\": "
+                         "  {\"example.com\": \"/example-com.zone\","
+                         "   \"example.org\": \"/example-org.zone\"}"
+                         "}")),
+        cache_config_("MasterFiles", NULL, *config_spec_, true),
+        accessor_(cache_config_)
+    {}
+
+private:
+    const isc::data::ConstElementPtr config_spec_;
+    const CacheConfig cache_config_;
+protected:
+    ZoneTableAccessorCache accessor_;
+};
+
+TEST_F(ZoneTableAccessorTest, iteratorFromCache) {
+    // Confirm basic iterator behavior.
+    ZoneTableAccessor::IteratorPtr it = accessor_.getIterator();
+    ASSERT_TRUE(it);
+    EXPECT_FALSE(it->isLast());
+    EXPECT_EQ(0, it->getCurrent().index); // index is always 0 for this version
+    EXPECT_EQ(Name("example.com"), it->getCurrent().origin);
+
+    it->next();
+    EXPECT_FALSE(it->isLast());
+    EXPECT_EQ(0, it->getCurrent().index);
+    EXPECT_EQ(Name("example.org"), it->getCurrent().origin);
+
+    it->next();                 // shouldn't cause disruption
+    EXPECT_TRUE(it->isLast());
+
+    // getCurrent() and next() will be rejected once iterator reaches the end
+    EXPECT_THROW(it->getCurrent(), isc::InvalidOperation);
+    EXPECT_THROW(it->next(), isc::InvalidOperation);
+}
+
+TEST_F(ZoneTableAccessorTest, emptyTable) {
+    // Empty zone table is possible, while mostly useless.
+    const CacheConfig empty_config(
+        "MasterFiles", NULL, *Element::fromJSON(
+            "{\"cache-enable\": true, \"params\": {}}"), true);
+    ZoneTableAccessorCache accessor(empty_config);
+    ZoneTableAccessor::IteratorPtr it = accessor.getIterator();
+    ASSERT_TRUE(it);
+    EXPECT_TRUE(it->isLast());
+    EXPECT_THROW(it->getCurrent(), isc::InvalidOperation);
+    EXPECT_THROW(it->next(), isc::InvalidOperation);
+}
+
+TEST_F(ZoneTableAccessorTest, disabledTable) {
+    // Zone table based on disabled cache is effectively empty.
+    const char* zones[] = { "example.org.", "example.com.", NULL };
+    MockDataSourceClient mock_client(zones);
+    const CacheConfig mock_config(
+        "mock", &mock_client, *Element::fromJSON(
+            "{\"cache-enable\": false,"
+            " \"cache-zones\": [\"example.com\", \"example.org\"]}"), true);
+    ZoneTableAccessorCache accessor(mock_config);
+    ZoneTableAccessor::IteratorPtr it = accessor.getIterator();
+    ASSERT_TRUE(it);
+    EXPECT_TRUE(it->isLast());
+    EXPECT_THROW(it->getCurrent(), isc::InvalidOperation);
+    EXPECT_THROW(it->next(), isc::InvalidOperation);
+}
+
+}

src/lib/datasrc/zone_finder.cc

@@ -13,8 +13,9 @@
 // PERFORMANCE OF THIS SOFTWARE.
 
 #include <datasrc/zone_finder.h>
-#include <datasrc/data_source.h>
+#include <datasrc/exceptions.h>
 
+#include <dns/rrclass.h>
 #include <dns/rdata.h>
 #include <dns/rrset.h>
 #include <dns/rrtype.h>

src/lib/datasrc/zone_loader.cc

@@ -16,7 +16,7 @@
 #include <datasrc/master_loader_callbacks.h>
 
 #include <datasrc/client.h>
-#include <datasrc/data_source.h>
+#include <datasrc/exceptions.h>
 #include <datasrc/zone_iterator.h>
 #include <datasrc/zone.h>
 #include <datasrc/logger.h>

src/lib/datasrc/zone_loader.h

@@ -15,7 +15,7 @@
 #ifndef DATASRC_ZONE_LOADER_H
 #define DATASRC_ZONE_LOADER_H
 
-#include <datasrc/data_source.h>
+#include <datasrc/exceptions.h>
 
 #include <dns/master_loader.h>
 

src/lib/datasrc/zone_table_accessor.h

@@ -0,0 +1,212 @@
+// Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#ifndef DATASRC_ZONE_TABLE_ACCESSOR_H
+#define DATASRC_ZONE_TABLE_ACCESSOR_H
+
+#include <dns/name.h>
+
+#include <boost/noncopyable.hpp>
+#include <boost/shared_ptr.hpp>
+
+#include <stdint.h>
+
+namespace isc {
+namespace datasrc {
+
+/// \brief Information of a zone stored in a data source zone table.
+///
+/// This is a straightforward composite type that represents an entry of
+/// the conceptual zone table referenced by \c ZoneTableAccessor.
+/// An object of this structure is specifically intended to be returned by
+/// \c ZoneTableIterator.
+///
+/// This is essentially a read-only tuple; only created by
+/// \c ZoneTableAccessor, and once created it will be immutable.
+///
+/// \note Once Trac #2144 is completed, this struct must be defined as
+/// non-assignable because it has a const member variable.
+struct ZoneSpec {
+    /// \brief Constructor.
+    ZoneSpec(uint32_t index_param, const dns::Name& origin_param) :
+        index(index_param), origin(origin_param)
+    {}
+
+    /// \brief Numeric zone index.
+    ///
+    /// In the current initial version, this field is just a placeholder.
+    /// In the future, we'll probably define it as a unique index in the table
+    /// for that particular zone so that applications can distinguish
+    /// and specify different zones efficiently. Until it's fixed, this field
+    /// shouldn't be used by applications.
+    const uint32_t index;
+
+    /// \brief The origin name of the zone.
+    const dns::Name origin;
+};
+
+/// \brief A simple iterator of zone table.
+///
+/// This is an abstract base class providing simple iteration operation
+/// over zones stored in a data source.  A concrete object of this class
+/// is expected to be returned by \c ZoneTableAccessor::getIterator().
+///
+/// The interface is intentionally simplified and limited: it works
+/// "forward-only", i.e, only goes from begin to end one time; it's not
+/// copyable, assignable, nor comparable.  For the latter reasons it's not
+/// compatible with standard iterator traits.  It's simplified because it's
+/// not clear what kind of primitive can be used in specific data sources.
+/// In particular, iteration in a database-based data source would be very
+/// restrictive.  So it's better to begin with minimal guaranteed features
+/// at the base class.  If we find it possible to loosen the restriction
+/// as we implement more derived versions, we may extend the features later.
+///
+/// Likewise, this iterator does not guarantee the ordering of the zones
+/// returned by \c getCurrent().  It's probably possible to ensure some
+/// sorted order, but until we can be sure it's the case for many cases
+/// in practice, we'll not rely on it.
+///
+/// A concrete object of this class is created by specific derived
+/// implementation for the corresponding data source.  The implementation
+/// must ensure the iterator is located at the "beginning" of the zone table,
+/// and that subsequent calls to \c next() go through all the zones
+/// one by one, until \c isLast() returns \c true.  The implementation must
+/// support the concept of "empty table"; in that case \c isLast() will
+/// return \c true from the beginning.
+class ZoneTableIterator : boost::noncopyable {
+protected:
+    /// \brief The constructor.
+    ///
+    /// This class is not expected to be instantiated directly, so the
+    /// constructor is hidden from normal applications as protected.
+    ZoneTableIterator() {}
+
+public:
+    /// \brief The destructor.
+    virtual ~ZoneTableIterator() {}
+
+    /// \brief Return if the iterator reaches the end of the zone table.
+    virtual bool isLast() const = 0;
+
+    /// \brief Move the iterator to the next zone of the table.
+    ///
+    /// This method must not be called once the iterator reaches the end
+    /// of the table.
+    ///
+    /// \throw InvalidOperation called after reaching the end of table.
+    void next() {
+        // Perform common check, and delegate the actual work to the protected
+        // method.
+        if (isLast()) {
+            isc_throw(InvalidOperation,
+                      "next() called on iterator beyond end of zone table");
+        }
+        nextImpl();
+    }
+
+    /// \brief Return the information of the zone at which the iterator is
+    /// currently located in the form of \c ZoneSpec.
+    ///
+    /// This method must not be called once the iterator reaches the end
+    /// of the zone table.
+    ///
+    /// \throw InvalidOperation called after reaching the end of table.
+    ///
+    /// \return Information of the "current" zone.
+    ZoneSpec getCurrent() const {
+        // Perform common check, and delegate the actual work to the protected
+        // method.
+        if (isLast()) {
+            isc_throw(InvalidOperation,
+                      "getCurrent() called on iterator beyond "
+                      "end of zone table");
+        }
+        return (getCurrentImpl());
+    }
+
+protected:
+    /// \brief Actual implementation of \c next().
+    ///
+    /// Each derived class must provide the implementation of \c next()
+    /// in its data source specific form, except for the common
+    /// validation check.
+    virtual void nextImpl() = 0;
+
+    /// \brief Actual implementation of \c getCurrent().
+    ///
+    /// Each derived class must provide the implementation of
+    /// \c getCurrent() in its data source specific form, except for the
+    /// common validation check.
+    virtual ZoneSpec getCurrentImpl() const = 0;
+};
+
+/// \brief An abstract accessor to conceptual zone table for a data source.
+///
+/// This is an abstract base class providing common interfaces to get access
+/// to a conceptual "zone table" corresponding to a specific data source.
+/// A zone table would contain a set of information about DNS zones stored in
+/// the data source.  It's "conceptual" in that the actual form of the
+/// information is specific to the data source implementation.
+///
+/// The initial version of this class only provides one simple feature:
+/// iterating over the table so that an application can get a list of
+/// all zones of a specific data source (of a specific RR class).  In
+/// future, this class will be extended so that, e.g., applications can
+/// add or remove zones.
+///
+/// \note It may make sense to move \c DataSourceClient::createZone()
+/// and \c DataSourceClient::deleteZone() to this class.
+class ZoneTableAccessor : boost::noncopyable {
+protected:
+    /// \brief The constructor.
+    ///
+    /// This class is not expected to be instantiated directly, so the
+    /// constructor is hidden from normal applications as protected.
+    ZoneTableAccessor() {}
+
+public:
+    /// \brief Shortcut type for a smart pointer of \c ZoneTableIterator
+    typedef boost::shared_ptr<ZoneTableIterator> IteratorPtr;
+
+    /// \brief The destructor.
+    virtual ~ZoneTableAccessor() {}
+
+    /// \brief Return a zone table iterator.
+    ///
+    /// In general, the specific implementation of the iterator object would
+    /// contain some form of reference to the underlying data source
+    /// (e.g., a database connection or a pointer to memory region), which
+    /// would be valid only until the object that created the instance of
+    /// the accessor is destroyed.  The iterator must not be used beyond
+    /// the lifetime of such a creator object, and normally it's expected to
+    /// be even more ephemeral: it would be created by this method in a
+    /// single method or function and only used in that limited scope.
+    ///
+    /// \throw std::bad_alloc Memory allocation for the iterator object failed.
+    /// \throw Others There will be other cases as more implementations
+    /// are added (in this initial version, it's not really decided yet).
+    ///
+    /// \return A smart pointer to a newly created iterator object.  Once
+    /// returned, the \c ZoneTableAccessor effectively releases its ownership.
+    virtual IteratorPtr getIterator() const = 0;
+};
+
+}
+}
+
+#endif  // DATASRC_ZONE_TABLE_ACCESSOR_H
+
+// Local Variables:
+// mode: c++
+// End:

src/lib/datasrc/zone_table_accessor_cache.cc

@@ -0,0 +1,60 @@
+// Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <datasrc/zone_table_accessor_cache.h>
+#include <datasrc/cache_config.h>
+
+#include <exceptions/exceptions.h>
+
+namespace isc {
+namespace datasrc {
+namespace internal {
+
+namespace {
+// This is a straightforward wrapper of CacheConfig::ConstZoneIterator to
+// implement ZoneTableIterator interfaces.
+class ZoneTableIteratorCache : public ZoneTableIterator {
+public:
+    ZoneTableIteratorCache(const CacheConfig& config) :
+        it_(config.begin()),
+        it_end_(config.end())
+    {}
+
+    virtual void nextImpl() {
+        ++it_;
+    }
+
+    virtual ZoneSpec getCurrentImpl() const {
+        return (ZoneSpec(0, it_->first)); // index is always 0 in this version.
+    }
+
+    virtual bool isLast() const {
+        return (it_ == it_end_);
+    }
+
+private:
+    CacheConfig::ConstZoneIterator it_;
+    CacheConfig::ConstZoneIterator const it_end_;
+};
+}
+
+ZoneTableAccessor::IteratorPtr
+ZoneTableAccessorCache::getIterator() const {
+    return (ZoneTableAccessor::IteratorPtr(
+                new ZoneTableIteratorCache(config_)));
+}
+
+}
+}
+}

src/lib/datasrc/zone_table_accessor_cache.h

@@ -0,0 +1,76 @@
+// Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#ifndef DATASRC_ZONE_TABLE_ACCESSOR_CACHE_H
+#define DATASRC_ZONE_TABLE_ACCESSOR_CACHE_H
+
+#include <datasrc/zone_table_accessor.h>
+
+#include <boost/noncopyable.hpp>
+
+namespace isc {
+namespace datasrc {
+namespace internal {
+class CacheConfig;
+
+/// \brief A \c ZoneTableAccessor implementation for in-memory cache.
+///
+/// This class implements the \c ZoneTableAccessor interface for in-memory
+/// cache.  Its conceptual table consists of the zones that are specified
+/// to be loaded into memory in configuration.  Note that these zones
+/// may or may not actually be loaded in memory.  In fact, this class object
+/// is intended to be used by applications that load these zones into memory,
+/// so that the application can get a list of zones to be loaded.  Also, even
+/// after loading, some zone may still not be loaded, e.g., due to an error
+/// in the corresponding zone file.
+///
+/// An object of this class is expected to be returned by
+/// \c ConfigurableClientList.  Normal applications shouldn't instantiate
+/// this class directly.  It's still defined to be publicly visible for
+/// testing purposes but, to clarify the intent, it's hidden in the
+/// "internal" namespace.
+class ZoneTableAccessorCache : public ZoneTableAccessor {
+public:
+    /// \brief Constructor.
+    ///
+    /// This class takes a \c CacheConfig object and holds it throughout
+    /// its lifetime.  The caller must ensure that the configuration is
+    /// valid throughout the lifetime of this accessor object.
+    ///
+    /// \throw None
+    ///
+    /// \param config The cache configuration that the accessor refers to.
+    ZoneTableAccessorCache(const CacheConfig& config) : config_(config) {}
+
+    /// \brief In-memory cache version of \c getIterator().
+    ///
+    /// As returned from this version of iterator, \c ZoneSpec::index
+    /// will always be set to 0 at the moment.
+    ///
+    /// \throw None except std::bad_alloc in case of memory allocation failure
+    virtual IteratorPtr getIterator() const;
+
+private:
+    const CacheConfig& config_;
+};
+
+}
+}
+}
+
+#endif  // DATASRC_ZONE_TABLE_ACCESSOR_CACHE_H
+
+// Local Variables:
+// mode: c++
+// End:

src/lib/dhcp/Makefile.am

@@ -35,6 +35,9 @@ libb10_dhcp___la_SOURCES += option_definition.cc option_definition.h
 libb10_dhcp___la_SOURCES += option_space.cc option_space.h
 libb10_dhcp___la_SOURCES += pkt6.cc pkt6.h
 libb10_dhcp___la_SOURCES += pkt4.cc pkt4.h
+libb10_dhcp___la_SOURCES += pkt_filter.h
+libb10_dhcp___la_SOURCES += pkt_filter_inet.cc pkt_filter_inet.h
+libb10_dhcp___la_SOURCES += pkt_filter_lpf.cc pkt_filter_lpf.h
 libb10_dhcp___la_SOURCES += std_option_defs.h
 
 libb10_dhcp___la_CXXFLAGS = $(AM_CXXFLAGS)

src/lib/dhcp/iface_mgr.cc

@@ -1,4 +1,4 @@
-// Copyright (C) 2011-2012  Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2011-2013  Internet Systems Consortium, Inc. ("ISC")
 //
 // Permission to use, copy, modify, and/or distribute this software for any
 // purpose with or without fee is hereby granted, provided that the above
@@ -22,6 +22,7 @@
 #include <dhcp/dhcp4.h>
 #include <dhcp/dhcp6.h>
 #include <dhcp/iface_mgr.h>
+#include <dhcp/pkt_filter_inet.h>
 #include <exceptions/exceptions.h>
 #include <util/io/pktinfo_utilities.h>
 
@@ -47,7 +48,7 @@ IfaceMgr::instance() {
     return (iface_mgr);
 }
 
-IfaceMgr::Iface::Iface(const std::string& name, int ifindex)
+Iface::Iface(const std::string& name, int ifindex)
     :name_(name), ifindex_(ifindex), mac_len_(0), hardware_type_(0),
      flag_loopback_(false), flag_up_(false), flag_running_(false),
      flag_multicast_(false), flag_broadcast_(false), flags_(0)
@@ -56,7 +57,7 @@ IfaceMgr::Iface::Iface(const std::string& name, int ifindex)
 }
 
 void
-IfaceMgr::Iface::closeSockets() {
+Iface::closeSockets() {
     for (SocketCollection::iterator sock = sockets_.begin();
          sock != sockets_.end(); ++sock) {
         close(sock->sockfd_);
@@ -65,14 +66,14 @@ IfaceMgr::Iface::closeSockets() {
 }
 
 std::string
-IfaceMgr::Iface::getFullName() const {
+Iface::getFullName() const {
     ostringstream tmp;
     tmp << name_ << "/" << ifindex_;
     return (tmp.str());
 }
 
 std::string
-IfaceMgr::Iface::getPlainMac() const {
+Iface::getPlainMac() const {
     ostringstream tmp;
     tmp.fill('0');
     tmp << hex;
@@ -86,18 +87,18 @@ IfaceMgr::Iface::getPlainMac() const {
     return (tmp.str());
 }
 
-void IfaceMgr::Iface::setMac(const uint8_t* mac, size_t len) {
-    if (len > IfaceMgr::MAX_MAC_LEN) {
+void Iface::setMac(const uint8_t* mac, size_t len) {
+    if (len > MAX_MAC_LEN) {
         isc_throw(OutOfRange, "Interface " << getFullName()
                   << " was detected to have link address of length "
                   << len << ", but maximum supported length is "
-                  << IfaceMgr::MAX_MAC_LEN);
+                  << MAX_MAC_LEN);
     }
     mac_len_ = len;
     memcpy(mac_, mac, len);
 }
 
-bool IfaceMgr::Iface::delAddress(const isc::asiolink::IOAddress& addr) {
+bool Iface::delAddress(const isc::asiolink::IOAddress& addr) {
     for (AddressCollection::iterator a = addrs_.begin();
          a!=addrs_.end(); ++a) {
         if (*a==addr) {
@@ -108,7 +109,7 @@ bool IfaceMgr::Iface::delAddress(const isc::asiolink::IOAddress& addr) {
     return (false);
 }
 
-bool IfaceMgr::Iface::delSocket(uint16_t sockfd) {
+bool Iface::delSocket(uint16_t sockfd) {
     list<SocketInfo>::iterator sock = sockets_.begin();
     while (sock!=sockets_.end()) {
         if (sock->sockfd_ == sockfd) {
@@ -124,7 +125,8 @@ bool IfaceMgr::Iface::delSocket(uint16_t sockfd) {
 IfaceMgr::IfaceMgr()
     :control_buf_len_(CMSG_SPACE(sizeof(struct in6_pktinfo))),
      control_buf_(new char[control_buf_len_]),
-     session_socket_(INVALID_SOCKET), session_callback_(NULL)
+     session_socket_(INVALID_SOCKET), session_callback_(NULL),
+     packet_filter_(new PktFilterInet())
 {
 
     try {
@@ -193,10 +195,23 @@ void IfaceMgr::stubDetectIfaces() {
     addInterface(iface);
 }
 
-bool IfaceMgr::openSockets4(const uint16_t port) {
+bool IfaceMgr::openSockets4(const uint16_t port, const bool use_bcast) {
     int sock;
     int count = 0;
 
+// This option is used to bind sockets to particular interfaces.
+// This is currently the only way to discover on which interface
+// the broadcast packet has been received. If this option is
+// not supported then only one interface should be confugured
+// to listen for broadcast traffic.
+#ifdef SO_BINDTODEVICE
+    const bool bind_to_device = true;
+#else
+    const bool bind_to_device = false;
+#endif
+
+    int bcast_num = 0;
+
     for (IfaceCollection::iterator iface = ifaces_.begin();
          iface != ifaces_.end();
          ++iface) {
@@ -207,8 +222,8 @@ bool IfaceMgr::openSockets4(const uint16_t port) {
             continue;
         }
 
-        AddressCollection addrs = iface->getAddresses();
-        for (AddressCollection::iterator addr = addrs.begin();
+        Iface::AddressCollection addrs = iface->getAddresses();
+        for (Iface::AddressCollection::iterator addr = addrs.begin();
              addr != addrs.end();
              ++addr) {
 
@@ -217,9 +232,40 @@ bool IfaceMgr::openSockets4(const uint16_t port) {
                 continue;
             }
 
-            sock = openSocket(iface->getName(), *addr, port);
+            // If selected interface is broadcast capable set appropriate
+            // options on the socket so as it can receive and send broadcast
+            // messages.
+            if (iface->flag_broadcast_ && use_bcast) {
+                // If our OS supports binding socket to a device we can listen
+                // for broadcast messages on multiple interfaces. Otherwise we
+                // bind to INADDR_ANY address but we can do it only once. Thus,
+                // if one socket has been bound we can't do it any further.
+                if (!bind_to_device && bcast_num > 0) {
+                    isc_throw(SocketConfigError, "SO_BINDTODEVICE socket option is"
+                              << " not supported on this OS; therefore, DHCP"
+                              << " server can only listen broadcast traffic on"
+                              << " a single interface");
+
+                } else {
+                    // We haven't open any broadcast sockets yet, so we can
+                    // open at least one more.
+                    sock = openSocket(iface->getName(), *addr, port, true, true);
+                    // Binding socket to an interface is not supported so we can't
+                    // open any more broadcast sockets. Increase the number of
+                    // opened broadcast sockets.
+                    if (!bind_to_device) {
+                        ++bcast_num;
+                    }
+                }
+
+            } else {
+                // Not broadcast capable, do not set broadcast flags.
+                sock = openSocket(iface->getName(), *addr, port, false, false);
+
+            }
             if (sock < 0) {
-                isc_throw(SocketConfigError, "failed to open unicast socket");
+                isc_throw(SocketConfigError, "failed to open IPv4 socket"
+                          << " supporting broadcast traffic");
             }
 
             count++;
@@ -242,8 +288,8 @@ bool IfaceMgr::openSockets6(const uint16_t port) {
             continue;
         }
 
-        AddressCollection addrs = iface->getAddresses();
-        for (AddressCollection::iterator addr = addrs.begin();
+        Iface::AddressCollection addrs = iface->getAddresses();
+        for (Iface::AddressCollection::iterator addr = addrs.begin();
              addr != addrs.end();
              ++addr) {
 
@@ -304,7 +350,7 @@ IfaceMgr::printIfaces(std::ostream& out /*= std::cout*/) {
          iface!=ifaces_.end();
          ++iface) {
 
-        const AddressCollection& addrs = iface->getAddresses();
+        const Iface::AddressCollection& addrs = iface->getAddresses();
 
         out << "Detected interface " << iface->getFullName()
             << ", hwtype=" << iface->getHWType()
@@ -318,7 +364,7 @@ IfaceMgr::printIfaces(std::ostream& out /*= std::cout*/) {
             << ")" << endl;
         out << "  " << addrs.size() << " addr(s):";
 
-        for (AddressCollection::const_iterator addr = addrs.begin();
+        for (Iface::AddressCollection::const_iterator addr = addrs.begin();
              addr != addrs.end(); ++addr) {
             out << "  " << addr->toText();
         }
@@ -326,7 +372,7 @@ IfaceMgr::printIfaces(std::ostream& out /*= std::cout*/) {
     }
 }
 
-IfaceMgr::Iface*
+Iface*
 IfaceMgr::getIface(int ifindex) {
     for (IfaceCollection::iterator iface=ifaces_.begin();
          iface!=ifaces_.end();
@@ -338,7 +384,7 @@ IfaceMgr::getIface(int ifindex) {
     return (NULL); // not found
 }
 
-IfaceMgr::Iface*
+Iface*
 IfaceMgr::getIface(const std::string& ifname) {
     for (IfaceCollection::iterator iface=ifaces_.begin();
          iface!=ifaces_.end();
@@ -351,13 +397,14 @@ IfaceMgr::getIface(const std::string& ifname) {
 }
 
 int IfaceMgr::openSocket(const std::string& ifname, const IOAddress& addr,
-                         const uint16_t port) {
+                         const uint16_t port, const bool receive_bcast,
+                         const bool send_bcast) {
     Iface* iface = getIface(ifname);
     if (!iface) {
         isc_throw(BadValue, "There is no " << ifname << " interface present.");
     }
     if (addr.isV4()) {
-        return openSocket4(*iface, addr, port);
+        return openSocket4(*iface, addr, port, receive_bcast, send_bcast);
 
     } else if (addr.isV6()) {
         return openSocket6(*iface, addr, port);
@@ -383,8 +430,8 @@ int IfaceMgr::openSocketFromIface(const std::string& ifname,
 
         // Interface is now detected. Search for address on interface
         // that matches address family (v6 or v4).
-        AddressCollection addrs = iface->getAddresses();
-        AddressCollection::iterator addr_it = addrs.begin();
+        Iface::AddressCollection addrs = iface->getAddresses();
+        Iface::AddressCollection::iterator addr_it = addrs.begin();
         while (addr_it != addrs.end()) {
             if (addr_it->getFamily() == family) {
                 // We have interface and address so let's open socket.
@@ -420,9 +467,9 @@ int IfaceMgr::openSocketFromAddress(const IOAddress& addr,
          iface != ifaces_.end();
          ++iface) {
 
-        AddressCollection addrs = iface->getAddresses();
+        Iface::AddressCollection addrs = iface->getAddresses();
 
-        for (AddressCollection::iterator addr_it = addrs.begin();
+        for (Iface::AddressCollection::iterator addr_it = addrs.begin();
              addr_it != addrs.end();
              ++addr_it) {
 
@@ -509,43 +556,6 @@ IfaceMgr::getLocalAddress(const IOAddress& remote_addr, const uint16_t port) {
     return IOAddress(local_address);
 }
 
-int IfaceMgr::openSocket4(Iface& iface, const IOAddress& addr, uint16_t port) {
-
-    struct sockaddr_in addr4;
-    memset(&addr4, 0, sizeof(sockaddr));
-    addr4.sin_family = AF_INET;
-    addr4.sin_port = htons(port);
-
-    addr4.sin_addr.s_addr = htonl(addr);
-    //addr4.sin_addr.s_addr = 0; // anyaddr: this will receive 0.0.0.0 => 255.255.255.255 traffic
-    // addr4.sin_addr.s_addr = 0xffffffffu; // broadcast address. This will receive 0.0.0.0 => 255.255.255.255 as well
-
-    int sock = socket(AF_INET, SOCK_DGRAM, 0);
-    if (sock < 0) {
-        isc_throw(SocketConfigError, "Failed to create UDP6 socket.");
-    }
-
-    if (bind(sock, (struct sockaddr *)&addr4, sizeof(addr4)) < 0) {
-        close(sock);
-        isc_throw(SocketConfigError, "Failed to bind socket " << sock << " to " << addr.toText()
-                  << "/port=" << port);
-    }
-
-    // if there is no support for IP_PKTINFO, we are really out of luck
-    // it will be difficult to undersand, where this packet came from
-#if defined(IP_PKTINFO)
-    int flag = 1;
-    if (setsockopt(sock, IPPROTO_IP, IP_PKTINFO, &flag, sizeof(flag)) != 0) {
-        close(sock);
-        isc_throw(SocketConfigError, "setsockopt: IP_PKTINFO: failed.");
-    }
-#endif
-
-    SocketInfo info(sock, addr, port);
-    iface.addSocket(info);
-
-    return (sock);
-}
 
 int IfaceMgr::openSocket6(Iface& iface, const IOAddress& addr, uint16_t port) {
 
@@ -620,6 +630,22 @@ int IfaceMgr::openSocket6(Iface& iface, const IOAddress& addr, uint16_t port) {
     return (sock);
 }
 
+int IfaceMgr::openSocket4(Iface& iface, const IOAddress& addr,
+                          const uint16_t port, const bool receive_bcast,
+                          const bool send_bcast) {
+
+    // Skip checking if the packet_filter_ is non-NULL because this check
+    // has been already done when packet filter object was set.
+
+    int sock = packet_filter_->openSocket(iface, addr, port,
+                                          receive_bcast, send_bcast);
+
+    SocketInfo info(sock, addr, port);
+    iface.addSocket(info);
+
+    return (sock);
+}
+
 bool
 IfaceMgr::joinMulticast(int sock, const std::string& ifname,
 const std::string & mcast) {
@@ -722,53 +748,17 @@ IfaceMgr::send(const Pkt6Ptr& pkt) {
 }
 
 bool
-IfaceMgr::send(const Pkt4Ptr& pkt)
-{
+IfaceMgr::send(const Pkt4Ptr& pkt) {
+
     Iface* iface = getIface(pkt->getIface());
     if (!iface) {
         isc_throw(BadValue, "Unable to send Pkt4. Invalid interface ("
                   << pkt->getIface() << ") specified.");
     }
 
-    memset(&control_buf_[0], 0, control_buf_len_);
-
-
-    // Set the target address we're sending to.
-    sockaddr_in to;
-    memset(&to, 0, sizeof(to));
-    to.sin_family = AF_INET;
-    to.sin_port = htons(pkt->getRemotePort());
-    to.sin_addr.s_addr = htonl(pkt->getRemoteAddr());
-
-    struct msghdr m;
-    // Initialize our message header structure.
-    memset(&m, 0, sizeof(m));
-    m.msg_name = &to;
-    m.msg_namelen = sizeof(to);
-
-    // Set the data buffer we're sending. (Using this wacky
-    // "scatter-gather" stuff... we only have a single chunk
-    // of data to send, so we declare a single vector entry.)
-    struct iovec v;
-    memset(&v, 0, sizeof(v));
-    // iov_base field is of void * type. We use it for packet
-    // transmission, so this buffer will not be modified.
-    v.iov_base = const_cast<void *>(pkt->getBuffer().getData());
-    v.iov_len = pkt->getBuffer().getLength();
-    m.msg_iov = &v;
-    m.msg_iovlen = 1;
-
-    // call OS-specific routines (like setting interface index)
-    os_send4(m, control_buf_, control_buf_len_, pkt);
-
-    pkt->updateTimestamp();
-
-    int result = sendmsg(getSocket(*pkt), &m, 0);
-    if (result < 0) {
-        isc_throw(SocketWriteError, "pkt4 send failed");
-    }
-
-    return (result);
+    // Skip checking if packet filter is non-NULL because it has been
+    // already checked when packet filter was set.
+    return (packet_filter_->send(getSocket(*pkt), pkt));
 }
 
 
@@ -792,8 +782,8 @@ IfaceMgr::receive4(uint32_t timeout_sec, uint32_t timeout_usec /* = 0 */) {
     /// provided set to indicated which sockets have something to read.
     for (iface = ifaces_.begin(); iface != ifaces_.end(); ++iface) {
 
-        const SocketCollection& socket_collection = iface->getSockets();
-        for (SocketCollection::const_iterator s = socket_collection.begin();
+        const Iface::SocketCollection& socket_collection = iface->getSockets();
+        for (Iface::SocketCollection::const_iterator s = socket_collection.begin();
              s != socket_collection.end(); ++s) {
 
             // Only deal with IPv4 addresses.
@@ -848,8 +838,8 @@ IfaceMgr::receive4(uint32_t timeout_sec, uint32_t timeout_usec /* = 0 */) {
 
     // Let's find out which interface/socket has the data
     for (iface = ifaces_.begin(); iface != ifaces_.end(); ++iface) {
-        const SocketCollection& socket_collection = iface->getSockets();
-        for (SocketCollection::const_iterator s = socket_collection.begin();
+        const Iface::SocketCollection& socket_collection = iface->getSockets();
+        for (Iface::SocketCollection::const_iterator s = socket_collection.begin();
              s != socket_collection.end(); ++s) {
             if (FD_ISSET(s->sockfd_, &sockets)) {
                 candidate = &(*s);
@@ -866,64 +856,9 @@ IfaceMgr::receive4(uint32_t timeout_sec, uint32_t timeout_usec /* = 0 */) {
     }
 
     // Now we have a socket, let's get some data from it!
-    struct sockaddr_in from_addr;
-    uint8_t buf[RCVBUFSIZE];
-
-    memset(&control_buf_[0], 0, control_buf_len_);
-    memset(&from_addr, 0, sizeof(from_addr));
-
-    // Initialize our message header structure.
-    struct msghdr m;
-    memset(&m, 0, sizeof(m));
-
-    // Point so we can get the from address.
-    m.msg_name = &from_addr;
-    m.msg_namelen = sizeof(from_addr);
-
-    struct iovec v;
-    v.iov_base = static_cast<void*>(buf);
-    v.iov_len = RCVBUFSIZE;
-    m.msg_iov = &v;
-    m.msg_iovlen = 1;
-
-    // Getting the interface is a bit more involved.
-    //
-    // We set up some space for a "control message". We have
-    // previously asked the kernel to give us packet
-    // information (when we initialized the interface), so we
-    // should get the destination address from that.
-    m.msg_control = &control_buf_[0];
-    m.msg_controllen = control_buf_len_;
-
-    result = recvmsg(candidate->sockfd_, &m, 0);
-    if (result < 0) {
-        isc_throw(SocketReadError, "failed to receive UDP4 data");
-    }
-
-    // We have all data let's create Pkt4 object.
-    Pkt4Ptr pkt = Pkt4Ptr(new Pkt4(buf, result));
-
-    pkt->updateTimestamp();
-
-    unsigned int ifindex = iface->getIndex();
-
-    IOAddress from(htonl(from_addr.sin_addr.s_addr));
-    uint16_t from_port = htons(from_addr.sin_port);
-
-    // Set receiving interface based on information, which socket was used to
-    // receive data. OS-specific info (see os_receive4()) may be more reliable,
-    // so this value may be overwritten.
-    pkt->setIndex(ifindex);
-    pkt->setIface(iface->getName());
-    pkt->setRemoteAddr(from);
-    pkt->setRemotePort(from_port);
-    pkt->setLocalPort(candidate->port_);
-
-    if (!os_receive4(m, pkt)) {
-        isc_throw(SocketReadError, "unable to find pktinfo");
-    }
-
-    return (pkt);
+    // Skip checking if packet filter is non-NULL because it has been
+    // already checked when packet filter was set.
+    return (packet_filter_->receive(*iface, *candidate));
 }
 
 Pkt6Ptr IfaceMgr::receive6(uint32_t timeout_sec, uint32_t timeout_usec /* = 0 */ ) {
@@ -945,8 +880,8 @@ Pkt6Ptr IfaceMgr::receive6(uint32_t timeout_sec, uint32_t timeout_usec /* = 0 */
     /// provided set to indicated which sockets have something to read.
     IfaceCollection::const_iterator iface;
     for (iface = ifaces_.begin(); iface != ifaces_.end(); ++iface) {
-        const SocketCollection& socket_collection = iface->getSockets();
-        for (SocketCollection::const_iterator s = socket_collection.begin();
+        const Iface::SocketCollection& socket_collection = iface->getSockets();
+        for (Iface::SocketCollection::const_iterator s = socket_collection.begin();
              s != socket_collection.end(); ++s) {
 
             // Only deal with IPv6 addresses.
@@ -1001,8 +936,8 @@ Pkt6Ptr IfaceMgr::receive6(uint32_t timeout_sec, uint32_t timeout_usec /* = 0 */
 
     // Let's find out which interface/socket has the data
     for (iface = ifaces_.begin(); iface != ifaces_.end(); ++iface) {
-        const SocketCollection& socket_collection = iface->getSockets();
-        for (SocketCollection::const_iterator s = socket_collection.begin();
+        const Iface::SocketCollection& socket_collection = iface->getSockets();
+        for (Iface::SocketCollection::const_iterator s = socket_collection.begin();
              s != socket_collection.end(); ++s) {
             if (FD_ISSET(s->sockfd_, &sockets)) {
                 candidate = &(*s);
@@ -1122,8 +1057,8 @@ uint16_t IfaceMgr::getSocket(const isc::dhcp::Pkt6& pkt) {
                   << pkt.getIface());
     }
 
-    const SocketCollection& socket_collection = iface->getSockets();
-    SocketCollection::const_iterator s;
+    const Iface::SocketCollection& socket_collection = iface->getSockets();
+    Iface::SocketCollection::const_iterator s;
     for (s = socket_collection.begin(); s != socket_collection.end(); ++s) {
         if ((s->family_ == AF_INET6) &&
             (!s->addr_.getAddress().to_v6().is_multicast())) {
@@ -1145,8 +1080,8 @@ uint16_t IfaceMgr::getSocket(isc::dhcp::Pkt4 const& pkt) {
                   << pkt.getIface());
     }
 
-    const SocketCollection& socket_collection = iface->getSockets();
-    SocketCollection::const_iterator s;
+    const Iface::SocketCollection& socket_collection = iface->getSockets();
+    Iface::SocketCollection::const_iterator s;
     for (s = socket_collection.begin(); s != socket_collection.end(); ++s) {
         if (s->family_ == AF_INET) {
             return (s->sockfd_);

src/lib/dhcp/iface_mgr.h

@@ -1,4 +1,4 @@
-// Copyright (C) 2011-2012  Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2011-2013  Internet Systems Consortium, Inc. ("ISC")
 //
 // Permission to use, copy, modify, and/or distribute this software for any
 // purpose with or without fee is hereby granted, provided that the above
@@ -20,6 +20,7 @@
 #include <dhcp/dhcp6.h>
 #include <dhcp/pkt4.h>
 #include <dhcp/pkt6.h>
+#include <dhcp/pkt_filter.h>
 
 #include <boost/noncopyable.hpp>
 #include <boost/scoped_array.hpp>
@@ -38,6 +39,13 @@ public:
         isc::Exception(file, line, what) { };
 };
 
+/// @brief IfaceMgr exception thrown when invalid packet filter object specified.
+class InvalidPacketFilter : public Exception {
+public:
+    InvalidPacketFilter(const char* file, size_t line, const char* what) :
+        isc::Exception(file, line, what) { };
+};
+
 /// @brief IfaceMgr exception thrown thrown when socket opening
 /// or configuration failed.
 class SocketConfigError : public Exception {
@@ -62,6 +70,219 @@ public:
         isc::Exception(file, line, what) { };
 };
 
+/// Holds information about socket.
+struct SocketInfo {
+    uint16_t sockfd_; /// socket descriptor
+    isc::asiolink::IOAddress addr_; /// bound address
+    uint16_t port_;   /// socket port
+    uint16_t family_; /// IPv4 or IPv6
+
+    /// @brief SocketInfo constructor.
+    ///
+    /// @param sockfd socket descriptor
+    /// @param addr an address the socket is bound to
+    /// @param port a port the socket is bound to
+    SocketInfo(uint16_t sockfd, const isc::asiolink::IOAddress& addr,
+               uint16_t port)
+        :sockfd_(sockfd), addr_(addr), port_(port), family_(addr.getFamily()) { }
+};
+
+
+/// @brief represents a single network interface
+///
+/// Iface structure represents network interface with all useful
+/// information, like name, interface index, MAC address and
+/// list of assigned addresses
+class Iface {
+public:
+
+    /// maximum MAC address length (Infiniband uses 20 bytes)
+    static const unsigned int MAX_MAC_LEN = 20;
+
+    /// type that defines list of addresses
+    typedef std::vector<isc::asiolink::IOAddress> AddressCollection;
+
+    /// type that holds a list of socket informations
+    /// @todo: Add SocketCollectionConstIter type
+    typedef std::list<SocketInfo> SocketCollection;
+
+    /// @brief Iface constructor.
+    ///
+    /// Creates Iface object that represents network interface.
+    ///
+    /// @param name name of the interface
+    /// @param ifindex interface index (unique integer identifier)
+    Iface(const std::string& name, int ifindex);
+
+    /// @brief Closes all open sockets on interface.
+    void closeSockets();
+
+    /// @brief Returns full interface name as "ifname/ifindex" string.
+    ///
+    /// @return string with interface name
+    std::string getFullName() const;
+
+    /// @brief Returns link-layer address a plain text.
+    ///
+    /// @return MAC address as a plain text (string)
+    std::string getPlainMac() const;
+
+    /// @brief Sets MAC address of the interface.
+    ///
+    /// @param mac pointer to MAC address buffer
+    /// @param macLen length of mac address
+    void setMac(const uint8_t* mac, size_t macLen);
+
+    /// @brief Returns MAC length.
+    ///
+    /// @return length of MAC address
+    size_t getMacLen() const { return mac_len_; }
+
+    /// @brief Returns pointer to MAC address.
+    ///
+    /// Note: Returned pointer is only valid as long as the interface object
+    /// that returned it.
+    const uint8_t* getMac() const { return mac_; }
+
+    /// @brief Sets flag_*_ fields based on bitmask value returned by OS
+    ///
+    /// Note: Implementation of this method is OS-dependent as bits have
+    /// different meaning on each OS.
+    ///
+    /// @param flags bitmask value returned by OS in interface detection
+    void setFlags(uint32_t flags);
+
+    /// @brief Returns interface index.
+    ///
+    /// @return interface index
+    uint16_t getIndex() const { return ifindex_; }
+
+    /// @brief Returns interface name.
+    ///
+    /// @return interface name
+    std::string getName() const { return name_; };
+
+    /// @brief Sets up hardware type of the interface.
+    ///
+    /// @param type hardware type
+    void setHWType(uint16_t type ) { hardware_type_ = type; }
+
+    /// @brief Returns hardware type of the interface.
+    ///
+    /// @return hardware type
+    uint16_t getHWType() const { return hardware_type_; }
+
+    /// @brief Returns all interfaces available on an interface.
+    ///
+    /// Care should be taken to not use this collection after Iface object
+    /// ceases to exist. That is easy in most cases as Iface objects are
+    /// created by IfaceMgr that is a singleton an is expected to be
+    /// available at all time. We may revisit this if we ever decide to
+    /// implement dynamic interface detection, but such fancy feature would
+    /// mostly be useful for clients with wifi/vpn/virtual interfaces.
+    ///
+    /// @return collection of addresses
+    const AddressCollection& getAddresses() const { return addrs_; }
+
+    /// @brief Adds an address to an interface.
+    ///
+    /// This only adds an address to collection, it does not physically
+    /// configure address on actual network interface.
+    ///
+    /// @param addr address to be added
+    void addAddress(const isc::asiolink::IOAddress& addr) {
+        addrs_.push_back(addr);
+    }
+
+    /// @brief Deletes an address from an interface.
+    ///
+    /// This only deletes address from collection, it does not physically
+    /// remove address configuration from actual network interface.
+    ///
+    /// @param addr address to be removed.
+    ///
+    /// @return true if removal was successful (address was in collection),
+    ///         false otherwise
+    bool delAddress(const isc::asiolink::IOAddress& addr);
+
+    /// @brief Adds socket descriptor to an interface.
+    ///
+    /// @param sock SocketInfo structure that describes socket.
+    void addSocket(const SocketInfo& sock) {
+        sockets_.push_back(sock);
+    }
+
+    /// @brief Closes socket.
+    ///
+    /// Closes socket and removes corresponding SocketInfo structure
+    /// from an interface.
+    ///
+    /// @param sockfd socket descriptor to be closed/removed.
+    /// @return true if there was such socket, false otherwise
+    bool delSocket(uint16_t sockfd);
+
+    /// @brief Returns collection of all sockets added to interface.
+    ///
+    /// When new socket is created with @ref IfaceMgr::openSocket
+    /// it is added to sockets collection on particular interface.
+    /// If socket is opened by other means (e.g. function that does
+    /// not use @ref IfaceMgr::openSocket) it will not be available
+    /// in this collection. Note that functions like
+    /// @ref IfaceMgr::openSocketFromIface use
+    /// @ref IfaceMgr::openSocket internally.
+    /// The returned reference is only valid during the lifetime of
+    /// the IfaceMgr object that returned it.
+    ///
+    /// @return collection of sockets added to interface
+    const SocketCollection& getSockets() const { return sockets_; }
+
+protected:
+    /// socket used to sending data
+    SocketCollection sockets_;
+
+    /// network interface name
+    std::string name_;
+
+    /// interface index (a value that uniquely indentifies an interface)
+    int ifindex_;
+
+    /// list of assigned addresses
+    AddressCollection addrs_;
+
+    /// link-layer address
+    uint8_t mac_[MAX_MAC_LEN];
+
+    /// length of link-layer address (usually 6)
+    size_t mac_len_;
+
+    /// hardware type
+    uint16_t hardware_type_;
+
+public:
+    /// @todo: Make those fields protected once we start supporting more
+    /// than just Linux
+
+    /// specifies if selected interface is loopback
+    bool flag_loopback_;
+
+    /// specifies if selected interface is up
+    bool flag_up_;
+
+    /// flag specifies if selected interface is running
+    /// (e.g. cable plugged in, wifi associated)
+    bool flag_running_;
+
+    /// flag specifies if selected interface is multicast capable
+    bool flag_multicast_;
+
+    /// flag specifies if selected interface is broadcast capable
+    bool flag_broadcast_;
+
+    /// interface flags (this value is as is returned by OS,
+    /// it may mean different things on different OSes)
+    uint32_t flags_;
+};
+
 /// @brief handles network interfaces, transmission and reception
 ///
 /// IfaceMgr is an interface manager class that detects available network
@@ -70,15 +291,9 @@ public:
 ///
 class IfaceMgr : public boost::noncopyable {
 public:
-    /// type that defines list of addresses
-    typedef std::vector<isc::asiolink::IOAddress> AddressCollection;
-
     /// defines callback used when commands are received over control session
     typedef void (*SessionCallback) (void);
 
-    /// maximum MAC address length (Infiniband uses 20 bytes)
-    static const unsigned int MAX_MAC_LEN = 20;
-
     /// @brief Packet reception buffer size
     ///
     /// RFC3315 states that server responses may be
@@ -88,211 +303,6 @@ public:
     /// we don't support packets larger than 1500.
     static const uint32_t RCVBUFSIZE = 1500;
 
-    /// Holds information about socket.
-    struct SocketInfo {
-        uint16_t sockfd_; /// socket descriptor
-        isc::asiolink::IOAddress addr_; /// bound address
-        uint16_t port_;   /// socket port
-        uint16_t family_; /// IPv4 or IPv6
-
-        /// @brief SocketInfo constructor.
-        ///
-        /// @param sockfd socket descriptor
-        /// @param addr an address the socket is bound to
-        /// @param port a port the socket is bound to
-        SocketInfo(uint16_t sockfd, const isc::asiolink::IOAddress& addr,
-                   uint16_t port)
-        :sockfd_(sockfd), addr_(addr), port_(port), family_(addr.getFamily()) { }
-    };
-
-    /// type that holds a list of socket informations
-    /// @todo: Add SocketCollectionConstIter type
-    typedef std::list<SocketInfo> SocketCollection;
-
-
-    /// @brief represents a single network interface
-    ///
-    /// Iface structure represents network interface with all useful
-    /// information, like name, interface index, MAC address and
-    /// list of assigned addresses
-    class Iface {
-    public:
-        /// @brief Iface constructor.
-        ///
-        /// Creates Iface object that represents network interface.
-        ///
-        /// @param name name of the interface
-        /// @param ifindex interface index (unique integer identifier)
-        Iface(const std::string& name, int ifindex);
-
-        /// @brief Closes all open sockets on interface.
-        void closeSockets();
-
-        /// @brief Returns full interface name as "ifname/ifindex" string.
-        ///
-        /// @return string with interface name
-        std::string getFullName() const;
-
-        /// @brief Returns link-layer address a plain text.
-        ///
-        /// @return MAC address as a plain text (string)
-        std::string getPlainMac() const;
-
-        /// @brief Sets MAC address of the interface.
-        ///
-        /// @param mac pointer to MAC address buffer
-        /// @param macLen length of mac address
-        void setMac(const uint8_t* mac, size_t macLen);
-
-        /// @brief Returns MAC length.
-        ///
-        /// @return length of MAC address
-        size_t getMacLen() const { return mac_len_; }
-
-        /// @brief Returns pointer to MAC address.
-        ///
-        /// Note: Returned pointer is only valid as long as the interface object
-        /// that returned it.
-        const uint8_t* getMac() const { return mac_; }
-
-        /// @brief Sets flag_*_ fields based on bitmask value returned by OS
-        ///
-        /// Note: Implementation of this method is OS-dependent as bits have
-        /// different meaning on each OS.
-        ///
-        /// @param flags bitmask value returned by OS in interface detection
-        void setFlags(uint32_t flags);
-
-        /// @brief Returns interface index.
-        ///
-        /// @return interface index
-        uint16_t getIndex() const { return ifindex_; }
-
-        /// @brief Returns interface name.
-        ///
-        /// @return interface name
-        std::string getName() const { return name_; };
-
-        /// @brief Sets up hardware type of the interface.
-        ///
-        /// @param type hardware type
-        void setHWType(uint16_t type ) { hardware_type_ = type; }
-
-        /// @brief Returns hardware type of the interface.
-        ///
-        /// @return hardware type
-        uint16_t getHWType() const { return hardware_type_; }
-
-        /// @brief Returns all interfaces available on an interface.
-        ///
-        /// Care should be taken to not use this collection after Iface object
-        /// ceases to exist. That is easy in most cases as Iface objects are
-        /// created by IfaceMgr that is a singleton an is expected to be
-        /// available at all time. We may revisit this if we ever decide to
-        /// implement dynamic interface detection, but such fancy feature would
-        /// mostly be useful for clients with wifi/vpn/virtual interfaces.
-        ///
-        /// @return collection of addresses
-        const AddressCollection& getAddresses() const { return addrs_; }
-
-        /// @brief Adds an address to an interface.
-        ///
-        /// This only adds an address to collection, it does not physically
-        /// configure address on actual network interface.
-        ///
-        /// @param addr address to be added
-        void addAddress(const isc::asiolink::IOAddress& addr) {
-            addrs_.push_back(addr);
-        }
-
-        /// @brief Deletes an address from an interface.
-        ///
-        /// This only deletes address from collection, it does not physically
-        /// remove address configuration from actual network interface.
-        ///
-        /// @param addr address to be removed.
-        ///
-        /// @return true if removal was successful (address was in collection),
-        ///         false otherwise
-        bool delAddress(const isc::asiolink::IOAddress& addr);
-
-        /// @brief Adds socket descriptor to an interface.
-        ///
-        /// @param sock SocketInfo structure that describes socket.
-        void addSocket(const SocketInfo& sock)
-            { sockets_.push_back(sock); }
-
-        /// @brief Closes socket.
-        ///
-        /// Closes socket and removes corresponding SocketInfo structure
-        /// from an interface.
-        ///
-        /// @param sockfd socket descriptor to be closed/removed.
-        /// @return true if there was such socket, false otherwise
-        bool delSocket(uint16_t sockfd);
-
-        /// @brief Returns collection of all sockets added to interface.
-        ///
-        /// When new socket is created with @ref IfaceMgr::openSocket
-        /// it is added to sockets collection on particular interface.
-        /// If socket is opened by other means (e.g. function that does
-        /// not use @ref IfaceMgr::openSocket) it will not be available
-        /// in this collection. Note that functions like
-        /// @ref IfaceMgr::openSocketFromIface use
-        /// @ref IfaceMgr::openSocket internally.
-        /// The returned reference is only valid during the lifetime of
-        /// the IfaceMgr object that returned it.
-        ///
-        /// @return collection of sockets added to interface
-        const SocketCollection& getSockets() const { return sockets_; }
-
-    protected:
-        /// socket used to sending data
-        SocketCollection sockets_;
-
-        /// network interface name
-        std::string name_;
-
-        /// interface index (a value that uniquely identifies an interface)
-        int ifindex_;
-
-        /// list of assigned addresses
-        AddressCollection addrs_;
-
-        /// link-layer address
-        uint8_t mac_[MAX_MAC_LEN];
-
-        /// length of link-layer address (usually 6)
-        size_t mac_len_;
-
-        /// hardware type
-        uint16_t hardware_type_;
-
-    public:
-        /// @todo: Make those fields protected once we start supporting more
-        /// than just Linux
-
-        /// specifies if selected interface is loopback
-        bool flag_loopback_;
-
-        /// specifies if selected interface is up
-        bool flag_up_;
-
-        /// flag specifies if selected interface is running
-        /// (e.g. cable plugged in, wifi associated)
-        bool flag_running_;
-
-        /// flag specifies if selected interface is multicast capable
-        bool flag_multicast_;
-
-        /// flag specifies if selected interface is broadcast capable
-        bool flag_broadcast_;
-
-        /// interface flags (this value is as is returned by OS,
-        /// it may mean different things on different OSes)
-        uint32_t flags_;
-    };
-
     // TODO performance improvement: we may change this into
     //      2 maps (ifindex-indexed and name-indexed) and
     //      also hide it (make it public make tests easier for now)
@@ -306,6 +316,16 @@ public:
     /// @return the only existing instance of interface manager
     static IfaceMgr& instance();
 
+    /// @brief Check if packet be sent directly to the client having no address.
+    ///
+    /// Checks if IfaceMgr can send DHCPv4 packet to the client
+    /// who hasn't got address assigned. If this is not supported
+    /// broadcast address should be used to send response to
+    /// the client.
+    ///
+    /// @return true if direct response is supported.
+    bool isDirectResponseSupported();
+
     /// @brief Returns interface with specified interface index
     ///
     /// @param ifindex index of searched interface
@@ -434,6 +454,10 @@ public:
     /// @param ifname name of the interface
     /// @param addr address to be bound.
     /// @param port UDP port.
+    /// @param receive_bcast configure IPv4 socket to receive broadcast messages.
+    /// This parameter is ignored for IPv6 sockets.
+    /// @param send_bcast configure IPv4 socket to send broadcast messages.
+    /// This parameter is ignored for IPv6 sockets.
     ///
     /// Method will throw if socket creation, socket binding or multicast
     /// join fails.
@@ -442,7 +466,9 @@ public:
     /// group join were all successful.
     int openSocket(const std::string& ifname,
                    const isc::asiolink::IOAddress& addr,
-                   const uint16_t port);
+                   const uint16_t port,
+                   const bool receive_bcast = false,
+                   const bool send_bcast = false);
 
     /// @brief Opens UDP/IP socket and binds it to interface specified.
     ///
@@ -504,18 +530,20 @@ public:
     /// @return true if any sockets were open
     bool openSockets6(const uint16_t port = DHCP6_SERVER_PORT);
 
-    /// @brief Closes all open sockets.
-    /// Is used in destructor, but also from Dhcpv4_srv and Dhcpv6_srv classes.
-    void closeSockets();
-
     /// Opens IPv4 sockets on detected interfaces.
     /// Will throw exception if socket creation fails.
     ///
     /// @param port specifies port number (usually DHCP4_SERVER_PORT)
+    /// @param use_bcast configure sockets to support broadcast messages.
     ///
     /// @throw SocketOpenFailure if tried and failed to open socket.
     /// @return true if any sockets were open
-    bool openSockets4(const uint16_t port = DHCP4_SERVER_PORT);
+    bool openSockets4(const uint16_t port = DHCP4_SERVER_PORT,
+                      const bool use_bcast = true);
+
+    /// @brief Closes all open sockets.
+    /// Is used in destructor, but also from Dhcpv4_srv and Dhcpv6_srv classes.
+    void closeSockets();
 
     /// @brief returns number of detected interfaces
     ///
@@ -534,6 +562,24 @@ public:
         session_callback_ = callback;
     }
 
+    /// @brief Set Packet Filter object to handle send/receive packets.
+    ///
+    /// Packet Filters expose low-level functions handling sockets opening
+    /// and sending/receiving packets through those sockets. This function
+    /// sets custom Packet Filter (represented by a class derived from PktFilter)
+    /// to be used by IfaceMgr.
+    ///
+    /// @param packet_filter new packet filter to be used by IfaceMgr to send/receive
+    /// packets and open sockets.
+    ///
+    /// @throw InvalidPacketFilter if provided packet filter object is NULL.
+    void setPacketFilter(const boost::shared_ptr<PktFilter>& packet_filter) {
+        if (!packet_filter) {
+            isc_throw(InvalidPacketFilter, "NULL packet filter object specified");
+        }
+        packet_filter_ = packet_filter;
+    }
+
     /// A value of socket descriptor representing "not specified" state.
     static const int INVALID_SOCKET = -1;
 
@@ -557,9 +603,13 @@ protected:
     /// @param iface reference to interface structure.
     /// @param addr an address the created socket should be bound to
     /// @param port a port that created socket should be bound to
+    /// @param receive_bcast configure socket to receive broadcast messages
+    /// @param send_bcast configure socket to send broadcast messages.
     ///
     /// @return socket descriptor
-    int openSocket4(Iface& iface, const isc::asiolink::IOAddress& addr, uint16_t port);
+    int openSocket4(Iface& iface, const isc::asiolink::IOAddress& addr,
+                    const uint16_t port, const bool receive_bcast = false,
+                    const bool send_bcast = false);
 
     /// @brief Opens IPv6 socket.
     ///
@@ -678,6 +728,16 @@ private:
     isc::asiolink::IOAddress
     getLocalAddress(const isc::asiolink::IOAddress& remote_addr,
                     const uint16_t port);
+
+    /// Holds instance of a class derived from PktFilter, used by the
+    /// IfaceMgr to open sockets and send/receive packets through these
+    /// sockets. It is possible to supply custom object using
+    /// setPacketFilter class. Various Packet Filters differ mainly by using
+    /// different types of sockets, e.g. SOCK_DGRAM,  SOCK_RAW and different
+    /// families, e.g. AF_INET, AF_PACKET etc. Another possible type of
+    /// Packet Filter is the one used for unit testing, which doesn't
+    /// open sockets but rather mimics their behavior (mock object).
+    boost::shared_ptr<PktFilter> packet_filter_;
 };
 
 }; // namespace isc::dhcp

src/lib/dhcp/iface_mgr_bsd.cc

@@ -1,4 +1,4 @@
-// Copyright (C) 2011  Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2011, 2013 Internet Systems Consortium, Inc. ("ISC")
 //
 // Permission to use, copy, modify, and/or distribute this software for any
 // purpose with or without fee is hereby granted, provided that the above
@@ -34,6 +34,11 @@ IfaceMgr::detectIfaces() {
     stubDetectIfaces();
 }
 
+bool
+IfaceMgr::isDirectResponseSupported() {
+    return (false);
+}
+
 void IfaceMgr::os_send4(struct msghdr& /*m*/,
                         boost::scoped_array<char>& /*control_buf*/,
                         size_t /*control_buf_len*/,

src/lib/dhcp/iface_mgr_linux.cc

@@ -1,4 +1,4 @@
-// Copyright (C) 2011-2012 Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2011-2013 Internet Systems Consortium, Inc. ("ISC")
 //
 // Permission to use, copy, modify, and/or distribute this software for any
 // purpose with or without fee is hereby granted, provided that the above
@@ -103,7 +103,7 @@ public:
     void rtnl_send_request(int family, int type);
     void rtnl_store_reply(NetlinkMessages& storage, const nlmsghdr* msg);
     void parse_rtattr(RTattribPtrs& table, rtattr* rta, int len);
-    void ipaddrs_get(IfaceMgr::Iface& iface, NetlinkMessages& addr_info);
+    void ipaddrs_get(Iface& iface, NetlinkMessages& addr_info);
     void rtnl_process_reply(NetlinkMessages& info);
     void release_list(NetlinkMessages& messages);
     void rtnl_close_socket();
@@ -277,7 +277,7 @@ void Netlink::parse_rtattr(RTattribPtrs& table, struct rtattr* rta, int len)
 ///
 /// @param iface interface representation (addresses will be added here)
 /// @param addr_info collection of parsed netlink messages
-void Netlink::ipaddrs_get(IfaceMgr::Iface& iface, NetlinkMessages& addr_info) {
+void Netlink::ipaddrs_get(Iface& iface, NetlinkMessages& addr_info) {
     uint8_t addr[V6ADDRESS_LEN];
     RTattribPtrs rta_tb;
 
@@ -494,13 +494,18 @@ void IfaceMgr::detectIfaces() {
     nl.release_list(addr_info);
 }
 
+bool
+IfaceMgr::isDirectResponseSupported() {
+    return (false);
+}
+
 /// @brief sets flag_*_ fields.
 ///
 /// This implementation is OS-specific as bits have different meaning
 /// on different OSes.
 ///
 /// @param flags flags bitfield read from OS
-void IfaceMgr::Iface::setFlags(uint32_t flags) {
+void Iface::setFlags(uint32_t flags) {
     flags_ = flags;
 
     flag_loopback_ = flags & IFF_LOOPBACK;
@@ -510,56 +515,15 @@ void IfaceMgr::Iface::setFlags(uint32_t flags) {
     flag_broadcast_ = flags & IFF_BROADCAST;
 }
 
-void IfaceMgr::os_send4(struct msghdr& m, boost::scoped_array<char>& control_buf,
-                        size_t control_buf_len, const Pkt4Ptr& pkt) {
-
-    // Setting the interface is a bit more involved.
-    //
-    // We have to create a "control message", and set that to
-    // define the IPv4 packet information. We could set the
-    // source address if we wanted, but we can safely let the
-    // kernel decide what that should be.
-    m.msg_control = &control_buf[0];
-    m.msg_controllen = control_buf_len;
-    struct cmsghdr* cmsg = CMSG_FIRSTHDR(&m);
-    cmsg->cmsg_level = IPPROTO_IP;
-    cmsg->cmsg_type = IP_PKTINFO;
-    cmsg->cmsg_len = CMSG_LEN(sizeof(struct in_pktinfo));
-    struct in_pktinfo* pktinfo =(struct in_pktinfo *)CMSG_DATA(cmsg);
-    memset(pktinfo, 0, sizeof(struct in_pktinfo));
-    pktinfo->ipi_ifindex = pkt->getIndex();
-    m.msg_controllen = cmsg->cmsg_len;
-}
-
-bool IfaceMgr::os_receive4(struct msghdr& m, Pkt4Ptr& pkt) {
-    struct cmsghdr* cmsg;
-    struct in_pktinfo* pktinfo;
-    struct in_addr to_addr;
-
-    memset(&to_addr, 0, sizeof(to_addr));
 
-    cmsg = CMSG_FIRSTHDR(&m);
-    while (cmsg != NULL) {
-        if ((cmsg->cmsg_level == IPPROTO_IP) &&
-            (cmsg->cmsg_type == IP_PKTINFO)) {
-            pktinfo = (struct in_pktinfo*)CMSG_DATA(cmsg);
+void IfaceMgr::os_send4(struct msghdr&, boost::scoped_array<char>&,
+                        size_t, const Pkt4Ptr&) {
+    return;
 
-            pkt->setIndex(pktinfo->ipi_ifindex);
-            pkt->setLocalAddr(IOAddress(htonl(pktinfo->ipi_addr.s_addr)));
-            return (true);
-
-            // This field is useful, when we are bound to unicast
-            // address e.g. 192.0.2.1 and the packet was sent to
-            // broadcast. This will return broadcast address, not
-            // the address we are bound to.
-
-            // XXX: Perhaps we should uncomment this:
-            // to_addr = pktinfo->ipi_spec_dst;
-        }
-        cmsg = CMSG_NXTHDR(&m, cmsg);
-    }
+}
 
-    return (false);
+bool IfaceMgr::os_receive4(struct msghdr&, Pkt4Ptr&) {
+    return (true);
 }
 
 } // end of isc::dhcp namespace

src/lib/dhcp/iface_mgr_sun.cc

@@ -1,4 +1,4 @@
-// Copyright (C) 2011  Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2011, 2013 Internet Systems Consortium, Inc. ("ISC")
 //
 // Permission to use, copy, modify, and/or distribute this software for any
 // purpose with or without fee is hereby granted, provided that the above
@@ -34,6 +34,11 @@ IfaceMgr::detectIfaces() {
     stubDetectIfaces();
 }
 
+bool
+IfaceMgr::isDirectResponseSupported() {
+    return (false);
+}
+
 void IfaceMgr::os_send4(struct msghdr& /*m*/,
                         boost::scoped_array<char>& /*control_buf*/,
                         size_t /*control_buf_len*/,

src/lib/dhcp/pkt_filter.h

@@ -0,0 +1,84 @@
+// Copyright (C) 2013 Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#ifndef PKT_FILTER_H
+#define PKT_FILTER_H
+
+#include <asiolink/io_address.h>
+
+namespace isc {
+namespace dhcp {
+
+struct SocketInfo;
+
+/// Forward declaration to the class representing interface
+class Iface;
+
+/// @brief Abstract packet handling class
+///
+/// This class represents low level method to send and receive DHCP packet.
+/// Different methods, represented by classes derived from this class, use
+/// different socket families and socket types. Also, various packet filtering
+/// methods can be implemented by derived classes, e.g. Linux Packet
+/// Filtering (LPF) or Berkeley Packet Filtering (BPF).
+///
+/// Low-level code operating on sockets may require special privileges to execute.
+/// For example: opening raw socket or opening socket on low port number requires
+/// root privileges. This makes it impossible or very hard to unit test the IfaceMgr.
+/// In order to overcome this problem, it is recommended to create mock object derived
+/// from this class that mimics the behavior of the real packet handling class making
+/// IfaceMgr testable.
+class PktFilter {
+public:
+
+    /// @brief Virtual Destructor
+    virtual ~PktFilter() { }
+
+    /// @brief Open socket.
+    ///
+    /// @param iface interface descriptor
+    /// @param addr address on the interface to be used to send packets.
+    /// @param port port number.
+    /// @param receive_bcast configure socket to receive broadcast messages
+    /// @param send_bcast configure socket to send broadcast messages.
+    ///
+    /// @return created socket's descriptor
+    virtual int openSocket(const Iface& iface,
+                           const isc::asiolink::IOAddress& addr,
+                           const uint16_t port,
+                           const bool receive_bcast,
+                           const bool send_bcast) = 0;
+
+    /// @brief Receive packet over specified socket.
+    ///
+    /// @param iface interface
+    /// @param socket_info structure holding socket information
+    ///
+    /// @return Received packet
+    virtual Pkt4Ptr receive(const Iface& iface,
+                            const SocketInfo& socket_info) = 0;
+
+    /// @brief Send packet over specified socket.
+    ///
+    /// @param sockfd socket descriptor
+    /// @param pkt packet to be sent
+    ///
+    /// @return result of sending the packet. It is 0 if successful.
+    virtual int send(uint16_t sockfd, const Pkt4Ptr& pkt) = 0;
+};
+
+} // namespace isc::dhcp
+} // namespace isc
+
+#endif // PKT_FILTER_H