Browse Source

[2472] Merge branch 'master' into trac2472

Stephen Morris 12 years ago
parent
commit
938ab8a39d

+ 5 - 0
ChangeLog

@@ -1,3 +1,8 @@
+506.	[doc]		jelte
+	Added a chapter about the use of the bindctl command tool to
+	to the BIND 10 guide.
+	(Trac #2305, git c4b0294b5bf4a9d32fb18ab62ca572f492788d72)
+
 505.	[bug]		jelte
 	Fixed a bug in b10-xfrin where a wrong call was made during the
 	final check of a TSIG-signed transfer, incorrectly rejecting the

+ 1 - 1
configure.ac

@@ -767,7 +767,7 @@ if test "$MYSQL_CONFIG" != "" ; then
     LIBS="$MYSQL_LIBS $LIBS"
 
     AC_LINK_IFELSE(
-            [AC_LANG_PROGRAM([#include <mysql/mysql.h>],
+            [AC_LANG_PROGRAM([#include <mysql.h>],
                              [MYSQL mysql_handle;
                               (void) mysql_init(&mysql_handle);
                              ])],

+ 590 - 14
doc/guide/bind10-guide.xml

@@ -1275,6 +1275,14 @@ TODO
       configuring BIND 10.
     </para></note>
 
+    <note><para>
+      <command>bindctl</command> has an internal command history, as
+      well as tab-completion for most of the commands and arguments.
+      However, these are only enabled if the python readline module
+      is available on the system. If not, neither of these
+      features will be supported.
+    </para></note>
+
     <para>
       The <command>bindctl</command> tool provides an interactive
       prompt for configuring, controlling, and querying the BIND 10
@@ -1284,22 +1292,590 @@ TODO
       communicate to any other components directly.
     </para>
 
-<!-- TODO: explain and show interface -->
+    <section id="bindctl_commandline_options">
+        <title>bindctl command-line options</title>
+        <variablelist>
+          <varlistentry>
+            <term>-a <replaceable>&lt;address&gt;</replaceable>, --address=<replaceable>&lt;address&gt;</replaceable></term>
+            <listitem>
+              <simpara>
+                  IP address that BIND 10's <command>b10-cmdctl</command>
+                  module is listening on. By default, this is 127.0.0.1.
+              </simpara>
+            </listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>-c <replaceable>&lt;certificate file&gt;</replaceable>, --certificate-chain=<replaceable>&lt;certificate file&gt;</replaceable></term>
+            <listitem>
+              <simpara>
+                  PEM-formatted server certificate file. When this option is
+                  given, <command>bindctl</command> will verify the server
+                  certificate using the given file as the root of the
+                  certificate chain. If not specified, <command>bindctl
+                  </command> does not validate the certificate.
+              </simpara>
+            </listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>--csv-file-dir=<replaceable>&lt;csv file&gt;</replaceable></term>
+            <listitem>
+              <simpara>
+                  <command>bindctl</command> stores the username and
+                  password for logging in in a file called
+                  <filename>default_user.csv</filename>;
+                  this option specifies the directory where this file is
+                  stored and read from. When not specified,
+                  <filename>~/.bind10/</filename> is used.
+                  <note>Currently, this file contains an unencrypted password.</note>
+              </simpara>
+            </listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>-h, --help</term>
+            <listitem>
+              <simpara>
+                  Shows a short overview of the command-line options of
+                  <command>bindctl</command>, and exits.
+              </simpara>
+            </listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>--version</term>
+            <listitem>
+              <simpara>
+                  Shows the version of <command>bindctl</command>, and exits.
+              </simpara>
+            </listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>-p <replaceable>&lt;port number&gt;</replaceable>, --port=<replaceable>&lt;port number&gt;</replaceable></term>
+            <listitem>
+              <simpara>
+                  Port number that BIND 10's <command>b10-cmdctl</command>
+                  module is listening on. By default, this is port 8080.
+              </simpara>
+            </listitem>
+          </varlistentry>
+        </variablelist>
+    </section>
 
-    <para>
-      Configuration changes are actually commands to
-      <command>b10-cfgmgr</command>. So when <command>bindctl</command>
-      sends a configuration, it is sent to <command>b10-cmdctl</command>
-      (over a HTTPS connection); then <command>b10-cmdctl</command>
-      sends the command (over a <command>b10-msgq</command> command
-      channel) to <command>b10-cfgmgr</command> which then stores
-      the details and relays (over a <command>b10-msgq</command> command
-      channel) the configuration on to the specified module.
-    </para>
+    <section id="bindctl_general_syntax">
+        <title>General syntax of bindctl commands</title>
+        The <command>bindctl</command> tool is an interactive
+        command-line tool, with dynamic commands depending on the
+        BIND 10 modules that are running. There are a number of
+        fixed commands that have no module and that are always
+        available.
 
-    <para>
-    </para>
+        The general syntax of a command is
+
+        <screen><userinput>&lt;module&gt; &lt;command&gt; <replaceable>[argument(s)]</replaceable></userinput></screen>
+
+        For example, the Boss module has a 'shutdown' command to shut down
+        BIND 10, with an optional argument 'help':
+
+        <screen><userinput>> Boss shutdown help</userinput>
+Command  shutdown 	(Shut down BIND 10)
+		help (Get help for command)
+This command has no parameters
+        </screen>
+        There are no mandatory arguments, only the optional 'help'.
+    </section>
+
+    <section id="bindctl_help">
+        <title>Bindctl help</title>
+        <command>help</command> is both a command and an option that is available to all other commands. When run as a command directly, it shows the available modules.
+        <screen>&gt; <userinput>help</userinput>
+usage: &lt;module name&gt; &lt;command name&gt; [param1 = value1 [, param2 = value2]]
+Type Tab character to get the hint of module/command/parameters.
+Type "help(? h)" for help on bindctl.
+Type "&lt;module_name&gt; help" for help on the specific module.
+Type "&lt;module_name&gt; &lt;command_name&gt; help" for help on the specific command.
+
+Available module names:
+<emphasis>(list of modules)</emphasis>
+        </screen>
+
+        When 'help' is used as a command to a module, it shows the supported commands for the module; for example:
+        <screen>&gt; <userinput>Boss help</userinput>
+Module  Boss 	Master process
+Available commands:
+    help        Get help for module.
+    shutdown    Shut down BIND 10
+    ping        Ping the boss process
+    show_processes
+            List the running BIND 10 processes
+        </screen>
+
+    And when added to a module command, it shows the description and parameters of that specific command; for example:
+    <screen>&gt; <userinput>Auth loadzone help</userinput>
+Command  loadzone 	((Re)load a specified zone)
+		help (Get help for command)
+Parameters:
+    class (string, optional)
+    origin (string, mandatory)
+    </screen>
+
+    </section>
 
+    <section id="bindctl_command_arguments">
+        <title>Command arguments</title>
+        <simpara>
+            Commands can have arguments, which can be either optional or
+            mandatory. They can be specified by name
+            (e.g. <command><replaceable>&lt;command&gt;</replaceable> <replaceable>&lt;argument name&gt;=&lt;argument value&gt;</replaceable></command>), or positionally,
+            (e.g. <command><replaceable>&lt;command&gt;</replaceable> <replaceable>&lt;argument value 1&gt;</replaceable> <replaceable>&lt;argument value 2&gt;</replaceable></command>).
+        </simpara>
+        <simpara>
+            <command><replaceable>&lt;command&gt;</replaceable> <replaceable>help</replaceable></command>
+            shows the arguments a command supports and which of those are
+            mandatory, and in which order the arguments are expected if
+            positional arguments are used.
+        </simpara>
+        <simpara>
+            For example, the <command>loadzone</command> command of the Auth
+            module, as shown in the last example of the previous section, has
+            two arguments, one of which is optional. The positional arguments in
+            this case are class first and origin second; for example:
+            <screen>&gt; <userinput>Auth loadzone IN example.com.</userinput></screen>
+            But since the class is optional (defaulting to IN), leaving it out
+            works as well:
+            <screen>&gt; <userinput>Auth loadzone example.com.</userinput></screen>
+        </simpara>
+        <simpara>
+            The arguments can also be provided with their names, in which
+            case the order does not matter:
+            <screen>&gt; <userinput>Auth loadzone origin="example.com." class="IN"</userinput></screen>
+        </simpara>
+    </section>
+
+    <section id="bindctl_module_commands">
+        <title>Module commands</title>
+        Each module has its own set of commands (if any), which will only be
+        available if the module is running. For instance, the
+        Auth module has a <command>loadzone</command> command.
+        The commands a module provides are documented in
+        this guide in the section of that module or in the module's
+        corresponding manual page.
+    </section>
+
+    <section>
+        <title>Configuration commands</title>
+        Configuration commands are used to view and change the configuration
+        of BIND 10 and its modules. Module configuration is only shown if
+        that module is running, but similar to commands, there are a number
+        of top-level configuration items that are always available (for
+        instance <varname>tsig_keys</varname> and
+        <varname>data_sources</varname>).
+
+        Configuration changes (set, unset, add and remove) are done locally
+        first, and have no immediate effect. The changes can be viewed with
+        <command>config diff</command>, and either reverted
+        (<command>config revert</command>), or committed
+        (<command>config commit</command>).
+        In the latter case, all local changes are submitted
+        to the configuration manager, which verifies them, and if they are
+        accepted, applied and saved in persistent storage.
+
+        When identifying items in configuration commands, the format is
+        <screen><userinput>Module/example/item</userinput></screen>
+        Sub-elements of names, lists and sets (see <xref linkend=
+        "bindctl_configuration_data_types"/>) are separated with the '/'
+        character, and list indices are identified with [<replaceable>&lt;index&gt;</replaceable>]; for example:
+
+        <screen><userinput>Module/example/list[2]/foo</userinput></screen>
+
+        <section id="bindctl_configuration_command_list">
+        <title>List of configuration commands</title>
+        The following configuration commands are available:
+        <variablelist>
+          <varlistentry>
+            <term>show [all] [item name]</term>
+            <listitem>
+              <simpara>
+                Shows the current configuration of the given item. If 'all'
+                is given, it will recurse through the entire set, and show
+                every nested value.
+              </simpara>
+            </listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>show_json [item name]</term>
+            <listitem>
+              <simpara>
+                Shows the full configuration of the given item in JSON format.
+              </simpara>
+            </listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>add &lt;item name&gt; [value]</term>
+            <listitem>
+              <simpara>
+                Add an entry to configuration list or a named set (see <xref
+                linkend="bindctl_configuration_data_types"/>).
+                When adding to a list, the command has one optional
+                argument, a value to add to the list. The value must
+                be in correct JSON and complete. When adding to a
+                named set, it has one mandatory parameter (the name to
+                add), and an optional parameter value, similar to when
+                adding to a list. In either case, when no value is
+                given, an entry will be constructed with default
+                values.
+              </simpara>
+            </listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>remove</term>
+            <listitem>
+              <simpara>
+                Remove an item from a configuration list or a named set.
+                When removing an item for a list, either the index needs to
+                be specified, or the complete value of the element to remove
+                must be specified (in JSON format).
+              </simpara>
+            </listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>set &lt;item name&gt; &lt;value&gt;</term>
+            <listitem>
+              <simpara>
+                  Directly set the value of the given item to the given value.
+              </simpara>
+            </listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>unset &lt;item name&gt;</term>
+            <listitem>
+              <simpara>
+                  Remove any user-specified value for the given item.
+              </simpara>
+            </listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>diff</term>
+            <listitem>
+              <simpara>
+                  Show all current local changes that have not been
+                  committed yet.
+              </simpara>
+            </listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>revert</term>
+            <listitem>
+              <simpara>
+                  Revert all local changes without committing them.
+              </simpara>
+            </listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>commit</term>
+            <listitem>
+              <simpara>
+                  Send all local changes to the configuration manager, which
+                  will validate them, and apply them if validation succeeds.
+              </simpara>
+            </listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>go</term>
+            <listitem>
+              <simpara>
+                  Go to a specific configuration part, similar to the 'cd'
+                  command in a shell.
+                  <note>There are a number of problems with the current
+                  implementation of go within <command>bindctl</command>,
+                  and we recommend not using it for general cases.</note>
+              </simpara>
+            </listitem>
+          </varlistentry>
+        </variablelist>
+      </section>
+
+      <section id="bindctl_configuration_data_types">
+        <title>Configuration data types</title>
+        Configuration data can be of different types, which can be modified
+        in ways that depend on the types. There are a few syntax
+        restrictions on these types, but only basic ones. Modules may impose
+        additional restrictions on the values of elements.
+        <variablelist>
+            <varlistentry>
+                <term>integer</term>
+                <listitem>
+                    <simpara>
+                        A basic integer; can be set directly with <command>config set</command>, to any integer value.
+                    </simpara>
+                </listitem>
+            </varlistentry>
+            <varlistentry>
+                <term>real</term>
+                <listitem>
+                    <simpara>
+                        A basic floating point number; can be set directly with <command>config set</command>, to any floating point value.
+                    </simpara>
+                </listitem>
+            </varlistentry>
+            <varlistentry>
+                <term>boolean</term>
+                <listitem>
+                    <simpara>
+                        A basic boolean value; can be set directly with <command>config set</command>, to either <command>true</command> or <command>false</command>.
+                    </simpara>
+                </listitem>
+            </varlistentry>
+            <varlistentry>
+                <term>string</term>
+                <listitem>
+                    <simpara>
+                        A basic string value; can be set directly with <command>config set,</command> so any string. Double quotation marks are optional.
+                    </simpara>
+                </listitem>
+            </varlistentry>
+            <varlistentry>
+                <term>null</term>
+                <listitem>
+                    <simpara>
+                        This is a special type representing 'no value at all'; usable in compound structures that have optional elements that are not set.
+                    </simpara>
+                </listitem>
+            </varlistentry>
+
+            <varlistentry>
+                <term>maps</term>
+                <listitem>
+                    <simpara>
+                        Maps are (pre-defined) compound collections of other
+                        elements of any other type. They are not usually
+                        modified directly, but their elements are. Every
+                        top-level element for a module is a map containing
+                        the configuration values for that map, which can
+                        themselves be maps again. For instance, the Auth
+                        module configuration is a map containing the
+                        elements '<varname>listen_on</varname>' (list) and '<varname>tcp_recv_timeout</varname>'
+                        (integer). When changing one of its values, they can
+                        be modified directly with <command>config set
+                        Auth/tcp_recv_timeout 3000</command>.
+                    </simpara>
+                    <simpara>
+                        Some map entries are optional. If they are, and
+                        currently have a value, the value can be unset by
+                        using either <command>config unset
+                        <replaceable>&lt;item name&gt;</replaceable>
+                        </command> or <command>config set
+                        <replaceable>&lt;item name&gt;</replaceable>
+                        null</command>.
+
+                    </simpara>
+                    <simpara>
+                        Maps <emphasis>can</emphasis> be modified as a whole,
+                        but using the full JSON representation of
+                        the entire map to set.
+
+                        Since this involves a lot of text, this is usually
+                        not recommended.
+                    </simpara>
+                    <simpara>
+                        Another example is the Logging virtual module, which
+                        is, like any module, a map, but it only contains one
+                        element: a list of loggers. Normally, an
+                        administrator would only modify that list (or its
+                        elements) directly, but it is possible to set the
+                        entire map in one command; for example:
+                        <command> config set Logging { "loggers": [] } </command>
+                    </simpara>
+                </listitem>
+            </varlistentry>
+
+            <varlistentry>
+                <term>list</term>
+                <listitem>
+                    <simpara>
+                        A list is a compound list of other elements of the
+                        same type. Elements can be added with <command>config
+                        add <replaceable>&lt;list name&gt; [value]</replaceable></command>, and removed with
+                        <command>config remove <replaceable>&lt;list name&gt; [value]</replaceable></command> or
+                        <command>config remove <replaceable>&lt;list name&gt;</replaceable><replaceable>&lt;index&gt;</replaceable></command>.
+                        The index is of the form <emphasis>square bracket, number,
+                        square bracket</emphasis> (e.g.
+                        <command>[0]</command>), and it immediately follows
+                        the list name (there is no separator or space
+                        between them). List indices start with 0 for the
+                        first element.
+                    </simpara>
+                    <simpara>
+                        For addition, if the value is omitted, an entry with
+                        default values will be added. For removal, either
+                        the index or the full value (in JSON format) needs
+                        to be specified.
+                    </simpara>
+                    <simpara>
+                        Lists can also be used with
+                        <command>config set</command>,
+                        but like maps, only by specifying the
+                        entire list value in JSON format.
+                    </simpara>
+                    <simpara>
+                        For example, this command shows the port number used for the second element of the list <varname>listen_on</varname> in the Auth module:
+                        <command> config show Auth/listen_on[1]/port</command>
+                    </simpara>
+                </listitem>
+            </varlistentry>
+
+            <varlistentry>
+                <term>named set</term>
+                <listitem>
+                    <simpara>
+                        Named sets are similar to lists, in that they are
+                        sets of elements of the same type, but they are not
+                        indexed by numbers, but by strings.
+                    </simpara>
+                    <simpara>
+                        Values can be added with
+                        <command>config add <replaceable>&lt;item name&gt; &lt;string&gt; [value]</replaceable></command>
+                        where 'string' is the name of the element. If 'value'
+                        is ommitted, default values will be used. Elements
+                        can be removed with <command>config remove
+                        <replaceable>&lt;item
+                        name&gt; &lt;string&gt;</replaceable></command>
+                    </simpara>
+                    <simpara>
+                        Elements in a named set can be addressed similarly
+                        to maps.
+                    </simpara>
+                    <simpara>
+                        For example, the <command>Boss/components</command>
+                        elements is a named set;
+                        adding, showing, and then removing an element
+                        can be done with the following three commands (note
+                        the '/'-character versus the space before
+                        'example_module'):
+                    </simpara>
+                    <simpara>
+                        <command>config add Boss/components example_module</command>
+                    </simpara>
+                    <simpara>
+                        <command>config show Boss/components/example_module</command>
+                    </simpara>
+                    <simpara>
+                        <command>config remove Boss/components example_module</command>
+                    </simpara>
+                </listitem>
+            </varlistentry>
+            <varlistentry>
+                <term>any</term>
+                <listitem>
+                    <simpara>
+                        The 'any' type is a special type that can have any
+                        form. Apart from that, it must consist of elements as
+                        described in this chapter, there is no restriction
+                        on which element types are used. This type is used
+                        in places where different data formats could be
+                        used. Element modification commands depend on the
+                        actual type of the value. For instance, if the value
+                        of an 'any' element is a list, <command>config add
+                        </command> and <command>config remove</command> work
+                        as for other lists.
+                    </simpara>
+                </listitem>
+            </varlistentry>
+        </variablelist>
+      </section>
+    </section>
+
+    <section>
+        <title>The execute command</title>
+        The <command>execute</command> command executes a set of commands,
+        either from a file
+        or from a pre-defined set. Currently, the only predefined set is
+        <command>init_authoritative_server</command>, which adds
+        <command>b10-auth</command>, <command>b10-xfrin</command>, and
+        <command>b10-xfrout</command> to the set of components to be
+        started by BIND 10. This
+        pre-defined set does not commit the changes, so these modules do not
+        show up for commands or configuration until the user enters
+        <command>config commit</command> after
+        <command>execute init_authoritative_server</command>. For example:
+
+        <screen>&gt; <userinput>execute init_authoritative_server</userinput></screen>
+
+        <screen>&gt; <userinput>execute file /tmp/example_commands</userinput></screen>
+
+        The optional argument <command>show</command> displays the exact set of
+        commands that would be executed; for example:
+
+        <screen>&gt; <userinput>execute init_authoritative_server show</userinput>
+!echo adding Authoritative server component
+config add /Boss/components b10-auth
+config set /Boss/components/b10-auth/kind needed
+config set /Boss/components/b10-auth/special auth
+!echo adding Xfrin component
+config add /Boss/components b10-xfrin
+config set /Boss/components/b10-xfrin/address Xfrin
+config set /Boss/components/b10-xfrin/kind dispensable
+!echo adding Xfrout component
+config add /Boss/components b10-xfrout
+config set /Boss/components/b10-xfrout/address Xfrout
+config set /Boss/components/b10-xfrout/kind dispensable
+!echo adding Zone Manager component
+config add /Boss/components b10-zonemgr
+config set /Boss/components/b10-zonemgr/address Zonemgr
+config set /Boss/components/b10-zonemgr/kind dispensable
+!echo Components added. Please enter "config commit" to
+!echo finalize initial setup and run the components.
+        </screen>
+
+        The optional <command>show</command> argument may also be used when
+        executing a script from a file; for example:
+
+        <screen><userinput>> execute file /tmp/example_commands show</userinput></screen>
+
+        <section id="bindctl_execute_directives">
+            <title>Execute directives</title>
+            Within sets of commands to be run with the <command>execute</command>
+            command, a number of directives are supported:
+            <variablelist>
+              <varlistentry>
+                <term>!echo <replaceable>&lt;string&gt;</replaceable></term>
+                <listitem>
+                  <simpara>
+                      Prints the given string to <command>bindctl</command>'s
+                      output.
+                  </simpara>
+                </listitem>
+              </varlistentry>
+              <varlistentry>
+                <term>!verbose on</term>
+                <listitem>
+                  <simpara>
+                      Enables verbose mode; all following commands that are to
+                      be executed are also printed.
+                  </simpara>
+                </listitem>
+              </varlistentry>
+              <varlistentry>
+                <term>!verbose off</term>
+                <listitem>
+                  <simpara>
+                      Disables verbose mode; following commands that are to
+                      be executed are no longer printed.
+                  </simpara>
+                </listitem>
+              </varlistentry>
+            </variablelist>
+        </section>
+
+        <section id="bindctl_execute_notes">
+            <title>Notes on execute scripts</title>
+            Within scripts, you can add or remove modules with the normal
+            configuration commands for <command>Boss/components</command>.
+            However, as module
+            configuration and commands do not show up until the module is
+            running, it is currently not possible to add a module and set
+            its configuration in one script. This will be addressed in the
+            future, but for now the only option is to add and configure
+            modules in separate commands and execute scripts.
+        </section>
+    </section>
   </chapter>
 
   <chapter id="common">
@@ -2990,7 +3566,7 @@ Dhcp6/subnet6	         []     list    (default)</screen>
         enclosed in square brackets, even though only one range of addresses
         is specified.</para>
         <para>It is possible to define more than one pool in a
-        subnet: continuing the previous example, further assume that 
+        subnet: continuing the previous example, further assume that
         2001:db8:1:0:5::/80 should be also be managed by the server. It could be written as
         2001:db8:1:0:5:: to 2001:db8:1::5:ffff:ffff:ffff, but typing so many 'f's
         is cumbersome. It can be expressed more simply as 2001:db8:1:0:5::/80. Both

+ 10 - 0
src/bin/auth/auth_messages.mes

@@ -98,6 +98,16 @@ This debug message is issued when the separate thread for maintaining data
 source clients successfully loaded the named zone of the named class as a
 result of the 'loadzone' command.
 
+% AUTH_DATASRC_CLIENTS_BUILDER_LOAD_ZONE_NOCACHE skipped loading zone %1/%2 due to no in-memory cache
+This debug message is issued when the separate thread for maintaining data
+source clients received a command to reload a zone but skipped it because
+the specified zone is not loaded in-memory (but served from an underlying
+data source).  This could happen if the loadzone command is manually issued
+by a user but the zone name is misspelled, but a more likely cause is
+that the command is sent from another BIND 10 module (such as xfrin or DDNS).
+In the latter case it can be simply ignored because there is no need
+for explicit reloading.
+
 % AUTH_DATASRC_CLIENTS_BUILDER_RECONFIGURE_CONFIG_ERROR Error in data source configuration: %1
 The thread for maintaining data source clients has received a command to
 reconfigure, but the parameter data (the new configuration) contains an

+ 7 - 3
src/bin/auth/auth_srv.cc

@@ -651,9 +651,10 @@ AuthSrvImpl::processNormalQuery(const IOMessage& io_message, Message& message,
         local_edns->setUDPSize(AuthSrvImpl::DEFAULT_LOCAL_UDPSIZE);
         message.setEDNS(local_edns);
     }
-    // Get access to data source client list through the holder and keep the
-    // holder until the processing and rendering is done to avoid inter-thread
-    // race.
+
+    // Get access to data source client list through the holder and keep
+    // the holder until the processing and rendering is done to avoid
+    // race with any other thread(s) such as the background loader.
     auth::DataSrcClientsMgr::Holder datasrc_holder(datasrc_clients_mgr_);
 
     try {
@@ -688,6 +689,9 @@ AuthSrvImpl::processNormalQuery(const IOMessage& io_message, Message& message,
     return (true);
     // The message can contain some data from the locked resource. But outside
     // this method, we touch only the RCode of it, so it should be safe.
+
+    // Lock on datasrc_clients_mgr_ acquired by datasrc_holder is
+    // released here upon its deletion.
 }
 
 bool

+ 15 - 6
src/bin/auth/datasrc_clients_mgr.h

@@ -581,6 +581,9 @@ DataSrcClientsBuilderBase<MutexType, CondVarType>::doLoadZone(
     try {
         boost::shared_ptr<datasrc::memory::ZoneWriter> zwriter =
             getZoneWriter(*client_list, rrclass, origin);
+        if (!zwriter) {
+            return;
+        }
 
         zwriter->load(); // this can take time but doesn't cause a race
         {   // install() can cause a race and must be in a critical section
@@ -614,8 +617,14 @@ DataSrcClientsBuilderBase<MutexType, CondVarType>::getZoneWriter(
     datasrc::ConfigurableClientList& client_list,
     const dns::RRClass& rrclass, const dns::Name& origin)
 {
-    const datasrc::ConfigurableClientList::ZoneWriterPair writerpair =
-        client_list.getCachedZoneWriter(origin);
+    // getCachedZoneWriter() could get access to an underlying data source
+    // that can cause a race condition with the main thread using that data
+    // source for lookup.  So we need to protect the access here.
+    datasrc::ConfigurableClientList::ZoneWriterPair writerpair;
+    {
+        typename MutexType::Locker locker(*map_mutex_);
+        writerpair = client_list.getCachedZoneWriter(origin);
+    }
 
     switch (writerpair.first) {
     case datasrc::ConfigurableClientList::ZONE_SUCCESS:
@@ -626,8 +635,10 @@ DataSrcClientsBuilderBase<MutexType, CondVarType>::getZoneWriter(
                   << "/" << rrclass << ": not found in any configured "
                   "data source.");
     case datasrc::ConfigurableClientList::ZONE_NOT_CACHED:
-        isc_throw(InternalCommandError, "failed to load zone " << origin
-                  << "/" << rrclass << ": not served from memory");
+        LOG_DEBUG(auth_logger, DBG_AUTH_OPS,
+                  AUTH_DATASRC_CLIENTS_BUILDER_LOAD_ZONE_NOCACHE)
+            .arg(origin).arg(rrclass);
+        break;                  // return NULL below
     case datasrc::ConfigurableClientList::CACHE_DISABLED:
         // This is an internal error. Auth server must have the cache
         // enabled.
@@ -636,8 +647,6 @@ DataSrcClientsBuilderBase<MutexType, CondVarType>::getZoneWriter(
                   "is somehow disabled");
     }
 
-    // all cases above should either return or throw, but some compilers
-    // still need a return statement
     return (boost::shared_ptr<datasrc::memory::ZoneWriter>());
 }
 } // namespace datasrc_clientmgr_internal

+ 15 - 6
src/bin/auth/tests/datasrc_clients_builder_unittest.cc

@@ -308,8 +308,12 @@ TEST_F(DataSrcClientsBuilderTest, loadZone) {
                                    "{\"class\": \"IN\","
                                    " \"origin\": \"test1.example\"}"));
     EXPECT_TRUE(builder.handleCommand(loadzone_cmd));
-    EXPECT_EQ(1, map_mutex.lock_count); // we should have acquired the lock
-    EXPECT_EQ(1, map_mutex.unlock_count); // and released it.
+
+    // loadZone involves two critical sections: one for getting the zone
+    // writer, and one for actually updating the zone data.  So the lock/unlock
+    // count should be incremented by 2.
+    EXPECT_EQ(2, map_mutex.lock_count);
+    EXPECT_EQ(2, map_mutex.unlock_count);
 
     newZoneChecks(clients_map, rrclass);
 }
@@ -381,7 +385,10 @@ TEST_F(DataSrcClientsBuilderTest,
               find(Name("example.org")).finder_->
               find(Name("example.org"), RRType::SOA())->code);
 
-    // attempt of reloading a zone but in-memory cache is disabled.
+    // attempt of reloading a zone but in-memory cache is disabled.  In this
+    // case the command is simply ignored.
+    const size_t orig_lock_count = map_mutex.lock_count;
+    const size_t orig_unlock_count = map_mutex.unlock_count;
     const ConstElementPtr config2(Element::fromJSON("{"
         "\"IN\": [{"
         "    \"type\": \"sqlite3\","
@@ -390,11 +397,13 @@ TEST_F(DataSrcClientsBuilderTest,
         "    \"cache-zones\": [\"example.org\"]"
         "}]}"));
     clients_map = configureDataSource(config2);
-    EXPECT_THROW(builder.handleCommand(
+    builder.handleCommand(
                      Command(LOADZONE, Element::fromJSON(
                                  "{\"class\": \"IN\","
-                                 " \"origin\": \"example.org\"}"))),
-                 TestDataSrcClientsBuilder::InternalCommandError);
+                                 " \"origin\": \"example.org\"}")));
+    // Only one mutex was needed because there was no actual reload.
+    EXPECT_EQ(orig_lock_count + 1, map_mutex.lock_count);
+    EXPECT_EQ(orig_unlock_count + 1, map_mutex.unlock_count);
 
     // basically impossible case: in-memory cache is completely disabled.
     // In this implementation of manager-builder, this should never happen,

+ 2 - 1
src/bin/bindctl/moduleinfo.py

@@ -221,7 +221,8 @@ class ModuleInfo:
 
     def module_help(self):
         """Prints the help info for this module to stdout"""
-        print("Module ", self, "\nAvailable commands:")
+        print("Module " + str(self))
+        print("Available commands:")
         for k in self.commands.values():
             n = k.get_name()
             if len(n) >= CONST_BINDCTL_HELP_INDENT_WIDTH:

+ 0 - 3
src/lib/dhcp/tests/Makefile.am

@@ -4,9 +4,6 @@ AM_CPPFLAGS = -I$(top_builddir)/src/lib -I$(top_srcdir)/src/lib
 AM_CPPFLAGS += $(BOOST_INCLUDES)
 AM_CPPFLAGS += -DTEST_DATA_BUILDDIR=\"$(abs_top_builddir)/src/lib/dhcp/tests\"
 AM_CPPFLAGS += -DINSTALL_PROG=\"$(abs_top_srcdir)/install-sh\"
-# Temp
-AM_CPPFLAGS += -I/usr/include/mysql -DBIG_JOINS=1 -fno-strict-aliasing
-AM_CPPFLAGS += -DUNIV_LINUX
 
 AM_CXXFLAGS = $(B10_CXXFLAGS)
 

+ 1 - 0
src/lib/dns/Makefile.am

@@ -97,6 +97,7 @@ libb10_dns___la_SOURCES += master_lexer_inputsource.h master_lexer_inputsource.c
 libb10_dns___la_SOURCES += labelsequence.h labelsequence.cc
 libb10_dns___la_SOURCES += masterload.h masterload.cc
 libb10_dns___la_SOURCES += master_lexer.h master_lexer.cc
+libb10_dns___la_SOURCES += master_lexer_state.h
 libb10_dns___la_SOURCES += message.h message.cc
 libb10_dns___la_SOURCES += messagerenderer.h messagerenderer.cc
 libb10_dns___la_SOURCES += name.h name.cc

+ 170 - 2
src/lib/dns/master_lexer.cc

@@ -16,6 +16,7 @@
 
 #include <dns/master_lexer.h>
 #include <dns/master_lexer_inputsource.h>
+#include <dns/master_lexer_state.h>
 
 #include <boost/shared_ptr.hpp>
 
@@ -32,10 +33,34 @@ typedef boost::shared_ptr<master_lexer_internal::InputSource> InputSourcePtr;
 using namespace master_lexer_internal;
 
 struct MasterLexer::MasterLexerImpl {
-    MasterLexerImpl() : token_(Token::NOT_STARTED) {}
+    MasterLexerImpl() : source_(NULL), token_(Token::NOT_STARTED),
+                        paren_count_(0), last_was_eol_(false)
+    {}
+
+    // A helper method to skip possible comments toward the end of EOL or EOF.
+    // commonly used by state classes.  It returns the corresponding "end-of"
+    // character in case it's a comment; otherwise it simply returns the
+    // current character.
+    int skipComment(int c) {
+        if (c == ';') {
+            while (true) {
+                c = source_->getChar();
+                if (c == '\n' || c == InputSource::END_OF_STREAM) {
+                    return (c);
+                }
+            }
+        }
+        return (c);
+    }
 
     std::vector<InputSourcePtr> sources_;
-    Token token_;
+    InputSource* source_;       // current source (NULL if sources_ is empty)
+    Token token_;               // currently recognized token (set by a state)
+
+    // These are used in states, and defined here only as a placeholder.
+    // The main lexer class does not need these members.
+    size_t paren_count_;        // nest count of the parentheses
+    bool last_was_eol_; // whether the lexer just passed an end-of-line
 };
 
 MasterLexer::MasterLexer() : impl_(new MasterLexerImpl) {
@@ -60,12 +85,14 @@ MasterLexer::pushSource(const char* filename, std::string* error) {
         return (false);
     }
 
+    impl_->source_ = impl_->sources_.back().get();
     return (true);
 }
 
 void
 MasterLexer::pushSource(std::istream& input) {
     impl_->sources_.push_back(InputSourcePtr(new InputSource(input)));
+    impl_->source_ = impl_->sources_.back().get();
 }
 
 void
@@ -75,6 +102,8 @@ MasterLexer::popSource() {
                   "MasterLexer::popSource on an empty source");
     }
     impl_->sources_.pop_back();
+    impl_->source_ = impl_->sources_.empty() ? NULL :
+        impl_->sources_.back().get();
 }
 
 std::string
@@ -115,5 +144,144 @@ MasterLexer::Token::getErrorText() const {
     return (error_text[val_.error_code_]);
 }
 
+namespace master_lexer_internal {
+// Below we implement state classes for state transitions of MasterLexer.
+// Note that these need to be defined here so that they can refer to
+// the details of MasterLexerImpl.
+
+typedef MasterLexer::Token Token; // convenience shortcut
+
+bool
+State::wasLastEOL(const MasterLexer& lexer) const {
+    return (lexer.impl_->last_was_eol_);
+}
+
+const MasterLexer::Token&
+State::getToken(const MasterLexer& lexer) const {
+    return (lexer.impl_->token_);
+}
+
+size_t
+State::getParenCount(const MasterLexer& lexer) const {
+    return (lexer.impl_->paren_count_);
+}
+
+namespace {
+class CRLF : public State {
+public:
+    CRLF() {}
+    virtual ~CRLF() {}          // see the base class for the destructor
+    virtual const State* handle(MasterLexer& lexer) const {
+        // We've just seen '\r'.  If this is part of a sequence of '\r\n',
+        // we combine them as a single END-OF-LINE.  Otherwise we treat the
+        // single '\r' as an EOL and continue tokeniziation from the character
+        // immediately after '\r'.  One tricky case is that there's a comment
+        // between '\r' and '\n'.  This implementation combines these
+        // characters and treats them as a single EOL (the behavior derived
+        // from BIND 9).  Technically this may not be correct, but in practice
+        // the caller wouldn't distinguish this case from the case it has
+        // two EOLs, so we simplify the process.
+        const int c = getLexerImpl(lexer)->skipComment(
+            getLexerImpl(lexer)->source_->getChar());
+        if (c != '\n') {
+            getLexerImpl(lexer)->source_->ungetChar();
+        }
+        getLexerImpl(lexer)->token_ = Token(Token::END_OF_LINE);
+        getLexerImpl(lexer)->last_was_eol_ = true;
+        return (NULL);
+    }
+};
+
+// Currently this is provided mostly as a place holder
+class String : public State {
+public:
+    String() {}
+    virtual ~String() {}      // see the base class for the destructor
+    virtual const State* handle(MasterLexer& /*lexer*/) const {
+        return (NULL);
+    }
+};
+
+// We use a common instance of a each state in a singleton-like way to save
+// construction overhead.  They are not singletons in its strict sense as
+// we don't prohibit direct construction of these objects.  But that doesn't
+// matter much anyway, because the definitions are completely hidden within
+// this file.
+const CRLF CRLF_STATE;
+const String STRING_STATE;
+}
+
+const State&
+State::getInstance(ID state_id) {
+    switch (state_id) {
+    case CRLF:
+        return (CRLF_STATE);
+    case String:
+        return (STRING_STATE);
+    }
+
+    // This is a bug of the caller, and this method is only expected to be
+    // used by tests, so we just forcefully make it fail by asserting the
+    // condition.
+    assert(false);
+    return (STRING_STATE); // a dummy return, to silence some compilers.
+}
+
+const State*
+State::start(MasterLexer& lexer, MasterLexer::Options options) {
+    // define some shortcuts
+    MasterLexer::MasterLexerImpl& lexerimpl = *lexer.impl_;
+    size_t& paren_count = lexerimpl.paren_count_;
+
+    while (true) {
+        const int c = lexerimpl.skipComment(lexerimpl.source_->getChar());
+        if (c == InputSource::END_OF_STREAM) {
+            lexerimpl.last_was_eol_ = false;
+            if (paren_count != 0) {
+                lexerimpl.token_ = Token(Token::UNBALANCED_PAREN);
+                paren_count = 0; // reset to 0; this helps in lenient mode.
+                return (NULL);
+            }
+            lexerimpl.token_ = Token(Token::END_OF_FILE);
+            return (NULL);
+        } else if (c == ' ' || c == '\t') {
+            // If requested and we are not in (), recognize the initial space.
+            if (lexerimpl.last_was_eol_ && paren_count == 0 &&
+                (options & MasterLexer::INITIAL_WS) != 0) {
+                lexerimpl.last_was_eol_ = false;
+                lexerimpl.token_ = Token(Token::INITIAL_WS);
+                return (NULL);
+            }
+        } else if (c == '\n') {
+            lexerimpl.last_was_eol_ = true;
+            if (paren_count == 0) { // we don't recognize EOL if we are in ()
+                lexerimpl.token_ = Token(Token::END_OF_LINE);
+                return (NULL);
+            }
+        } else if (c == '\r') {
+            if (paren_count == 0) { // check if we are in () (see above)
+                return (&CRLF_STATE);
+            }
+        } else if (c == '(') {
+            lexerimpl.last_was_eol_ = false;
+            ++paren_count;
+        } else if (c == ')') {
+            lexerimpl.last_was_eol_ = false;
+            if (paren_count == 0) {
+                lexerimpl.token_ = Token(Token::UNBALANCED_PAREN);
+                return (NULL);
+            }
+            --paren_count;
+        } else {
+            // Note: in #2373 we should probably ungetChar().
+            lexerimpl.last_was_eol_ = false;
+            return (&STRING_STATE);
+        }
+        // no code should be here; we just continue the loop.
+    }
+}
+
+} // namespace master_lexer_internal
+
 } // end of namespace dns
 } // end of namespace isc

+ 28 - 1
src/lib/dns/master_lexer.h

@@ -24,6 +24,9 @@
 
 namespace isc {
 namespace dns {
+namespace master_lexer_internal {
+class State;
+}
 
 /// \brief Tokenizer for parsing DNS master files.
 ///
@@ -64,9 +67,22 @@ namespace dns {
 /// this class does not throw for an error that would be reported as an
 /// exception in other classes.
 class MasterLexer {
+    friend class master_lexer_internal::State;
 public:
     class Token;       // we define it separately for better readability
 
+    /// \brief Options for getNextToken.
+    ///
+    /// A compound option, indicating multiple options are set, can be
+    /// specified using the logical OR operator (operator|()).
+    enum Options {
+        NONE = 0,               ///< No option
+        INITIAL_WS = 1, ///< recognize begin-of-line spaces after an
+                        ///< end-of-line
+        QSTRING = 2,    ///< recognize quoted string
+        NUMBER = 4   ///< recognize numeric text as integer
+    };
+
     /// \brief The constructor.
     ///
     /// \throw std::bad_alloc Internal resource allocation fails (rare case).
@@ -167,6 +183,16 @@ private:
     MasterLexerImpl* impl_;
 };
 
+/// \brief Operator to combine \c MasterLexer options
+///
+/// This is a trivial shortcut so that compound options can be specified
+/// in an intuitive way.
+inline MasterLexer::Options
+operator|(MasterLexer::Options o1, MasterLexer::Options o2) {
+    return (static_cast<MasterLexer::Options>(
+                static_cast<unsigned>(o1) | static_cast<unsigned>(o2)));
+}
+
 /// \brief Tokens for \c MasterLexer
 ///
 /// This is a simple value-class encapsulating a type of a lexer token and
@@ -192,7 +218,8 @@ public:
     enum Type {
         END_OF_LINE, ///< End of line detected (if asked for detecting it)
         END_OF_FILE, ///< End of file detected (if asked for detecting it)
-        INITIAL_WS,  ///< White spaces at the beginning of a line
+        INITIAL_WS,  ///< White spaces at the beginning of a line after an
+                     ///< end of line
         NOVALUE_TYPE_MAX = INITIAL_WS, ///< Max integer corresponding to
                                        /// no-value (type only) types.
                                        /// Mainly for internal use.

+ 144 - 0
src/lib/dns/master_lexer_state.h

@@ -0,0 +1,144 @@
+// Copyright (C) 2012  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#ifndef MASTER_LEXER_STATE_H
+#define MASTER_LEXER_STATE_H 1
+
+#include <dns/master_lexer.h>
+
+namespace isc {
+namespace dns {
+
+namespace master_lexer_internal {
+
+/// \brief Tokenization state for \c MasterLexer.
+///
+/// This is a base class of classes that represent various states of a single
+/// tokenization session of \c MasterLexer, i.e., the states used for a
+/// single call to \c MasterLexer::getNextToken().
+///
+/// It follows the convention of the state design pattern: each derived class
+/// corresponds to a specific state, and the state transition takes place
+/// through the virtual method named \c handle().  The \c handle() method
+/// takes the main \c MasterLexer object that holds all necessary internal
+/// context, and updates it as necessary; each \c State derived class is
+/// completely stateless.
+///
+/// The initial transition takes place in a static method of the base class,
+/// \c start().  This is mainly for implementation convenience; we need to
+/// pass options given to \c MasterLexer::getNextToken() for the initial
+/// state, so it makes more sense to separate the interface for the transition
+/// from the initial state.
+///
+/// When an object of a specific state class completes the session, it
+/// normally sets the identified token in the lexer, and returns NULL;
+/// if more transition is necessary, it returns a pointer to the next state
+/// object.
+///
+/// As is usual in the state design pattern, the \c State class is made
+/// a friend class of \c MasterLexer and can refer to its internal details.
+/// This is intentional; essentially its a part of \c MasterLexer and
+/// is defined as a separate class only for implementation clarity and better
+/// testability.  It's defined in a publicly visible header, but that's only
+/// for testing purposes.  No normal application or even no other classes of
+/// this library are expected to use this class.
+class State {
+public:
+    /// \brief Virtual destructor.
+    ///
+    /// In our usage this actually doesn't matter, but some compilers complain
+    /// about it and we need to silence them.
+    virtual ~State() {}
+
+    /// \brief Begin state transitions to get the next token.
+    ///
+    /// This is the first method that \c MasterLexer needs to call for a
+    /// tokenization session.  The lexer passes a reference to itself
+    /// and options given in \c getNextToken().
+    ///
+    /// \throw InputSource::ReadError Unexpected I/O error
+    /// \throw std::bad_alloc Internal resource allocation failure
+    ///
+    /// \param lexer The lexer object that holds the main context.
+    /// \param options The options passed to getNextToken().
+    /// \return A pointer to the next state object or NULL if the transition
+    /// is completed.
+    static const State* start(MasterLexer& lexer,
+                              MasterLexer::Options options);
+
+    /// \brief Handle the process of one specific state.
+    ///
+    /// This method is expected to be called on the object returned by
+    /// start(), and keep called on the returned object until NULL is
+    /// returned.  The call chain will form the complete state transition.
+    ///
+    /// \throw InputSource::ReadError Unexpected I/O error
+    /// \throw std::bad_alloc Internal resource allocation failure
+    ///
+    /// \param lexer The lexer object that holds the main context.
+    /// \return A pointer to the next state object or NULL if the transition
+    /// is completed.
+    virtual const State* handle(MasterLexer& lexer) const = 0;
+
+    /// \brief Types of states.
+    ///
+    /// Specific states are basically hidden within the implementation,
+    /// but we'd like to allow tests to examine them, so we provide
+    /// a way to get an instance of a specific state.
+    enum ID {
+        CRLF,                  ///< Just seen a carriage-return character
+        String                 ///< Handling a string token
+    };
+
+    /// \brief Returns a \c State instance of the given state.
+    ///
+    /// This is provided only for testing purposes so tests can check
+    /// the behavior of each state separately.  \c MasterLexer shouldn't
+    /// need this method.
+    static const State& getInstance(ID state_id);
+
+    /// \name Read-only accessors for testing purposes.
+    ///
+    /// These allow tests to inspect some selected portion of the internal
+    /// states of \c MasterLexer.  These shouldn't be used except for testing
+    /// purposes.
+    ///@{
+    bool wasLastEOL(const MasterLexer& lexer) const;
+    const MasterLexer::Token& getToken(const MasterLexer& lexer) const;
+    size_t getParenCount(const MasterLexer& lexer) const;
+    ///@}
+
+protected:
+    /// \brief An accessor to the internal implementation class of
+    /// \c MasterLexer.
+    ///
+    /// This is provided for specific derived classes as they are not direct
+    /// friends of \c MasterLexer.
+    ///
+    /// \param lexer The lexer object that holds the main context.
+    /// \return A pointer to the implementation class object of the given
+    /// lexer.  This is never NULL.
+    MasterLexer::MasterLexerImpl* getLexerImpl(MasterLexer& lexer) const {
+        return (lexer.impl_);
+    }
+};
+
+} // namespace master_lexer_internal
+} // namespace dns
+} // namespace isc
+#endif  // MASTER_LEXER_STATE_H
+
+// Local Variables:
+// mode: c++
+// End:

+ 1 - 0
src/lib/dns/tests/Makefile.am

@@ -27,6 +27,7 @@ run_unittests_SOURCES += labelsequence_unittest.cc
 run_unittests_SOURCES += messagerenderer_unittest.cc
 run_unittests_SOURCES += master_lexer_token_unittest.cc
 run_unittests_SOURCES += master_lexer_unittest.cc
+run_unittests_SOURCES += master_lexer_state_unittest.cc
 run_unittests_SOURCES += name_unittest.cc
 run_unittests_SOURCES += nsec3hash_unittest.cc
 run_unittests_SOURCES += rrclass_unittest.cc rrtype_unittest.cc

+ 256 - 0
src/lib/dns/tests/master_lexer_state_unittest.cc

@@ -0,0 +1,256 @@
+// Copyright (C) 2012  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <dns/master_lexer.h>
+#include <dns/master_lexer_inputsource.h>
+#include <dns/master_lexer_state.h>
+
+#include <gtest/gtest.h>
+
+#include <sstream>
+
+using namespace isc::dns;
+using namespace master_lexer_internal;
+
+namespace {
+typedef MasterLexer::Token Token; // shortcut
+
+class MasterLexerStateTest : public ::testing::Test {
+protected:
+    MasterLexerStateTest() : common_options(MasterLexer::INITIAL_WS),
+                             s_null(NULL),
+                             s_crlf(State::getInstance(State::CRLF)),
+                             s_string(State::getInstance(State::String)),
+                             options(MasterLexer::NONE),
+                             orig_options(options)
+    {}
+
+    // Specify INITIAL_WS as common initial options.
+    const MasterLexer::Options common_options;
+    MasterLexer lexer;
+    const State* const s_null;
+    const State& s_crlf;
+    const State& s_string;
+    std::stringstream ss;
+    MasterLexer::Options options, orig_options;
+};
+
+// Common check for the end-of-file condition.
+// Token is set to END_OF_FILE, and the lexer was NOT last eol state.
+// Passed state can be any valid one; they are stateless, just providing the
+// interface for inspection.
+void
+eofCheck(const State& state, MasterLexer& lexer) {
+    EXPECT_EQ(Token::END_OF_FILE, state.getToken(lexer).getType());
+    EXPECT_FALSE(state.wasLastEOL(lexer));
+}
+
+TEST_F(MasterLexerStateTest, startAndEnd) {
+    // A simple case: the input is empty, so we begin with start and
+    // are immediately done.
+    lexer.pushSource(ss);
+    EXPECT_EQ(s_null, State::start(lexer, common_options));
+    eofCheck(s_crlf, lexer);
+}
+
+TEST_F(MasterLexerStateTest, startToEOL) {
+    ss << "\n";
+    lexer.pushSource(ss);
+
+    EXPECT_EQ(s_null, State::start(lexer, common_options));
+    EXPECT_TRUE(s_crlf.wasLastEOL(lexer));
+    EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
+
+    // The next lexer session will reach EOF.  Same eof check should pass.
+    EXPECT_EQ(s_null, State::start(lexer, common_options));
+    eofCheck(s_crlf, lexer);
+}
+
+TEST_F(MasterLexerStateTest, space) {
+    // repeat '\t\n' twice (see below), then space after EOL
+    ss << " \t\n\t\n ";
+    lexer.pushSource(ss);
+
+    // by default space characters and tabs will be ignored.  We check this
+    // twice; at the second iteration, it's a white space at the beginning
+    // of line, but since we don't specify INITIAL_WS option, it's treated as
+    // normal space and ignored.
+    for (size_t i = 0; i < 2; ++i) {
+        EXPECT_EQ(s_null, State::start(lexer, MasterLexer::NONE));
+        EXPECT_TRUE(s_crlf.wasLastEOL(lexer));
+        EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
+    }
+
+    // Now we specify the INITIAL_WS option.  It will be recognized and the
+    // corresponding token will be returned.
+    EXPECT_EQ(s_null, State::start(lexer, MasterLexer::INITIAL_WS));
+    EXPECT_FALSE(s_crlf.wasLastEOL(lexer));
+    EXPECT_EQ(Token::INITIAL_WS, s_crlf.getToken(lexer).getType());
+}
+
+TEST_F(MasterLexerStateTest, parentheses) {
+    ss << "\n(\na\n )\n "; // 1st \n is to check if 'was EOL' is set to false
+    lexer.pushSource(ss);
+
+    EXPECT_EQ(s_null, State::start(lexer, common_options)); // handle \n
+
+    // Now handle '('.  It skips \n and recognize 'a' as string
+    EXPECT_EQ(0, s_crlf.getParenCount(lexer)); // check pre condition
+    EXPECT_EQ(&s_string, State::start(lexer, common_options));
+    EXPECT_EQ(1, s_crlf.getParenCount(lexer)); // check post condition
+    EXPECT_FALSE(s_crlf.wasLastEOL(lexer));
+
+    // skip 'a' (note: until #2373 it's actually skipped as part of the '('
+    // handling)
+    s_string.handle(lexer);
+
+    // Then handle ')'.  '\n' before ')' isn't recognized because
+    // it's canceled due to the '('.  Likewise, the space after the '\n'
+    // shouldn't be recognized but should be just ignored.
+    EXPECT_EQ(s_null, State::start(lexer, common_options));
+    EXPECT_EQ(0, s_crlf.getParenCount(lexer));
+
+    // Now, temporarily disabled options are restored: Both EOL and the
+    // initial WS are recognized
+    EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
+    EXPECT_EQ(s_null, State::start(lexer, common_options));
+    EXPECT_EQ(Token::INITIAL_WS, s_crlf.getToken(lexer).getType());
+}
+
+TEST_F(MasterLexerStateTest, nestedParentheses) {
+    // This is an unusual, but allowed (in this implementation) case.
+    ss << "(a(b)\n c)\n ";
+    lexer.pushSource(ss);
+
+    EXPECT_EQ(&s_string, State::start(lexer, common_options)); // consume '('
+    s_string.handle(lexer);                      // consume 'a'
+    EXPECT_EQ(&s_string, State::start(lexer, common_options)); // consume '('
+    s_string.handle(lexer);                     // consume 'b'
+    EXPECT_EQ(2, s_crlf.getParenCount(lexer)); // now the count is 2
+
+    // Close the inner most parentheses.  count will be decreased, but option
+    // shouldn't be restored yet, so the intermediate EOL or initial WS won't
+    // be recognized.
+    EXPECT_EQ(&s_string, State::start(lexer, common_options)); // consume ')'
+    s_string.handle(lexer);                      // consume 'c'
+    EXPECT_EQ(1, s_crlf.getParenCount(lexer));
+
+    // Close the outermost parentheses.  count will be reset to 0, and original
+    // options are restored.
+    EXPECT_EQ(s_null, State::start(lexer, common_options));
+
+    // Now, temporarily disabled options are restored: Both EOL and the
+    // initial WS are recognized
+    EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
+    EXPECT_EQ(s_null, State::start(lexer, common_options));
+    EXPECT_EQ(Token::INITIAL_WS, s_crlf.getToken(lexer).getType());
+}
+
+TEST_F(MasterLexerStateTest, unbalancedParentheses) {
+    // Only closing paren is provided.  We prepend a \n to check if it's
+    // correctly canceled after detecting the error.
+    ss << "\n)";
+    ss << "(a";
+    lexer.pushSource(ss);
+
+    EXPECT_EQ(s_null, State::start(lexer, common_options)); // consume '\n'
+    EXPECT_TRUE(s_crlf.wasLastEOL(lexer)); // this \n was remembered
+
+    // Now checking ')'.  The result should be error, count shouldn't be
+    // changed.  "last EOL" should be canceled.
+    EXPECT_EQ(0, s_crlf.getParenCount(lexer));
+    EXPECT_EQ(s_null, State::start(lexer, common_options));
+    EXPECT_EQ(0, s_crlf.getParenCount(lexer));
+    ASSERT_EQ(Token::ERROR, s_crlf.getToken(lexer).getType());
+    EXPECT_EQ(Token::UNBALANCED_PAREN, s_crlf.getToken(lexer).getErrorCode());
+    EXPECT_FALSE(s_crlf.wasLastEOL(lexer));
+
+    // Reach EOF with a dangling open parenthesis.
+    EXPECT_EQ(&s_string, State::start(lexer, common_options)); // consume '('
+    s_string.handle(lexer);                      // consume 'a'
+    EXPECT_EQ(1, s_crlf.getParenCount(lexer));
+    EXPECT_EQ(s_null, State::start(lexer, common_options));    // reach EOF
+    ASSERT_EQ(Token::ERROR, s_crlf.getToken(lexer).getType());
+    EXPECT_EQ(Token::UNBALANCED_PAREN, s_crlf.getToken(lexer).getErrorCode());
+    EXPECT_EQ(0, s_crlf.getParenCount(lexer)); // should be reset to 0
+}
+
+TEST_F(MasterLexerStateTest, startToComment) {
+    // Begin with 'start', skip space, then encounter a comment.  Skip
+    // the rest of the line, and recognize the new line.  Note that the
+    // second ';' is simply ignored.
+    ss << "  ;a;\n";
+    ss << ";a;";           // Likewise, but the comment ends with EOF.
+    lexer.pushSource(ss);
+
+    // Comment ending with EOL
+    EXPECT_EQ(s_null, State::start(lexer, common_options));
+    EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
+
+    // Comment ending with EOF
+    EXPECT_EQ(s_null, State::start(lexer, common_options));
+    EXPECT_EQ(Token::END_OF_FILE, s_crlf.getToken(lexer).getType());
+}
+
+TEST_F(MasterLexerStateTest, commentAfterParen) {
+    // comment after an opening parenthesis.  The code that is tested by
+    // other tests should also ensure that it works correctly, but we
+    // check it explicitly.
+    ss << "( ;this is a comment\na)\n";
+    lexer.pushSource(ss);
+
+    // consume '(', skip comments, consume 'a', then consume ')'
+    EXPECT_EQ(&s_string, State::start(lexer, common_options));
+    s_string.handle(lexer);
+    EXPECT_EQ(s_null, State::start(lexer, common_options));
+    EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
+}
+
+TEST_F(MasterLexerStateTest, crlf) {
+    ss << "\r\n";               // case 1
+    ss << "\r ";                // case 2
+    ss << "\r;comment\na";      // case 3
+    ss << "\r";                 // case 4
+    lexer.pushSource(ss);
+
+    // 1. A sequence of \r, \n is recognized as a single 'end-of-line'
+    EXPECT_EQ(&s_crlf, State::start(lexer, common_options)); // recognize '\r'
+    EXPECT_EQ(s_null, s_crlf.handle(lexer));   // recognize '\n'
+    EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
+    EXPECT_TRUE(s_crlf.wasLastEOL(lexer));
+
+    // 2. Single '\r' (not followed by \n) is recognized as a single
+    // 'end-of-line'.  then there will be "initial WS"
+    EXPECT_EQ(&s_crlf, State::start(lexer, common_options)); // recognize '\r'
+    // see ' ', "unget" it
+    EXPECT_EQ(s_null, s_crlf.handle(lexer));
+    EXPECT_EQ(s_null, State::start(lexer, common_options)); // recognize ' '
+    EXPECT_EQ(Token::INITIAL_WS, s_crlf.getToken(lexer).getType());
+
+    // 3. comment between \r and \n
+    EXPECT_EQ(&s_crlf, State::start(lexer, common_options)); // recognize '\r'
+    // skip comments, recognize '\n'
+    EXPECT_EQ(s_null, s_crlf.handle(lexer));
+    EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
+    EXPECT_EQ(&s_string, State::start(lexer, common_options));
+
+    // 4. \r then EOF
+    EXPECT_EQ(&s_crlf, State::start(lexer, common_options)); // recognize '\r'
+    // see EOF, then "unget" it
+    EXPECT_EQ(s_null, s_crlf.handle(lexer));
+    EXPECT_EQ(s_null, State::start(lexer, common_options));  // recognize EOF
+    EXPECT_EQ(Token::END_OF_FILE, s_crlf.getToken(lexer).getType());
+}
+
+}

+ 0 - 2
tests/tools/perfdhcp/templates/Makefile.am

@@ -5,8 +5,6 @@ SUBDIRS = .
 CLEANFILES = test1.hex test2.hex test3.hex test4.hex test5.hex
 
 perfdhcpdir = $(pkgdatadir)
-perfdhcp_DATA = discover-example.hex request4-example.hex
-perfdhcp_DATA += solicit-example.hex request6-example.hex
 
 EXTRA_DIST = discover-example.hex request4-example.hex
 EXTRA_DIST += solicit-example.hex request6-example.hex