Merge branch 'master' into trac253

ChangeLog

@@ -1,9 +1,30 @@
+  174.	[bug]*		jinmei
+	src/lib/dns: revised dnssectime functions so that they don't rely
+	on the time_t type (whose size varies on different systems, which
+	can lead to subtle bugs like some form of "year 2038 problem").
+	Also handled 32-bit wrap around issues more explicitly, with more
+	detailed tests.  The function API has been changed, but the effect
+	should be minimal because these functions are mostly private.
+	(Trac #61, git 09ece8cdd41c0f025e8b897b4883885d88d4ba5d)
+
+  173.	[bug]		jerry
+	python/isc/notify: A notify_out test fails without network
+	connectivity, encapsulate the socket behavior using a mock
+	socket class to fix it.
+	(Trac #346, git 319debfb957641f311102739a15059f8453c54ce)
+
+  172.  [func]      jelte
+	Improved the bindctl cli in various ways, mainly concerning
+	list and map item addressing, the correct display of actual values,
+	and internal help.
+	(Trac #384, git e5fb3bc1ed5f3c0aec6eb40a16c63f3d0fc6a7b2)
+
   171.  [func]      feng, jerry, jinmei, vorner
 	b10-auth, src/lib/datasrc: in memory data source now works as a
 	complete data source for authoritative DNS servers and b10-auth
 	uses it.  It still misses major features, however, including
 	DNSSEC support and zone transfer.
-	(Last trac #552, but many more,
+	(Last trac #553, but many more,
 	git 6f031a09a248e7684723c000f3e8cc981dcdb349)
 
   170.	[bug]		jinmei

src/bin/auth/auth.spec.pre.in

@@ -12,45 +12,44 @@
         "item_type": "list",
         "item_optional": true,
         "item_default": [],
-	"list_item_spec": {
-          "item_name": "list_element",
+        "list_item_spec":
+        { "item_name": "list_element",
           "item_type": "map",
           "item_optional": false,
           "item_default": {},
-	  "map_item_spec": [
-	    { "item_name": "type",
-	      "item_type": "string",
-	      "item_optional": false,
-	      "item_default": ""
-	    },
-	    { "item_name": "class",
-	      "item_type": "string",
-	      "item_optional": false,
-	      "item_default": "IN"
-	    },
-	    { "item_name": "zones",
-	      "item_type": "list",
-	      "item_optional": false,
-	      "item_default": [],
-	      "list_item_spec": {
-	        "item_name": "list_element",
-	        "item_type": "map",
-	        "item_optional": true,
-	        "map_item_spec": [
-		  { "item_name": "origin",
-		    "item_type": "string",
-		    "item_optional": false,
-		    "item_default": ""
-		  },
-		  { "item_name": "file",
-		    "item_type": "string",
-		    "item_optional": false,
-		    "item_default": ""
-		  }
-		]
-	      }
-	    }
-	  ]
+          "map_item_spec": [
+          { "item_name": "type",
+            "item_type": "string",
+            "item_optional": false,
+            "item_default": ""
+          },
+          { "item_name": "class",
+            "item_type": "string",
+            "item_optional": false,
+            "item_default": "IN"
+          },
+          { "item_name": "zones",
+            "item_type": "list",
+            "item_optional": false,
+            "item_default": [],
+            "list_item_spec":
+            { "item_name": "list_element",
+              "item_type": "map",
+              "item_optional": true,
+              "item_default": { "origin": "", "file": "" },
+              "map_item_spec": [
+              { "item_name": "origin",
+                "item_type": "string",
+                "item_optional": false,
+                "item_default": ""
+              },
+              { "item_name": "file",
+                "item_type": "string",
+                "item_optional": false,
+                "item_default": ""
+              }]
+            }
+          }]
         }
       },
       { "item_name": "statistics-interval",

src/bin/bindctl/bindcmd.py

@@ -51,7 +51,6 @@ except ImportError:
     my_readline = sys.stdin.readline
 
 CSV_FILE_NAME = 'default_user.csv'
-FAIL_TO_CONNECT_WITH_CMDCTL = "Fail to connect with b10-cmdctl module, is it running?"
 CONFIG_MODULE_NAME = 'config'
 CONST_BINDCTL_HELP = """
 usage: <module name> <command name> [param1 = value1 [, param2 = value2]]
@@ -92,10 +91,13 @@ class BindCmdInterpreter(Cmd):
         Cmd.__init__(self)
         self.location = ""
         self.prompt_end = '> '
-        self.prompt = self.prompt_end
+        if sys.stdin.isatty():
+            self.prompt = self.prompt_end
+        else:
+            self.prompt = ""
         self.ruler = '-'
         self.modules = OrderedDict()
-        self.add_module_info(ModuleInfo("help", desc = "Get help for bindctl"))
+        self.add_module_info(ModuleInfo("help", desc = "Get help for bindctl."))
         self.server_port = server_port
         self.conn = ValidatedHTTPSConnection(self.server_port,
                                              ca_certs=pem_file)
@@ -119,8 +121,8 @@ class BindCmdInterpreter(Cmd):
 
             self.cmdloop()
         except FailToLogin as err:
-            print(err)
-            print(FAIL_TO_CONNECT_WITH_CMDCTL)
+            # error already printed when this was raised, ignoring
+            pass
         except KeyboardInterrupt:
             print('\nExit from bindctl')
 
@@ -270,8 +272,10 @@ class BindCmdInterpreter(Cmd):
         return line 
 
     def postcmd(self, stop, line):
-        '''Update the prompt after every command'''
-        self.prompt = self.location + self.prompt_end
+        '''Update the prompt after every command, but only if we
+           have a tty as output'''
+        if sys.stdin.isatty():
+            self.prompt = self.location + self.prompt_end
         return stop
 
     def _prepare_module_commands(self, module_spec):
@@ -375,7 +379,14 @@ class BindCmdInterpreter(Cmd):
         if cmd.command == "help" or ("help" in cmd.params.keys()):
             self._handle_help(cmd)
         elif cmd.module == CONFIG_MODULE_NAME:
-            self.apply_config_cmd(cmd)
+            try:
+                self.apply_config_cmd(cmd)
+            except isc.cc.data.DataTypeError as dte:
+                print("Error: " + str(dte))
+            except isc.cc.data.DataNotFoundError as dnfe:
+                print("Error: " + str(dnfe))
+            except KeyError as ke:
+                print("Error: missing " + str(ke))
         else:
             self.apply_cmd(cmd)
 
@@ -396,9 +407,24 @@ class BindCmdInterpreter(Cmd):
 
     def do_help(self, name):
         print(CONST_BINDCTL_HELP)
-        for k in self.modules.keys():
-            print("\t", self.modules[k])
-                
+        for k in self.modules.values():
+            n = k.get_name()
+            if len(n) >= CONST_BINDCTL_HELP_INDENT_WIDTH:
+                print("    %s" % n)
+                print(textwrap.fill(k.get_desc(),
+                      initial_indent="            ",
+                      subsequent_indent="    " +
+                      " " * CONST_BINDCTL_HELP_INDENT_WIDTH,
+                      width=70))
+            else:
+                print(textwrap.fill("%s%s%s" %
+                    (k.get_name(),
+                     " "*(CONST_BINDCTL_HELP_INDENT_WIDTH - len(k.get_name())),
+                     k.get_desc()),
+                    initial_indent="    ",
+                    subsequent_indent="    " +
+                    " " * CONST_BINDCTL_HELP_INDENT_WIDTH,
+                    width=70))
     
     def onecmd(self, line):
         if line == 'EOF' or line.lower() == "quit":
@@ -411,7 +437,19 @@ class BindCmdInterpreter(Cmd):
         Cmd.onecmd(self, line)
 
     def remove_prefix(self, list, prefix):
-        return [(val[len(prefix):]) for val in list]
+        """Removes the prefix already entered, and all elements from the
+           list that don't match it"""
+        if prefix.startswith('/'):
+            prefix = prefix[1:]
+
+        new_list = []
+        for val in list:
+            if val.startswith(prefix):
+                new_val = val[len(prefix):]
+                if new_val.startswith("/"):
+                    new_val = new_val[1:]
+                new_list.append(new_val)
+        return new_list
 
     def complete(self, text, state):
         if 0 == state:
@@ -502,8 +540,7 @@ class BindCmdInterpreter(Cmd):
             self._validate_cmd(cmd)
             self._handle_cmd(cmd)
         except (IOError, http.client.HTTPException) as err:
-            print('Error!', err)
-            print(FAIL_TO_CONNECT_WITH_CMDCTL)
+            print('Error: ', err)
         except BindCtlException as err:
             print("Error! ", err)
             self._print_correct_usage(err)
@@ -541,87 +578,115 @@ class BindCmdInterpreter(Cmd):
            Raises a KeyError if the command was not complete
         '''
         identifier = self.location
-        try:
-            if 'identifier' in cmd.params:
-                if not identifier.endswith("/"):
-                    identifier += "/"
-                if cmd.params['identifier'].startswith("/"):
-                    identifier = cmd.params['identifier']
-                else:
-                    identifier += cmd.params['identifier']
-
-                # Check if the module is known; for unknown modules
-                # we currently deny setting preferences, as we have
-                # no way yet to determine if they are ok.
-                module_name = identifier.split('/')[1]
-                if self.config_data is None or \
-                   not self.config_data.have_specification(module_name):
-                    print("Error: Module '" + module_name + "' unknown or not running")
-                    return
+        if 'identifier' in cmd.params:
+            if not identifier.endswith("/"):
+                identifier += "/"
+            if cmd.params['identifier'].startswith("/"):
+                identifier = cmd.params['identifier']
+            else:
+                if cmd.params['identifier'].startswith('['):
+                    identifier = identifier[:-1]
+                identifier += cmd.params['identifier']
+
+            # Check if the module is known; for unknown modules
+            # we currently deny setting preferences, as we have
+            # no way yet to determine if they are ok.
+            module_name = identifier.split('/')[1]
+            if module_name != "" and (self.config_data is None or \
+               not self.config_data.have_specification(module_name)):
+                print("Error: Module '" + module_name + "' unknown or not running")
+                return
 
-            if cmd.command == "show":
-                values = self.config_data.get_value_maps(identifier)
-                for value_map in values:
-                    line = value_map['name']
-                    if value_map['type'] in [ 'module', 'map', 'list' ]:
-                        line += "/"
-                    else:
-                        line += ":\t" + json.dumps(value_map['value'])
-                    line += "\t" + value_map['type']
-                    line += "\t"
-                    if value_map['default']:
-                        line += "(default)"
-                    if value_map['modified']:
-                        line += "(modified)"
-                    print(line)
-            elif cmd.command == "add":
-                self.config_data.add_value(identifier, cmd.params['value'])
-            elif cmd.command == "remove":
-                if 'value' in cmd.params:
-                    self.config_data.remove_value(identifier, cmd.params['value'])
+        if cmd.command == "show":
+            # check if we have the 'all' argument
+            show_all = False
+            if 'argument' in cmd.params:
+                if cmd.params['argument'] == 'all':
+                    show_all = True
+                elif 'identifier' not in cmd.params:
+                    # no 'all', no identifier, assume this is the
+                    #identifier
+                    identifier += cmd.params['argument']
                 else:
-                    self.config_data.remove_value(identifier, None)
-            elif cmd.command == "set":
-                if 'identifier' not in cmd.params:
-                    print("Error: missing identifier or value")
+                    print("Error: unknown argument " + cmd.params['argument'] + ", or multiple identifiers given")
+                    return
+            values = self.config_data.get_value_maps(identifier, show_all)
+            for value_map in values:
+                line = value_map['name']
+                if value_map['type'] in [ 'module', 'map' ]:
+                    line += "/"
+                elif value_map['type'] == 'list' \
+                     and value_map['value'] != []:
+                    # do not print content of non-empty lists if
+                    # we have more data to show
+                    line += "/"
                 else:
-                    parsed_value = None
-                    try:
-                        parsed_value = json.loads(cmd.params['value'])
-                    except Exception as exc:
-                        # ok could be an unquoted string, interpret as such
-                        parsed_value = cmd.params['value']
-                    self.config_data.set_value(identifier, parsed_value)
-            elif cmd.command == "unset":
-                self.config_data.unset(identifier)
-            elif cmd.command == "revert":
-                self.config_data.clear_local_changes()
-            elif cmd.command == "commit":
-                self.config_data.commit()
-            elif cmd.command == "diff":
-                print(self.config_data.get_local_changes());
-            elif cmd.command == "go":
-                self.go(identifier)
-        except isc.cc.data.DataTypeError as dte:
-            print("Error: " + str(dte))
-        except isc.cc.data.DataNotFoundError as dnfe:
-            print("Error: " + identifier + " not found")
-        except KeyError as ke:
-            print("Error: missing " + str(ke))
-            raise ke
+                    line += "\t" + json.dumps(value_map['value'])
+                line += "\t" + value_map['type']
+                line += "\t"
+                if value_map['default']:
+                    line += "(default)"
+                if value_map['modified']:
+                    line += "(modified)"
+                print(line)
+        elif cmd.command == "show_json":
+            if identifier == "":
+                print("Need at least the module to show the configuration in JSON format")
+            else:
+                data, default = self.config_data.get_value(identifier)
+                print(json.dumps(data))
+        elif cmd.command == "add":
+            if 'value' in cmd.params:
+                self.config_data.add_value(identifier, cmd.params['value'])
+            else:
+                self.config_data.add_value(identifier)
+        elif cmd.command == "remove":
+            if 'value' in cmd.params:
+                self.config_data.remove_value(identifier, cmd.params['value'])
+            else:
+                self.config_data.remove_value(identifier, None)
+        elif cmd.command == "set":
+            if 'identifier' not in cmd.params:
+                print("Error: missing identifier or value")
+            else:
+                parsed_value = None
+                try:
+                    parsed_value = json.loads(cmd.params['value'])
+                except Exception as exc:
+                    # ok could be an unquoted string, interpret as such
+                    parsed_value = cmd.params['value']
+                self.config_data.set_value(identifier, parsed_value)
+        elif cmd.command == "unset":
+            self.config_data.unset(identifier)
+        elif cmd.command == "revert":
+            self.config_data.clear_local_changes()
+        elif cmd.command == "commit":
+            self.config_data.commit()
+        elif cmd.command == "diff":
+            print(self.config_data.get_local_changes());
+        elif cmd.command == "go":
+            self.go(identifier)
 
     def go(self, identifier):
         '''Handles the config go command, change the 'current' location
-           within the configuration tree'''
-        # this is just to see if it exists
-        self.config_data.get_value(identifier)
-        # some sanitizing
-        identifier = identifier.replace("//", "/")
-        if not identifier.startswith("/"):
-            identifier = "/" + identifier
-        if identifier.endswith("/"):
-            identifier = identifier[:-1]
-        self.location = identifier
+           within the configuration tree. '..' will be interpreted as
+           'up one level'.'''
+        id_parts = isc.cc.data.split_identifier(identifier)
+
+        new_location = ""
+        for id_part in id_parts:
+            if (id_part == ".."):
+                # go 'up' one level
+                new_location, a, b = new_location.rpartition("/")
+            else:
+                new_location += "/" + id_part
+        # check if exists, if not, revert and error
+        v,d = self.config_data.get_value(new_location)
+        if v is None:
+            print("Error: " + identifier + " not found")
+            return
+
+        self.location = new_location
 
     def apply_cmd(self, cmd):
         '''Handles a general module command'''

src/bin/bindctl/bindctl-source.py.in

@@ -33,51 +33,60 @@ isc.util.process.rename()
 # number, and the overall BIND 10 version number (set in configure.ac).
 VERSION = "bindctl 20101201 (BIND 10 @PACKAGE_VERSION@)"
 
+DEFAULT_IDENTIFIER_DESC = "The identifier specifies the config item. Child elements are separated with the '/' character. List indices can be specified with '[i]', where i is an integer specifying the index, starting with 0. Examples: 'Boss/start_auth', 'Recurse/listen_on[0]/address'. If no identifier is given, shows the item at the current location."
+
 def prepare_config_commands(tool):
     '''Prepare fixed commands for local configuration editing'''
-    module = ModuleInfo(name = CONFIG_MODULE_NAME, desc = "Configuration commands")
-    cmd = CommandInfo(name = "show", desc = "Show configuration")
-    param = ParamInfo(name = "identifier", type = "string", optional=True)
+    module = ModuleInfo(name = CONFIG_MODULE_NAME, desc = "Configuration commands.")
+    cmd = CommandInfo(name = "show", desc = "Show configuration.")
+    param = ParamInfo(name = "argument", type = "string", optional=True, desc = "If you specify the argument 'all' (before the identifier), recursively show all child elements for the given identifier.")
+    cmd.add_param(param)
+    param = ParamInfo(name = "identifier", type = "string", optional=True, desc = DEFAULT_IDENTIFIER_DESC)
+    cmd.add_param(param)
+    module.add_command(cmd)
+
+    cmd = CommandInfo(name = "show_json", desc = "Show full configuration in JSON format.")
+    param = ParamInfo(name = "identifier", type = "string", optional=True, desc = DEFAULT_IDENTIFIER_DESC)
     cmd.add_param(param)
     module.add_command(cmd)
 
-    cmd = CommandInfo(name = "add", desc = "Add entry to configuration list")
-    param = ParamInfo(name = "identifier", type = "string", optional=True)
+    cmd = CommandInfo(name = "add", desc = "Add an entry to configuration list. If no value is given, a default value is added.")
+    param = ParamInfo(name = "identifier", type = "string", optional=True, desc = DEFAULT_IDENTIFIER_DESC)
     cmd.add_param(param)
-    param = ParamInfo(name = "value", type = "string", optional=False)
+    param = ParamInfo(name = "value", type = "string", optional=True, desc = "Specifies a value to add to the list. It must be in correct JSON format and complete.")
     cmd.add_param(param)
     module.add_command(cmd)
 
-    cmd = CommandInfo(name = "remove", desc = "Remove entry from configuration list")
-    param = ParamInfo(name = "identifier", type = "string", optional=True)
+    cmd = CommandInfo(name = "remove", desc = "Remove entry from configuration list.")
+    param = ParamInfo(name = "identifier", type = "string", optional=True, desc = DEFAULT_IDENTIFIER_DESC)
     cmd.add_param(param)
-    param = ParamInfo(name = "value", type = "string", optional=True)
+    param = ParamInfo(name = "value", type = "string", optional=True, desc = "Specifies a value to remove from the list. It must be in correct JSON format and complete.")
     cmd.add_param(param)
     module.add_command(cmd)
 
-    cmd = CommandInfo(name = "set", desc = "Set a configuration value")
-    param = ParamInfo(name = "identifier", type = "string", optional=True)
+    cmd = CommandInfo(name = "set", desc = "Set a configuration value.")
+    param = ParamInfo(name = "identifier", type = "string", optional=True, desc = DEFAULT_IDENTIFIER_DESC)
     cmd.add_param(param)
-    param = ParamInfo(name = "value", type = "string", optional=False)
+    param = ParamInfo(name = "value", type = "string", optional=False, desc = "Specifies a value to set. It must be in correct JSON format and complete.")
     cmd.add_param(param)
     module.add_command(cmd)
 
-    cmd = CommandInfo(name = "unset", desc = "Unset a configuration value")
-    param = ParamInfo(name = "identifier", type = "string", optional=False)
+    cmd = CommandInfo(name = "unset", desc = "Unset a configuration value (i.e. revert to the default, if any).")
+    param = ParamInfo(name = "identifier", type = "string", optional=False, desc = DEFAULT_IDENTIFIER_DESC)
     cmd.add_param(param)
     module.add_command(cmd)
 
-    cmd = CommandInfo(name = "diff", desc = "Show all local changes")
+    cmd = CommandInfo(name = "diff", desc = "Show all local changes that have not been committed.")
     module.add_command(cmd)
 
-    cmd = CommandInfo(name = "revert", desc = "Revert all local changes")
+    cmd = CommandInfo(name = "revert", desc = "Revert all local changes.")
     module.add_command(cmd)
 
-    cmd = CommandInfo(name = "commit", desc = "Commit all local changes")
+    cmd = CommandInfo(name = "commit", desc = "Commit all local changes.")
     module.add_command(cmd)
 
-    cmd = CommandInfo(name = "go", desc = "Go to a specific configuration part")
-    param = ParamInfo(name = "identifier", type="string", optional=False)
+    cmd = CommandInfo(name = "go", desc = "Go to a specific configuration part.")
+    param = ParamInfo(name = "identifier", type="string", optional=False, desc = DEFAULT_IDENTIFIER_DESC)
     cmd.add_param(param)
     module.add_command(cmd)
 
@@ -115,15 +124,12 @@ def set_bindctl_options(parser):
                       help = 'PEM formatted server certificate validation chain file')
 
 if __name__ == '__main__':
-    try:
-        parser = OptionParser(version = VERSION)
-        set_bindctl_options(parser)
-        (options, args) = parser.parse_args()
-        server_addr = options.addr + ':' + str(options.port)
-        tool = BindCmdInterpreter(server_addr, pem_file=options.cert_chain)
-        prepare_config_commands(tool)
-        tool.run()
-    except Exception as e:
-        print(e, "\nFailed to connect with b10-cmdctl module, is it running?")
+    parser = OptionParser(version = VERSION)
+    set_bindctl_options(parser)
+    (options, args) = parser.parse_args()
+    server_addr = options.addr + ':' + str(options.port)
+    tool = BindCmdInterpreter(server_addr, pem_file=options.cert_chain)
+    prepare_config_commands(tool)
+    tool.run()
 
 

src/bin/bindctl/cmdparse.py

@@ -33,6 +33,7 @@ param_value_str  = "(?P<param_value>[^\'\" ][^, ]+)"
 param_value_with_quota_str  = "[\"\'](?P<param_value>.+?)(?<!\\\)[\"\']"
 next_params_str = "(?P<blank>\s*)(?P<comma>,?)(?P<next_params>.*)$"
 
+
 PARAM_WITH_QUOTA_PATTERN = re.compile(param_name_str + 
                                       param_value_with_quota_str + 
                                       next_params_str)
@@ -40,8 +41,58 @@ PARAM_PATTERN = re.compile(param_name_str + param_value_str + next_params_str)
 # Used for module and command name
 NAME_PATTERN = re.compile("^\s*(?P<name>[\w]+)(?P<blank>\s*)(?P<others>.*)$")
 
+# this removes all whitespace in the given string, except when
+# between " quotes
+_remove_unquoted_whitespace = \
+    lambda text:'"'.join( it if i%2 else ''.join(it.split())
+        for i,it in enumerate(text.split('"'))  )
+
+
+def _remove_list_and_map_whitespace(text):
+    """Returns a string where the whitespace between matching [ and ]
+       is removed, unless quoted"""
+    # regular expression aren't really the right tool, since we may have
+    # nested structures
+    result = []
+    start_pos = 0
+    pos = 0
+    list_count = 0
+    map_count = 0
+    cur_start_list_pos = None
+    cur_start_map_pos = None
+    for i in text:
+        if i == '[' and map_count == 0:
+            if list_count == 0:
+                result.append(text[start_pos:pos + 1])
+                cur_start_list_pos = pos + 1
+            list_count = list_count + 1
+        elif i == ']' and map_count == 0:
+            if list_count > 0:
+                list_count = list_count - 1
+                if list_count == 0:
+                    result.append(_remove_unquoted_whitespace(text[cur_start_list_pos:pos + 1]))
+                    start_pos = pos + 1
+        if i == '{' and list_count == 0:
+            if map_count == 0:
+                result.append(text[start_pos:pos + 1])
+                cur_start_map_pos = pos + 1
+            map_count = map_count + 1
+        elif i == '}' and list_count == 0:
+            if map_count > 0:
+                map_count = map_count - 1
+                if map_count == 0:
+                    result.append(_remove_unquoted_whitespace(text[cur_start_map_pos:pos + 1]))
+                    start_pos = pos + 1
+        
+
+        pos = pos + 1
+    if start_pos <= len(text):
+        result.append(text[start_pos:len(text)])
+    return "".join(result)
+    
+    
 class BindCmdParse:
-    """ This class will parse the command line usr input into three part
+    """ This class will parse the command line user input into three parts:
     module name, command, parameters
     the first two parts are strings and parameter is one hash, 
     parameters part is optional
@@ -86,9 +137,12 @@ class BindCmdParse:
 
             self._parse_params(param_str)
 
+    def _remove_list_whitespace(self, text):
+        return ""
 
     def _parse_params(self, param_text):
         """convert a=b,c=d into one hash """
+        param_text = _remove_list_and_map_whitespace(param_text)
         
         # Check parameter name "help"
         param = NAME_PATTERN.match(param_text)

src/bin/bindctl/moduleinfo.py

@@ -16,6 +16,8 @@
 """This module holds classes representing modules, commands and
    parameters for use in bindctl"""
 
+import textwrap
+
 try:
     from collections import OrderedDict
 except ImportError:
@@ -30,6 +32,9 @@ MODULE_NODE_NAME = 'module'
 COMMAND_NODE_NAME = 'command'
 PARAM_NODE_NAME = 'param'
 
+# this is used to align the descriptions in help output
+CONST_BINDCTL_HELP_INDENT_WIDTH=12
+
 
 class ParamInfo:
     """One parameter of one command.
@@ -52,6 +57,12 @@ class ParamInfo:
     def __str__(self):        
         return str("\t%s <type: %s> \t(%s)" % (self.name, self.type, self.desc))
 
+    def get_name(self):
+        return "%s <type: %s>" % (self.name, self.type)
+
+    def get_desc(self):
+        return self.desc
+
 class CommandInfo:
     """One command which is provided by one bind10 module, it has zero
        or more parameters
@@ -63,13 +74,18 @@ class CommandInfo:
         self.params = OrderedDict()        
         # Set default parameter "help"
         self.add_param(ParamInfo("help", 
-                                  desc = "Get help for command",
+                                  desc = "Get help for command.",
                                   optional = True))
                 
     def __str__(self):
         return str("%s \t(%s)" % (self.name, self.desc))
-        
 
+    def get_name(self):
+        return self.name
+
+    def get_desc(self):
+        return self.desc;
+    
     def add_param(self, paraminfo):
         """Add a ParamInfo object to this CommandInfo"""
         self.params[paraminfo.name] = paraminfo
@@ -144,22 +160,30 @@ class CommandInfo:
         del params["help"]
 
         if len(params) == 0:
-            print("\tNo parameters for the command")
+            print("No parameters for the command")
             return
         
-        print("\n\tMandatory parameters:")
+        print("\nMandatory parameters:")
         mandatory_infos = []
         for info in params.values():            
             if not info.is_optional:
-                print("\t", info)
+                print("    %s" % info.get_name())
+                print(textwrap.fill(info.get_desc(),
+                      initial_indent="        ",
+                      subsequent_indent="        ",
+                      width=70))
                 mandatory_infos.append(info)
 
         optional_infos = [info for info in params.values() 
                           if info not in mandatory_infos]
         if len(optional_infos) > 0:
-            print("\n\tOptional parameters:")      
+            print("\nOptional parameters:")      
             for info in optional_infos:
-                    print("\t", info)
+                print("    %s" % info.get_name())
+                print(textwrap.fill(info.get_desc(),
+                      initial_indent="        ",
+                      subsequent_indent="        ",
+                      width=70))
 
 
 class ModuleInfo:
@@ -172,11 +196,17 @@ class ModuleInfo:
         self.desc = desc
         self.commands = OrderedDict()         
         self.add_command(CommandInfo(name = "help", 
-                                     desc = "Get help for module"))
+                                     desc = "Get help for module."))
         
     def __str__(self):
         return str("%s \t%s" % (self.name, self.desc))
-        
+
+    def get_name(self):
+        return self.name
+
+    def get_desc(self):
+        return self.desc
+
     def add_command(self, command_info):
         """Add a CommandInfo to this ModuleInfo."""
         self.commands[command_info.name] = command_info
@@ -201,8 +231,24 @@ class ModuleInfo:
     def module_help(self):
         """Prints the help info for this module to stdout"""
         print("Module ", self, "\nAvailable commands:")
-        for k in self.commands.keys():
-            print("\t", self.commands[k])
+        for k in self.commands.values():
+            n = k.get_name()
+            if len(n) >= CONST_BINDCTL_HELP_INDENT_WIDTH:
+                print("    %s" % n)
+                print(textwrap.fill(k.get_desc(),
+                      initial_indent="            ",
+                      subsequent_indent="    " +
+                      " " * CONST_BINDCTL_HELP_INDENT_WIDTH,
+                      width=70))
+            else:
+                print(textwrap.fill("%s%s%s" %
+                    (k.get_name(),
+                     " "*(CONST_BINDCTL_HELP_INDENT_WIDTH - len(k.get_name())),
+                     k.get_desc()),
+                    initial_indent="    ",
+                    subsequent_indent="    " +
+                    " " * CONST_BINDCTL_HELP_INDENT_WIDTH,
+                    width=70))
             
     def command_help(self, command):
         """Prints the help info for the command with the given name.

src/bin/bindctl/tests/Makefile.am

@@ -1,5 +1,5 @@
 PYCOVERAGE_RUN = @PYCOVERAGE_RUN@
-PYTESTS = bindctl_test.py
+PYTESTS = bindctl_test.py cmdparse_test.py
 EXTRA_DIST = $(PYTESTS)
 
 # test using command-line arguments, so use check-local target instead of TESTS

src/bin/bindctl/tests/bindctl_test.py

@@ -17,6 +17,8 @@
 import unittest
 import isc.cc.data
 import os
+from isc.config.config_data import ConfigData, MultiConfigData
+from isc.config.module_spec import ModuleSpec
 from bindctl import cmdparse
 from bindctl import bindcmd
 from bindctl.moduleinfo import *
@@ -238,11 +240,101 @@ class TestNameSequence(unittest.TestCase):
             assert self.random_names[i] == module_names[i+1]
             i = i + 1
 
-    def test_apply_cfg_command(self):
+# tine class to fake a UIModuleCCSession, but only the config data
+# parts for the next set of tests
+class FakeCCSession(MultiConfigData):
+    def __init__(self):
+        self._local_changes = {}
+        self._current_config = {}
+        self._specifications = {}
+        self.add_foo_spec()
+
+    def add_foo_spec(self):
+        spec = { "module_name": "foo",
+                 "config_data": [
+                 { "item_name": "an_int",
+                   "item_type": "integer",
+                   "item_optional": False,
+                   "item_default": 1
+                 },
+                 { "item_name": "a_list",
+                   "item_type": "list",
+                   "item_optional": False,
+                   "item_default": [],
+                   "list_item_spec":
+                   { "item_name": "a_string",
+                     "item_type": "string",
+                     "item_optional": False,
+                     "item_default": "bar"
+                   }
+                 }
+                 ]
+               }
+        self.set_specification(ModuleSpec(spec))
+    
+
+class TestConfigCommands(unittest.TestCase):
+    def setUp(self):
+        self.tool = bindcmd.BindCmdInterpreter()
+        mod_info = ModuleInfo(name = "foo")
+        self.tool.add_module_info(mod_info)
+        self.tool.config_data = FakeCCSession()
+        
+    def test_apply_cfg_command_int(self):
         self.tool.location = '/'
-        cmd = cmdparse.BindCmdParse("config set identifier=\"foo/bar\" value=\"5\"")
+
+        self.assertEqual((1, MultiConfigData.DEFAULT),
+                         self.tool.config_data.get_value("/foo/an_int"))
+
+        cmd = cmdparse.BindCmdParse("config set identifier=\"foo/an_int\" value=\"5\"")
         self.tool.apply_config_cmd(cmd)
+        self.assertEqual((5, MultiConfigData.LOCAL),
+                         self.tool.config_data.get_value("/foo/an_int"))
+
+        # this should raise a NotFoundError
+        cmd = cmdparse.BindCmdParse("config set identifier=\"foo/bar\" value=\"[]\"")
+        self.assertRaises(isc.cc.data.DataNotFoundError, self.tool.apply_config_cmd, cmd)
+
+        # this should raise a TypeError
+        cmd = cmdparse.BindCmdParse("config set identifier=\"foo/an_int\" value=\"[]\"")
+        self.assertRaises(isc.cc.data.DataTypeError, self.tool.apply_config_cmd, cmd)
+
+    # this is a very specific one for use with a set of list tests
+    # to try out the flexibility of the parser (only in the next test)
+    def clt(self, full_cmd_string, item_value):
+        cmd = cmdparse.BindCmdParse(full_cmd_string)
+        self.tool.apply_config_cmd(cmd)
+        self.assertEqual(([item_value], MultiConfigData.LOCAL),
+                         self.tool.config_data.get_value("/foo/a_list"))
+
+    def test_apply_cfg_command_list(self):
+        self.tool.location = '/'
+
+        self.assertEqual(([], MultiConfigData.DEFAULT),
+                         self.tool.config_data.get_value("/foo/a_list"))
+
+        self.clt("config set identifier=\"foo/a_list\" value=[\"a\"]", "a")
+        self.clt("config set identifier=\"foo/a_list\" value =[\"b\"]", "b")
+        self.clt("config set identifier=\"foo/a_list\" value= [\"c\"]", "c")
+        self.clt("config set identifier=\"foo/a_list\" value = [\"d\"]", "d")
+        self.clt("config set identifier =\"foo/a_list\" value=[\"e\"]", "e")
+        self.clt("config set identifier= \"foo/a_list\" value=[\"f\"]", "f")
+        self.clt("config set identifier = \"foo/a_list\" value=[\"g\"]", "g")
+        self.clt("config set identifier = \"foo/a_list\" value = [\"h\"]", "h")
+        self.clt("config set identifier = \"foo/a_list\" value=[\"i\" ]", "i")
+        self.clt("config set identifier = \"foo/a_list\" value=[ \"j\"]", "j")
+        self.clt("config set identifier = \"foo/a_list\" value=[ \"k\" ]", "k")
+
+        # this should raise a TypeError
+        cmd = cmdparse.BindCmdParse("config set identifier=\"foo/a_list\" value=\"a\"")
+        self.assertRaises(isc.cc.data.DataTypeError, self.tool.apply_config_cmd, cmd)
+        
+        cmd = cmdparse.BindCmdParse("config set identifier=\"foo/a_list\" value=[1]")
+        self.assertRaises(isc.cc.data.DataTypeError, self.tool.apply_config_cmd, cmd)
+
+
     
+
 class FakeBindCmdInterpreter(bindcmd.BindCmdInterpreter):
     def __init__(self):
         pass

src/bin/bindctl/tests/cmdparse_test.py

@@ -0,0 +1,88 @@
+# Copyright (C) 2009  Internet Systems Consortium.
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
+# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
+# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+
+import unittest
+from bindctl import cmdparse
+
+class TestCmdParse(unittest.TestCase):
+
+    def test_remove_unquoted_whitespace(self):
+        self.assertEqual(cmdparse._remove_unquoted_whitespace("a"), "a")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace(" a"), "a")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace("a "), "a")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace(" a "), "a")
+        self.assertNotEqual(cmdparse._remove_unquoted_whitespace("a"), "a ")
+        self.assertNotEqual(cmdparse._remove_unquoted_whitespace(" a"), " a")
+        self.assertNotEqual(cmdparse._remove_unquoted_whitespace("a "), "a ")
+        self.assertNotEqual(cmdparse._remove_unquoted_whitespace(" a "), " a ")
+        self.assertNotEqual(cmdparse._remove_unquoted_whitespace(" a "), "b")
+
+        self.assertEqual(cmdparse._remove_unquoted_whitespace("\"abc\""), "\"abc\"")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace(" \"abc\""), "\"abc\"")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace("\"abc\" "), "\"abc\"")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace(" \"abc\" "), "\"abc\"")
+        
+        self.assertEqual(cmdparse._remove_unquoted_whitespace("\" abc\""), "\" abc\"")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace(" \"a bc\""), "\"a bc\"")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace("\"ab c\" "), "\"ab c\"")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace(" \"abc \" "), "\"abc \"")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace(" \" a b c \" "), "\" a b c \"")
+        
+        self.assertEqual(cmdparse._remove_unquoted_whitespace("a\" abc\"a"), "a\" abc\"a")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace("a \"a bc\"a"), "a\"a bc\"a")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace("a\"ab c\" a"), "a\"ab c\"a")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace("a \"abc \" a"), "a\"abc \"a")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace("a \" a b c \" a"), "a\" a b c \"a")
+
+    # short-hand function to make the set of tests more readable
+    def rws(self, a, b):
+        self.assertEqual(cmdparse._remove_list_and_map_whitespace(a), b)
+
+    def test_remove_list_whitespace(self):
+        self.rws("a", "a")
+        self.rws(" a ", " a ")
+        self.rws(" [a] ", " [a] ")
+        self.rws(" [ a] ", " [a] ")
+        self.rws(" [ a ] ", " [a] ")
+        self.rws(" [ a b c ] ", " [abc] ")
+        self.rws(" [ a \"b c\" ] ", " [a\"b c\"] ")
+        self.rws("a [ a \"b c\" ] a", "a [a\"b c\"] a")
+        self.rws("a] [ a \"b c\" ] a", "a] [a\"b c\"] a")
+        self.rws(" [ a [b c] ] ", " [a[bc]] ")
+        self.rws(" [ a b][ c d ] ", " [ab][cd] ")
+        self.rws(" [ a b] [ c d ] ", " [ab] [cd] ")
+        
+        self.rws("a", "a")
+        self.rws(" a ", " a ")
+        self.rws(" {a} ", " {a} ")
+        self.rws(" { a} ", " {a} ")
+        self.rws(" { a } ", " {a} ")
+        self.rws(" { a b c } ", " {abc} ")
+        self.rws(" { a \"b c\" } ", " {a\"b c\"} ")
+        self.rws("a { a \"b c\" } a", "a {a\"b c\"} a")
+        self.rws("a} { a \"b c\" } a", "a} {a\"b c\"} a")
+        self.rws(" { a {b c} } ", " {a{bc}} ")
+        self.rws(" { a b}{ c d } ", " {ab}{cd} ")
+        self.rws(" { a b} { c d } ", " {ab} {cd} ")
+
+        self.rws(" [ a b]{ c d } ", " [ab]{cd} ")
+        self.rws(" [ a b{ c d }] ", " [ab{cd}] ")
+        self.rws(" [ a b{ \"c d\" }] ", " [ab{\"c d\"}] ")
+        
+
+if __name__== "__main__":
+    unittest.main()
+    

src/lib/asiolink/tests/recursive_query_unittest.cc

@@ -674,7 +674,7 @@ TEST_F(RecursiveQueryTest, forwardClientTimeout) {
     RecursiveQuery query(*dns_service_,
                          singleAddress(TEST_IPV4_ADDR, port),
                          singleAddress(TEST_IPV4_ADDR, port),
-                         50, 120, 1000, 4);
+                         200, 480, 4000, 4);
     Question question(Name("example.net"), RRClass::IN(), RRType::A());
     OutputBufferPtr buffer(new OutputBuffer(0));
     query.resolve(question, answer, buffer, &server);
@@ -718,7 +718,7 @@ TEST_F(RecursiveQueryTest, forwardLookupTimeout) {
     RecursiveQuery query(*dns_service_,
                          singleAddress(TEST_IPV4_ADDR, port),
                          singleAddress(TEST_IPV4_ADDR, port),
-                         50, 4000, 120, 5);
+                         200, 4000, 480, 5);
     Question question(Name("example.net"), RRClass::IN(), RRType::A());
     OutputBufferPtr buffer(new OutputBuffer(0));
     query.resolve(question, answer, buffer, &server);

src/lib/cache/cache_entry_key.cc

@@ -12,8 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
-
 #include <sstream>
 #include "cache_entry_key.h"
 

src/lib/cache/cache_entry_key.h

@@ -12,8 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
-
 #ifndef __CACHE_ENTRY_KEY_H
 #define __CACHE_ENTRY_KEY_H
 

src/lib/cache/local_zone_data.cc

@@ -12,8 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
-
 #include <dns/rrset.h>
 #include "local_zone_data.h"
 #include "cache_entry_key.h"

src/lib/cache/local_zone_data.h

@@ -12,8 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
-
 #ifndef _LOCAL_ZONE_DATA
 #define _LOCAL_ZONE_DATA
 

src/lib/cache/message_cache.cc

@@ -12,8 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
-
 #include <config.h>
 
 #include <nsas/nsas_entry_compare.h>

src/lib/cache/message_cache.h

@@ -12,8 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
-
 #ifndef __MESSAGE_CACHE_H
 #define __MESSAGE_CACHE_H
 

src/lib/cache/message_entry.cc

@@ -12,8 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
-
 #include <config.h>
 
 #include <limits>

src/lib/cache/message_entry.h

@@ -12,8 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
-
 #ifndef __MESSAGE_ENTRY_H
 #define __MESSAGE_ENTRY_H
 

src/lib/cache/resolver_cache.cc

@@ -12,8 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
-
 #include <config.h>
 
 #include "resolver_cache.h"

src/lib/cache/resolver_cache.h

@@ -12,8 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
-
 #ifndef __RESOLVER_CACHE_H
 #define __RESOLVER_CACHE_H
 

src/lib/cache/rrset_cache.cc

@@ -12,8 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
-
 #include <config.h>
 
 #include <string>

src/lib/cache/rrset_cache.h

@@ -12,8 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
-
 #ifndef __RRSET_CACHE_H
 #define __RRSET_CACHE_H
 

src/lib/cache/rrset_copy.cc

@@ -12,8 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
-
 #include "rrset_copy.h"
 
 using namespace isc::dns;

src/lib/cache/rrset_copy.h

@@ -12,8 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
-
 #ifndef __RRSET_COPY_
 #define __RRSET_COPY_
 

src/lib/cache/rrset_entry.cc

@@ -12,8 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
-
 #include <config.h>
 
 #include <dns/message.h>

src/lib/cache/rrset_entry.h

@@ -12,8 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
-
 #ifndef __RRSET_ENTRY_H
 #define __RRSET_ENTRY_H
 

src/lib/cache/tests/cache_test_messagefromfile.h

@@ -12,7 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
 #include <vector>
 #include <dns/tests/unittest_util.h>
 #include <dns/buffer.h>

src/lib/cache/tests/cache_test_sectioncount.h

@@ -12,7 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
 #include <vector>
 #include <dns/tests/unittest_util.h>
 #include <dns/buffer.h>

src/lib/cache/tests/local_zone_data_unittest.cc

@@ -12,7 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
 #include <config.h>
 #include <string>
 #include <gtest/gtest.h>

src/lib/cache/tests/message_cache_unittest.cc

@@ -12,7 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
 #include <config.h>
 #include <string>
 #include <gtest/gtest.h>

src/lib/cache/tests/message_entry_unittest.cc

@@ -12,7 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
 #include <config.h>
 #include <string>
 #include <gtest/gtest.h>

src/lib/cache/tests/resolver_cache_unittest.cc

@@ -12,7 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
 #include <config.h>
 #include <string>
 #include <gtest/gtest.h>

src/lib/cache/tests/rrset_cache_unittest.cc

@@ -12,7 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
 #include <config.h>
 #include <string>
 #include <gtest/gtest.h>

src/lib/cache/tests/rrset_entry_unittest.cc

@@ -12,7 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id$
 #include <config.h>
 #include <string>
 #include <gtest/gtest.h>

src/lib/cache/tests/run_unittests.cc

@@ -12,7 +12,6 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-// $Id: run_unittests.cc 3020 2010-09-26 03:47:26Z jinmei $
 #include <config.h>
 
 #include <gtest/gtest.h>

src/lib/config/tests/testdata/spec22.spec

@@ -1,6 +1,6 @@
 {
   "module_spec": {
-    "module_name": "Spec2",
+    "module_name": "Spec22",
     "config_data": [
       { "item_name": "value1",
         "item_type": "integer",
@@ -81,7 +81,7 @@
       { "item_name": "value9",
         "item_type": "map",
         "item_optional": false,
-        "item_default": {},
+        "item_default": { "v91": "def", "v92": {} },
         "map_item_spec": [
           { "item_name": "v91",
             "item_type": "string",

src/lib/dns/dnssectime.cc

@@ -12,6 +12,10 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
+#include <stdint.h>
+
+#include <sys/time.h>
+
 #include <string>
 #include <iomanip>
 #include <iostream>
@@ -26,30 +30,121 @@
 
 using namespace std;
 
+namespace {
+int days[12] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 };
+
+inline bool
+isLeap(const int y) {
+    return ((((y) % 4) == 0 && ((y) % 100) != 0) || ((y) % 400) == 0);
+}
+
+unsigned int
+yearSecs(const int year) {
+    return ((isLeap(year) ? 366 : 365 ) * 86400);
+}
+
+unsigned int
+monthSecs(const int month, const int year) {
+    return ((days[month] + ((month == 1 && isLeap(year)) ? 1 : 0 )) * 86400);
+}
+}
+
 namespace isc {
 namespace dns {
 
 string
-timeToText(const time_t timeval) {
-    struct tm* const t = gmtime(&timeval);
-
-    // gmtime() will keep most values within range, but it can
-    // produce a five-digit year; check for this.
-    if ((t->tm_year + 1900) > 9999) {
-        isc_throw(InvalidTime, "Time value out of range: year > 9999");
+timeToText64(uint64_t value) {
+    struct tm tm;
+    unsigned int secs;
+
+    // We cannot rely on gmtime() because time_t may not be of 64 bit
+    // integer.  The following conversion logic is borrowed from BIND 9.
+    tm.tm_year = 70;
+    while ((secs = yearSecs(tm.tm_year + 1900)) <= value) {
+        value -= secs;
+        ++tm.tm_year;
+        if (tm.tm_year + 1900 > 9999) {
+            isc_throw(InvalidTime,
+                      "Time value out of range (year > 9999): " <<
+                      tm.tm_year + 1900);
+        }
+    }
+    tm.tm_mon = 0;
+    while ((secs = monthSecs(tm.tm_mon, tm.tm_year + 1900)) <= value) {
+        value -= secs;
+        tm.tm_mon++;
     }
+    tm.tm_mday = 1;
+    while (86400 <= value) {
+        value -= 86400;
+        ++tm.tm_mday;
+    }
+    tm.tm_hour = 0;
+    while (3600 <= value) {
+        value -= 3600;
+        ++tm.tm_hour;
+    }
+    tm.tm_min = 0;
+    while (60 <= value) {
+        value -= 60;
+        ++tm.tm_min;
+    }
+    tm.tm_sec = value;    // now t < 60, so this substitution is safe.
 
     ostringstream oss;
     oss << setfill('0')
-        << setw(4) << t->tm_year + 1900
-        << setw(2) << t->tm_mon + 1
-        << setw(2) << t->tm_mday 
-        << setw(2) << t->tm_hour
-        << setw(2) << t->tm_min
-        << setw(2) << t->tm_sec;
+        << setw(4) << tm.tm_year + 1900
+        << setw(2) << tm.tm_mon + 1
+        << setw(2) << tm.tm_mday
+        << setw(2) << tm.tm_hour
+        << setw(2) << tm.tm_min
+        << setw(2) << tm.tm_sec;
     return (oss.str());
 }
 
+// timeToText32() below uses the current system time.  To test it with
+// unusual current time values we introduce the following function pointer;
+// when it's non NULL, we call it to get the (normally faked) current time.
+// Otherwise we use the standard gettimeofday(2).  This hook is specifically
+// intended for testing purposes, so, even if it's visible outside of this
+// library, it's not even declared in a header file.
+namespace dnssectime {
+namespace detail {
+int64_t (*gettimeFunction)() = NULL;
+}
+}
+
+namespace {
+int64_t
+gettimeofdayWrapper() {
+    using namespace dnssectime::detail;
+    if (gettimeFunction != NULL) {
+        return (gettimeFunction());
+    }
+
+    struct timeval now;
+    gettimeofday(&now, NULL);
+
+    return (static_cast<int64_t>(now.tv_sec));
+}
+}
+
+string
+timeToText32(const uint32_t value) {
+    // We first adjust the time to the closest epoch based on the current time.
+    // Note that the following variables must be signed in order to handle
+    // time until year 2038 correctly.
+    const int64_t start = gettimeofdayWrapper() - 0x7fffffff;
+    int64_t base = 0;
+    int64_t t;
+    while ((t = (base + value)) < start) {
+        base += 0x100000000LL;
+    }
+
+    // Then convert it to text.
+    return (timeToText64(t));
+}
+
 namespace {
 const size_t DATE_LEN = 14;      // YYYYMMDDHHmmSS
 
@@ -62,27 +157,20 @@ checkRange(const int min, const int max, const int value,
     }
     isc_throw(InvalidTime, "Invalid " << valname << "value: " << value);
 }
-
-int days[12] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 };
-
-inline bool
-isLeap(const int y) {
-    return ((((y) % 4) == 0 && ((y) % 100) != 0) || ((y) % 400) == 0);
-}
 }
 
-time_t
-timeFromText(const string& time_txt) {
-    // first try reading YYYYMMDDHHmmSS format
-    int year, month, day, hour, minute, second;
-
+uint64_t
+timeFromText64(const string& time_txt) {
+    // Confirm the source only consists digits.  sscanf() allows some
+    // minor exceptions.
     for (int i = 0; i < time_txt.length(); ++i) {
         if (!isdigit(time_txt.at(i))) {
-            isc_throw(InvalidTime,
-                      "Couldn't convert non-numeric time value: " << time_txt); 
+            isc_throw(InvalidTime, "Couldn't convert non-numeric time value: "
+                      << time_txt);
         }
     }
 
+    int year, month, day, hour, minute, second;
     if (time_txt.length() != DATE_LEN ||
         sscanf(time_txt.c_str(), "%4d%2d%2d%2d%2d%2d",
                &year, &month, &day, &hour, &minute, &second) != 6)
@@ -98,9 +186,9 @@ timeFromText(const string& time_txt) {
     checkRange(0, 59, minute, "minute");
     checkRange(0, 60, second, "second"); // 60 == leap second.
 
-    time_t timeval = second + (60 * minute) + (3600 * hour) +
+    uint64_t timeval = second + (60 * minute) + (3600 * hour) +
         ((day - 1) * 86400);
-    for (int m = 0; m < (month - 1); m++) {
+    for (int m = 0; m < (month - 1); ++m) {
             timeval += days[m] * 86400;
     }
     if (isLeap(year) && month > 2) {
@@ -112,5 +200,12 @@ timeFromText(const string& time_txt) {
 
     return (timeval);
 }
+
+uint32_t
+timeFromText32(const string& time_txt) {
+    // The implicit conversion from uint64_t to uint32_t should just work here,
+    // because we only need to drop higher 32 bits.
+    return (timeFromText64(time_txt));
+}
 }
 }

src/lib/dns/dnssectime.h

@@ -17,7 +17,6 @@
 
 #include <sys/types.h>
 #include <stdint.h>
-#include <time.h>
 
 #include <exceptions/exceptions.h>
 
@@ -40,11 +39,102 @@ public:
         isc::Exception(file, line, what) {}
 };
 
-time_t
-timeFromText(const std::string& time_txt);
+///
+/// \name DNSSEC time conversion functions.
+///
+/// These functions convert between times represented in seconds (in integer)
+/// since epoch and those in the textual form used in the RRSIG records.
+/// For integers we provide both 32-bit and 64-bit versions.
+/// The RRSIG expiration and inception fields are both 32-bit unsigned
+/// integers, so 32-bit versions would be more useful for protocol operations.
+/// However, with 32-bit integers we need to take into account wrap-around
+/// points and compare values using the serial number arithmetic as specified
+/// in RFC4034, which would be more error prone.  We therefore provide 64-bit
+/// versions, too.
+///
+/// The timezone is always UTC for these functions.
+//@{
+/// Convert textual DNSSEC time to integer, 64-bit version.
+///
+/// The textual form must only consist of digits and be in the form of
+/// YYYYMMDDHHmmSS, where:
+/// - YYYY must be between 1970 and 9999
+/// - MM must be between 01 and 12
+/// - DD must be between 01 and 31 and must be a valid day for the month
+///   represented in 'MM'.  For example, if MM is 04, DD cannot be 31.
+///   DD can be 29 when MM is 02 only when YYYY is a leap year.
+/// - HH must be between 00 and 23
+/// - mm must be between 00 and 59
+/// - SS must be between 00 and 60
+///
+/// For all fields the range includes the begin and end values.  Note that
+/// 60 is allowed for 'SS', intending a leap second, although in real operation
+/// it's unlikely to be specified.
+///
+/// If the given text is valid, this function converts it to an unsigned
+/// 64-bit number of seconds since epoch (1 January 1970 00:00:00) and returns
+/// the converted value.  64 bits are sufficient to represent all possible
+/// values for the valid format uniquely, so there is no overflow.
+///
+/// \note RFC4034 also defines the textual form of an unsigned decimal integer
+/// for the corresponding time in seconds.  This function doesn't support
+/// this form, and if given it throws an exception of class \c InvalidTime.
+///
+/// \exception InvalidTime The given textual representation is invalid.
+///
+/// \param time_txt Textual time in the form of YYYYMMDDHHmmSS
+/// \return Seconds since epoch corresponding to \c time_txt
+uint64_t
+timeFromText64(const std::string& time_txt);
 
+/// Convert textual DNSSEC time to integer, 32-bit version.
+///
+/// This version is the same as \c timeFromText64() except that the return
+/// value is wrapped around to an unsigned 32-bit integer, simply dropping
+/// the upper 32 bits.
+uint32_t
+timeFromText32(const std::string& time_txt);
+
+/// Convert integral DNSSEC time to textual form, 64-bit version.
+///
+/// This function takes an integer that would be seconds since epoch and
+/// converts it in the form of YYYYMMDDHHmmSS.  For example, if \c value is
+/// 0, it returns "19700101000000".  If the value corresponds to a point
+/// of time on and after year 10,000, which cannot be represented in the
+/// YYYY... form, an exception of class \c InvalidTime will be thrown.
+///
+/// \exception InvalidTime The given time specifies on or after year 10,000.
+/// \exception Other A standard exception, if resource allocation for the
+/// returned text fails.
+///
+/// \param value Seconds since epoch to be converted.
+/// \return Textual representation of \c value in the form of YYYYMMDDHHmmSS.
 std::string
-timeToText(const time_t timeval);
+timeToText64(uint64_t value);
+
+/// Convert integral DNSSEC time to textual form, 32-bit version.
+///
+/// This version is the same as \c timeToText64(), but the time value
+/// is expected to be the lower 32 bits of the full 64-bit value.
+/// These two will be different on and after a certain point of time
+/// in year 2106, so this function internally resolves the ambiguity
+/// using the current system time at the time of function call;
+/// it first identifies the range of [N*2^32 - 2^31, N*2^32 + 2^31)
+/// that contains the current time, and interprets \c value in the context
+/// of that range.  It then applies the same process as \c timeToText64().
+///
+/// There is one important exception in this processing, however.
+/// Until 19 Jan 2038 03:14:08 (2^31 seconds since epoch), this range
+/// would contain time before epoch.  In order to ensure the returned
+/// value is also a valid input to \c timeFromText, this function uses
+/// a special range [0, 2^32) until that time.  As a result, all upper
+/// half of the 32-bit values are treated as a future time.  For example,
+/// 2^32-1 (the highest value in 32-bit unsigned integers) will be converted
+/// to "21060207062815", instead of "19691231235959".
+std::string
+timeToText32(const uint32_t value);
+
+//@}
 }
 }
 

src/lib/dns/rdata/generic/rrsig_46.cc

@@ -93,8 +93,8 @@ RRSIG::RRSIG(const string& rrsig_str) :
         isc_throw(InvalidRdataText, "RRSIG labels out of range");
     }
 
-    uint32_t timeexpire = timeFromText(expire_txt);
-    uint32_t timeinception = timeFromText(inception_txt);
+    const uint32_t timeexpire = timeFromText32(expire_txt);
+    const uint32_t timeinception = timeFromText32(inception_txt);
 
     vector<uint8_t> signature;
     decodeBase64(signaturebuf.str(), signature);
@@ -157,15 +157,12 @@ RRSIG::~RRSIG() {
 
 string
 RRSIG::toText() const {
-    string expire = timeToText(impl_->timeexpire_);
-    string inception = timeToText(impl_->timeinception_);
-
     return (impl_->covered_.toText() +
             " " + boost::lexical_cast<string>(static_cast<int>(impl_->algorithm_))
             + " " + boost::lexical_cast<string>(static_cast<int>(impl_->labels_))
             + " " + boost::lexical_cast<string>(impl_->originalttl_)
-            + " " + expire
-            + " " + inception
+            + " " + timeToText32(impl_->timeexpire_)
+            + " " + timeToText32(impl_->timeinception_)
             + " " + boost::lexical_cast<string>(impl_->tag_)
             + " " + impl_->signer_.toText()
             + " " + encodeBase64(impl_->signature_));

src/lib/dns/tests/dnssectime_unittest.cc

@@ -23,48 +23,141 @@
 using namespace std;
 using namespace isc::dns;
 
+// See dnssectime.cc
+namespace isc {
+namespace dns {
+namespace dnssectime {
+namespace detail {
+extern int64_t (*gettimeFunction)();
+}
+}
+}
+}
+
 namespace {
 
-TEST(DNSSECTimeTest, fromText) {
+class DNSSECTimeTest : public ::testing::Test {
+protected:
+    ~DNSSECTimeTest() {
+        dnssectime::detail::gettimeFunction = NULL;
+    }
+};
+
+TEST_F(DNSSECTimeTest, fromText) {
+    // In most cases (in practice) the 32-bit and 64-bit versions should
+    // behave identically, so we'll mainly test the 32-bit version, which
+    // will be more commonly used in actual code (because many of the wire
+    // format time field are 32-bit).  The subtle cases where these two
+    // return different values will be tested at the end of this test case.
+
     // These are bogus and should be rejected
-    EXPECT_THROW(timeFromText("2011 101120000"), InvalidTime);
-    EXPECT_THROW(timeFromText("201101011200-0"), InvalidTime);
+    EXPECT_THROW(timeFromText32("2011 101120000"), InvalidTime);
+    EXPECT_THROW(timeFromText32("201101011200-0"), InvalidTime);
 
-    // Short length
-    EXPECT_THROW(timeFromText("20100223"), InvalidTime);
+    // Short length (or "decimal integer" version of representation;
+    // it's valid per RFC4034, but is not supported in this implementation)
+    EXPECT_THROW(timeFromText32("20100223"), InvalidTime);
 
     // Leap year checks
-    EXPECT_THROW(timeFromText("20110229120000"), InvalidTime);
-    EXPECT_THROW(timeFromText("21000229120000"), InvalidTime);
-    EXPECT_NO_THROW(timeFromText("20000229120000"));
-    EXPECT_NO_THROW(timeFromText("20120229120000"));
+    EXPECT_THROW(timeFromText32("20110229120000"), InvalidTime);
+    EXPECT_THROW(timeFromText32("21000229120000"), InvalidTime);
+    EXPECT_NO_THROW(timeFromText32("20000229120000"));
+    EXPECT_NO_THROW(timeFromText32("20120229120000"));
 
     // unusual case: this implementation allows SS=60 for "leap seconds"
-    EXPECT_NO_THROW(timeFromText("20110101120060"));
+    EXPECT_NO_THROW(timeFromText32("20110101120060"));
 
     // Out of range parameters
-    EXPECT_THROW(timeFromText("19100223214617"), InvalidTime); // YY<1970
-    EXPECT_THROW(timeFromText("20110001120000"), InvalidTime); // MM=00
-    EXPECT_THROW(timeFromText("20111301120000"), InvalidTime); // MM=13
-    EXPECT_THROW(timeFromText("20110100120000"), InvalidTime); // DD=00
-    EXPECT_THROW(timeFromText("20110132120000"), InvalidTime); // DD=32
-    EXPECT_THROW(timeFromText("20110431120000"), InvalidTime); // 'Apr31'
-    EXPECT_THROW(timeFromText("20110101250000"), InvalidTime); // HH=25
-    EXPECT_THROW(timeFromText("20110101126000"), InvalidTime); // mm=60
-    EXPECT_THROW(timeFromText("20110101120061"), InvalidTime); // SS=61
+    EXPECT_THROW(timeFromText32("19100223214617"), InvalidTime); // YY<1970
+    EXPECT_THROW(timeFromText32("20110001120000"), InvalidTime); // MM=00
+    EXPECT_THROW(timeFromText32("20111301120000"), InvalidTime); // MM=13
+    EXPECT_THROW(timeFromText32("20110100120000"), InvalidTime); // DD=00
+    EXPECT_THROW(timeFromText32("20110132120000"), InvalidTime); // DD=32
+    EXPECT_THROW(timeFromText32("20110431120000"), InvalidTime); // 'Apr31'
+    EXPECT_THROW(timeFromText32("20110101250000"), InvalidTime); // HH=25
+    EXPECT_THROW(timeFromText32("20110101126000"), InvalidTime); // mm=60
+    EXPECT_THROW(timeFromText32("20110101120061"), InvalidTime); // SS=61
+
+    // Feb 7, 06:28:15 UTC 2106 is the possible maximum time that can be
+    // represented as an unsigned 32bit integer without overflow.
+    EXPECT_EQ(4294967295LU, timeFromText32("21060207062815"));
+
+    // After that, timeFromText32() should start returning the second count
+    // modulo 2^32.
+    EXPECT_EQ(0, timeFromText32("21060207062816"));
+    EXPECT_EQ(10, timeFromText32("21060207062826"));
+
+    // On the other hand, the 64-bit version should return monotonically
+    // increasing counters.
+    EXPECT_EQ(4294967296LL, timeFromText64("21060207062816"));
+    EXPECT_EQ(4294967306LL, timeFromText64("21060207062826"));
 }
 
-TEST(DNSSECTimeTest, toText) {
-    EXPECT_EQ("19700101000000", timeToText(0));
-    EXPECT_EQ("20100311233000", timeToText(1268350200));
+// This helper templated function tells timeToText32 a faked current time.
+// The template parameter is that faked time in the form of int64_t seconds
+// since epoch.
+template <int64_t NOW>
+int64_t
+testGetTime() {
+    return (NOW);
 }
 
-TEST(DNSSECTimeTest, overflow) {
+// Seconds since epoch for the year 10K eve.  Commonly used in some tests
+// below.
+const uint64_t YEAR10K_EVE = 253402300799LL;
+
+TEST_F(DNSSECTimeTest, toText) {
+    // Check a basic case with the default (normal) gettimeFunction
+    // based on the "real current time".
+    // Note: this will fail after year 2078, but at that point we won't use
+    // this program anyway:-)
+    EXPECT_EQ("20100311233000", timeToText32(1268350200));
+
+    // Set the current time to: Feb 18 09:04:14 UTC 2012 (an arbitrary choice
+    // in the range of the first half of uint32 since epoch).
+    dnssectime::detail::gettimeFunction = testGetTime<1329555854LL>;
+
+    // Test the "year 2038" problem.
+    // Check the result of toText() for "INT_MIN" in int32_t.  It's in the
+    // 68-year range from the faked current time, so the result should be
+    // in year 2038, instead of 1901.
+    EXPECT_EQ("20380119031408", timeToText64(0x80000000L));
+    EXPECT_EQ("20380119031408", timeToText32(0x80000000L));
+
+    // A controversial case: what should we do with "-1"?  It's out of range
+    // in future, but according to RFC time before epoch doesn't seem to be
+    // considered "in-range" either.  Our toText() implementation handles
+    // this range as a special case and always treats them as future time
+    // until year 2038.  This won't be a real issue in practice, though,
+    // since such too large values won't be used in actual deployment by then.
+    EXPECT_EQ("21060207062815", timeToText32(0xffffffffL));
+
+    // After the singular point of year 2038, the first half of uint32 can
+    // point to a future time.
+    // Set the current time to: Apr 1 00:00:00 UTC 2038:
+    dnssectime::detail::gettimeFunction = testGetTime<2153692800LL>;
+    // then time "10" is Feb 7 06:28:26 UTC 2106
+    EXPECT_EQ("21060207062826", timeToText32(10));
+    // in 64-bit, it's 2^32 + 10
+    EXPECT_EQ("21060207062826", timeToText64(0x10000000aLL));
+
+    // After year 2106, the upper half of uint32 can point to past time
+    // (as it should).
+    dnssectime::detail::gettimeFunction = testGetTime<0x10000000aLL>;
+    EXPECT_EQ("21060207062815", timeToText32(0xffffffffL));
+
+    // Try very large time value.  Actually it's the possible farthest time
+    // that can be represented in the form of YYYYMMDDHHmmSS.
+    EXPECT_EQ("99991231235959", timeToText64(YEAR10K_EVE));
+    dnssectime::detail::gettimeFunction = testGetTime<YEAR10K_EVE - 10>;
+    EXPECT_EQ("99991231235959", timeToText32(4294197631LU));
+}
+
+TEST_F(DNSSECTimeTest, overflow) {
     // Jan 1, Year 10,000.
-    if (sizeof(time_t) > 4) {
-        EXPECT_THROW(timeToText(static_cast<time_t>(253402300800LL)),
-                     InvalidTime);
-    }
+    EXPECT_THROW(timeToText64(253402300800LL), InvalidTime);
+    dnssectime::detail::gettimeFunction = testGetTime<YEAR10K_EVE - 10>;
+    EXPECT_THROW(timeToText32(4294197632LU), InvalidTime);
 }
 
 }

src/lib/dns/tests/rdata_mx_unittest.cc

@@ -74,12 +74,9 @@ TEST_F(Rdata_MX_Test, toWireRenderer) {
 TEST_F(Rdata_MX_Test, toWireBuffer) {
     renderer.writeName(Name("example.com"));
     rdata_mx.toWire(obuffer);
-}
 
-TEST_F(Rdata_MX_Test, DISABLED_toWireBuffer) {
-// XXX: does not pass
     vector<unsigned char> data;
-    UnitTestUtil::readWireData("rdata_mx_toWire1", data);
+    UnitTestUtil::readWireData("rdata_mx_toWire2", data);
     EXPECT_PRED_FORMAT4(UnitTestUtil::matchWireData, obuffer.getData(),
                         obuffer.getLength(), &data[0], data.size());
 }

src/lib/dns/tests/testdata/Makefile.am

@@ -50,7 +50,8 @@ EXTRA_DIST += name_toWire5.spec name_toWire6.spec
 EXTRA_DIST += question_fromWire question_toWire1 question_toWire2
 EXTRA_DIST += rdata_cname_fromWire rdata_dname_fromWire rdata_dnskey_fromWire
 EXTRA_DIST += rdata_ds_fromWire rdata_in_a_fromWire rdata_in_aaaa_fromWire
-EXTRA_DIST += rdata_mx_fromWire rdata_mx_toWire1 rdata_ns_fromWire
+EXTRA_DIST += rdata_mx_fromWire rdata_mx_toWire1 rdata_mx_toWire2
+EXTRA_DIST += rdata_ns_fromWire
 EXTRA_DIST += rdata_nsec_fromWire1 rdata_nsec_fromWire2 rdata_nsec_fromWire3
 EXTRA_DIST += rdata_nsec_fromWire4.spec rdata_nsec_fromWire5.spec
 EXTRA_DIST += rdata_nsec_fromWire6.spec rdata_nsec_fromWire7.spec

src/lib/dns/tests/testdata/rdata_mx_toWire2

@@ -0,0 +1,12 @@
+#
+# compressed MX RDATA stored in an output buffer
+#
+# sentinel name: example.com.
+# 0  1  2  3  4  5  6  7  8  9 10  1  2 (bytes)
+#(7) e  x  a  m  p  l  e (3) c  o  m  .
+ 07 65 78 61 6d 70 6c 65 03 63 6f 6d 00
+# PREFERENCE: 10
+  00 0a
+# EXCHANGE: not compressed
+#(4) m  x ptr=0
+ 02 6d 78 07 65 78 61 6d 70 6c 65 03 63 6f 6d 00

src/lib/python/isc/cc/data.py

@@ -100,8 +100,8 @@ def split_identifier_list_indices(identifier):
 
     i = id_str.find('[')
     if i < 0:
-        if identifier.find(']') >= 0:
-            raise DataTypeError("Bad format in identifier: " + str(identifier))
+        if id_str.find(']') >= 0:
+            raise DataTypeError("Bad format in identifier (] but no [): " + str(identifier))
         return identifier, None
 
     # keep the non-index part of that to replace later
@@ -110,7 +110,7 @@ def split_identifier_list_indices(identifier):
     while i >= 0:
         e = id_str.find(']')
         if e < i + 1:
-            raise DataTypeError("Bad format in identifier: " + str(identifier))
+            raise DataTypeError("Bad format in identifier (] before [): " + str(identifier))
         try:
             indices.append(int(id_str[i+1:e]))
         except ValueError:
@@ -118,9 +118,9 @@ def split_identifier_list_indices(identifier):
         id_str = id_str[e + 1:]
         i = id_str.find('[')
         if i > 0:
-            raise DataTypeError("Bad format in identifier: " + str(identifier))
+            raise DataTypeError("Bad format in identifier ([ within []): " + str(identifier))
     if id.find(']') >= 0 or len(id_str) > 0:
-        raise DataTypeError("Bad format in identifier: " + str(identifier))
+        raise DataTypeError("Bad format in identifier (extra ]): " + str(identifier))
 
     # we replace the final part of the original identifier with
     # the stripped string

src/lib/python/isc/config/ccsession.py

@@ -374,20 +374,34 @@ class UIModuleCCSession(MultiConfigData):
         self._set_current_config(config)
 
 
-    def add_value(self, identifier, value_str):
+    def add_value(self, identifier, value_str = None):
         """Add a value to a configuration list. Raises a DataTypeError
            if the value does not conform to the list_item_spec field
-           of the module config data specification"""
+           of the module config data specification. If value_str is
+           not given, we add the default as specified by the .spec
+           file."""
         module_spec = self.find_spec_part(identifier)
         if (type(module_spec) != dict or "list_item_spec" not in module_spec):
             raise isc.cc.data.DataNotFoundError(str(identifier) + " is not a list")
-        value = isc.cc.data.parse_value_str(value_str)
+
         cur_list, status = self.get_value(identifier)
         if not cur_list:
             cur_list = []
+
+        # Hmm. Do we need to check for duplicates?
+        value = None
+        if value_str is not None:
+            value = isc.cc.data.parse_value_str(value_str)
+        else:
+            if "item_default" in module_spec["list_item_spec"]:
+                value = module_spec["list_item_spec"]["item_default"]
+
+        if value is None:
+            raise isc.cc.data.DataNotFoundError("No value given and no default for " + str(identifier))
+            
         if value not in cur_list:
             cur_list.append(value)
-        self.set_value(identifier, cur_list)
+            self.set_value(identifier, cur_list)
 
     def remove_value(self, identifier, value_str):
         """Remove a value from a configuration list. The value string

src/lib/python/isc/config/config_data.py

@@ -121,6 +121,7 @@ def find_spec_part(element, identifier):
         # strip list selector part
         # don't need it for the spec part, so just drop it
         id, list_indices = isc.cc.data.split_identifier_list_indices(id_part)
+        # is this part still needed? (see below)
         if type(cur_el) == dict and 'map_item_spec' in cur_el.keys():
             found = False
             for cur_el_item in cur_el['map_item_spec']:
@@ -128,15 +129,24 @@ def find_spec_part(element, identifier):
                     cur_el = cur_el_item
                     found = True
             if not found:
-                raise isc.cc.data.DataNotFoundError(id + " in " + str(cur_el))
+                raise isc.cc.data.DataNotFoundError(id + " not found")
+        elif type(cur_el) == dict and 'list_item_spec' in cur_el.keys():
+            cur_el = cur_el['list_item_spec']
         elif type(cur_el) == list:
             found = False
             for cur_el_item in cur_el:
                 if cur_el_item['item_name'] == id:
                     cur_el = cur_el_item
+                    # if we need to go further, we may need to 'skip' a step here
+                    # but not if we're done
+                    if id_parts[-1] != id_part and type(cur_el) == dict:
+                        if "map_item_spec" in cur_el:
+                            cur_el = cur_el["map_item_spec"]
+                        elif "list_item_spec" in cur_el:
+                            cur_el = cur_el["list_item_spec"]
                     found = True
             if not found:
-                raise isc.cc.data.DataNotFoundError(id + " in " + str(cur_el))
+                raise isc.cc.data.DataNotFoundError(id + " not found")
         else:
             raise isc.cc.data.DataNotFoundError("Not a correct config specification")
     return cur_el
@@ -354,26 +364,63 @@ class MultiConfigData:
            See get_value() for a general way to find a configuration
            value
         """
-        if identifier[0] == '/':
-            identifier = identifier[1:]
-        module, sep, id = identifier.partition("/")
         try:
+            if identifier[0] == '/':
+                identifier = identifier[1:]
+            module, sep, id = identifier.partition("/")
+            # if there is a 'higher-level' list index specified, we need
+            # to check if that list specification has a default that
+            # overrides the more specific default in the final spec item
+            # (ie. list_default = [1, 2, 3], list_item_spec=int, default=0)
+            # def default list[1] should return 2, not 0
+            id_parts = isc.cc.data.split_identifier(id)
+            id_prefix = ""
+            while len(id_parts) > 0:
+                id_part = id_parts.pop(0)
+                item_id, list_indices = isc.cc.data.split_identifier_list_indices(id_part)
+                id_list = module + "/" + id_prefix + "/" + item_id
+                id_prefix += "/" + id_part
+                if list_indices is not None:
+                    # there's actually two kinds of default here for
+                    # lists; they can have a default value (like an
+                    # empty list), but their elements can  also have
+                    # default values.
+                    # So if the list item *itself* is a default,
+                    # we need to get the value out of that. If not, we
+                    # need to find the default for the specific element.
+                    list_value, type = self.get_value(id_list) 
+                    list_spec = find_spec_part(self._specifications[module].get_config_spec(), id_prefix)
+                    if type == self.DEFAULT:
+                        if 'item_default' in list_spec:
+                            list_value = list_spec['item_default']
+                            for i in list_indices:
+                                if i < len(list_value):
+                                    list_value = list_value[i]
+                                else:
+                                    # out of range, return None
+                                    return None
+                                
+                            if len(id_parts) > 0:
+                                rest_of_id = "/".join(id_parts)
+                                return isc.cc.data.find(list_value, rest_of_id)
+                            else:
+                                return list_value
+                    else:
+                        # we do have a non-default list, see if our indices
+                        # exist
+                        for i in list_indices:
+                            if i < len(list_value):
+                                list_value = list_value[i]
+                            else:
+                                # out of range, return None
+                                return None
+                    
             spec = find_spec_part(self._specifications[module].get_config_spec(), id)
             if 'item_default' in spec:
-                id, list_indices = isc.cc.data.split_identifier_list_indices(id)
-                if list_indices is not None and \
-                   type(spec['item_default']) == list:
-                    if len(list_indices) == 1:
-                        default_list = spec['item_default']
-                        index = list_indices[0]
-                        if index < len(default_list):
-                            return default_list[index]
-                        else:
-                            return None
-                else:
-                    return spec['item_default']
+                return spec['item_default']
             else:
                 return None
+
         except isc.cc.data.DataNotFoundError as dnfe:
             return None
 
@@ -398,13 +445,60 @@ class MultiConfigData:
                 return value, self.DEFAULT
         return None, self.NONE
 
-    def get_value_maps(self, identifier = None):
+    def _append_value_item(self, result, spec_part, identifier, all, first = False):
+        # Look at the spec; it is a list of items, or a map containing 'item_name' etc
+        if type(spec_part) == list:
+            for spec_part_element in spec_part:
+                spec_part_element_name = spec_part_element['item_name']
+                self._append_value_item(result, spec_part_element, identifier + "/" + spec_part_element_name, all)
+        elif type(spec_part) == dict:
+            # depending on item type, and the value of argument 'all'
+            # we need to either add an item, or recursively go on
+            # In the case of a list that is empty, we do need to show that
+            item_name = spec_part['item_name']
+            item_type = spec_part['item_type']
+            if item_type == "list" and (all or first):
+                spec_part_list = spec_part['list_item_spec']
+                list_value, status = self.get_value(identifier)
+                if list_value is None:
+                    print("Error: identifier '%s' not found" % identifier)
+                    return
+                if type(list_value) != list:
+                    # the identifier specified a single element
+                    self._append_value_item(result, spec_part_list, identifier, all)
+                else:
+                    list_len = len(list_value)
+                    if len(list_value) == 0 and (all or first):
+                        entry = _create_value_map_entry(identifier,
+                                                        item_type,
+                                                        [], status)
+                        result.append(entry)
+                    else:
+                        for i in range(len(list_value)):
+                            self._append_value_item(result, spec_part_list, "%s[%d]" % (identifier, i), all)
+            elif item_type == "map":
+                # just show the specific contents of a map, we are
+                # almost never interested in just its name
+                spec_part_map = spec_part['map_item_spec']
+                self._append_value_item(result, spec_part_map, identifier, all)
+            else:
+                value, status = self.get_value(identifier)
+                entry = _create_value_map_entry(identifier,
+                                                item_type,
+                                                value, status)
+                result.append(entry)
+        return
+
+
+    def get_value_maps(self, identifier = None, all = False):
         """Returns a list of dicts, containing the following values:
            name: name of the entry (string)
            type: string containing the type of the value (or 'module')
            value: value of the entry if it is a string, int, double or bool
-           modified: true if the value is a local change
-           default: true if the value has been changed
+           modified: true if the value is a local change that has not
+                     been committed
+           default: true if the value has not been changed (i.e. the
+                    value is the default from the specification)
            TODO: use the consts for those last ones
            Throws DataNotFoundError if the identifier is bad
         """
@@ -412,8 +506,14 @@ class MultiConfigData:
         if not identifier:
             # No identifier, so we need the list of current modules
             for module in self._specifications.keys():
-                entry = _create_value_map_entry(module, 'module', None)
-                result.append(entry)
+                if all:
+                    spec = self.get_module_spec(module)
+                    if spec:
+                        spec_part = spec.get_config_spec()
+                        self._append_value_item(result, spec_part, module, all, True)
+                else:
+                    entry = _create_value_map_entry(module, 'module', None)
+                    result.append(entry)
         else:
             if identifier[0] == '/':
                 identifier = identifier[1:]
@@ -421,42 +521,7 @@ class MultiConfigData:
             spec = self.get_module_spec(module)
             if spec:
                 spec_part = find_spec_part(spec.get_config_spec(), id)
-                if type(spec_part) == list:
-                    # list of items to show
-                    for item in spec_part:
-                        value, status = self.get_value("/" + identifier\
-                                              + "/" + item['item_name'])
-                        entry = _create_value_map_entry(item['item_name'],
-                                                        item['item_type'],
-                                                        value, status)
-                        result.append(entry)
-                elif type(spec_part) == dict:
-                    # Sub-specification
-                    item = spec_part
-                    if item['item_type'] == 'list':
-                        li_spec = item['list_item_spec']
-                        value, status =  self.get_value("/" + identifier)
-                        if type(value) == list:
-                            for list_value in value:
-                                result_part2 = _create_value_map_entry(
-                                                   li_spec['item_name'],
-                                                   li_spec['item_type'],
-                                                   list_value)
-                                result.append(result_part2)
-                        elif value is not None:
-                            entry = _create_value_map_entry(
-                                        li_spec['item_name'],
-                                        li_spec['item_type'],
-                                        value, status)
-                            result.append(entry)
-                    else:
-                        value, status = self.get_value("/" + identifier)
-                        if value is not None:
-                            entry = _create_value_map_entry(
-                                        item['item_name'],
-                                        item['item_type'],
-                                        value, status)
-                            result.append(entry)
+                self._append_value_item(result, spec_part, identifier, all, True)
         return result
 
     def set_value(self, identifier, value):

src/lib/python/isc/config/tests/config_data_test.py

@@ -358,6 +358,8 @@ class TestMultiConfigData(unittest.TestCase):
         self.assertEqual(1, value)
         value = self.mcd.get_default_value("Spec2/item5[0]")
         self.assertEqual('a', value)
+        value = self.mcd.get_default_value("Spec2/item5[1]")
+        self.assertEqual('b', value)
         value = self.mcd.get_default_value("Spec2/item5[5]")
         self.assertEqual(None, value)
         value = self.mcd.get_default_value("Spec2/item5[0][1]")
@@ -392,6 +394,10 @@ class TestMultiConfigData(unittest.TestCase):
         self.assertEqual(None, value)
         self.assertEqual(MultiConfigData.NONE, status)
 
+        value, status = self.mcd.get_value("Spec2/item5")
+        self.assertEqual(['a', 'b'], value)
+        self.assertEqual(MultiConfigData.DEFAULT, status)
+
         value, status = self.mcd.get_value("Spec2/item5[0]")
         self.assertEqual("a", value)
         self.assertEqual(MultiConfigData.DEFAULT, status)
@@ -400,6 +406,11 @@ class TestMultiConfigData(unittest.TestCase):
         self.assertEqual(None, value)
         self.assertEqual(MultiConfigData.NONE, status)
 
+        value, status = self.mcd.get_value("Spec2/item5[1]")
+        self.assertEqual("b", value)
+        self.assertEqual(MultiConfigData.DEFAULT, status)
+
+
 
     def test_get_value_maps(self):
         maps = self.mcd.get_value_maps()
@@ -423,32 +434,34 @@ class TestMultiConfigData(unittest.TestCase):
         self.mcd._set_current_config({ "Spec2": { "item1": 2 } })
         self.mcd.set_value("Spec2/item3", False)
         maps = self.mcd.get_value_maps("/Spec2")
-        self.assertEqual([{'default': False, 'type': 'integer', 'name': 'item1', 'value': 2, 'modified': False},
-                          {'default': True, 'type': 'real', 'name': 'item2', 'value': 1.1, 'modified': False},
-                          {'default': False, 'type': 'boolean', 'name': 'item3', 'value': False, 'modified': True},
-                          {'default': True, 'type': 'string', 'name': 'item4', 'value': 'test', 'modified': False},
-                          {'default': True, 'type': 'list', 'name': 'item5', 'value': ['a', 'b'], 'modified': False},
-                          {'default': True, 'type': 'map', 'name': 'item6', 'value': {}, 'modified': False}], maps)
+        self.assertEqual([{'default': False, 'type': 'integer', 'name': 'Spec2/item1', 'value': 2, 'modified': False},
+                          {'default': True, 'type': 'real', 'name': 'Spec2/item2', 'value': 1.1, 'modified': False},
+                          {'default': False, 'type': 'boolean', 'name': 'Spec2/item3', 'value': False, 'modified': True},
+                          {'default': True, 'type': 'string', 'name': 'Spec2/item4', 'value': 'test', 'modified': False},
+                          {'default': True, 'type': 'list', 'name': 'Spec2/item5', 'value': ['a', 'b'], 'modified': False},
+                          {'default': True, 'type': 'string', 'name': 'Spec2/item6/value1', 'value': 'default', 'modified': False},
+                          {'default': False, 'type': 'integer', 'name': 'Spec2/item6/value2', 'value': None, 'modified': False}], maps)
         maps = self.mcd.get_value_maps("Spec2")
-        self.assertEqual([{'default': False, 'type': 'integer', 'name': 'item1', 'value': 2, 'modified': False},
-                          {'default': True, 'type': 'real', 'name': 'item2', 'value': 1.1, 'modified': False},
-                          {'default': False, 'type': 'boolean', 'name': 'item3', 'value': False, 'modified': True},
-                          {'default': True, 'type': 'string', 'name': 'item4', 'value': 'test', 'modified': False},
-                          {'default': True, 'type': 'list', 'name': 'item5', 'value': ['a', 'b'], 'modified': False},
-                          {'default': True, 'type': 'map', 'name': 'item6', 'value': {}, 'modified': False}], maps)
+        self.assertEqual([{'default': False, 'type': 'integer', 'name': 'Spec2/item1', 'value': 2, 'modified': False},
+                          {'default': True, 'type': 'real', 'name': 'Spec2/item2', 'value': 1.1, 'modified': False},
+                          {'default': False, 'type': 'boolean', 'name': 'Spec2/item3', 'value': False, 'modified': True},
+                          {'default': True, 'type': 'string', 'name': 'Spec2/item4', 'value': 'test', 'modified': False},
+                          {'default': True, 'type': 'list', 'name': 'Spec2/item5', 'value': ['a', 'b'], 'modified': False},
+                          {'default': True, 'type': 'string', 'name': 'Spec2/item6/value1', 'value': 'default', 'modified': False},
+                          {'default': False, 'type': 'integer', 'name': 'Spec2/item6/value2', 'value': None, 'modified': False}], maps)
         maps = self.mcd.get_value_maps("/Spec2/item5")
-        self.assertEqual([{'default': False, 'type': 'string', 'name': 'list_element', 'value': 'a', 'modified': False},
-                          {'default': False, 'type': 'string', 'name': 'list_element', 'value': 'b', 'modified': False}], maps)
+        self.assertEqual([{'default': True, 'type': 'string', 'name': 'Spec2/item5[0]', 'value': 'a', 'modified': False},
+                          {'default': True, 'type': 'string', 'name': 'Spec2/item5[1]', 'value': 'b', 'modified': False}], maps)
         maps = self.mcd.get_value_maps("/Spec2/item5[0]")
-        self.assertEqual([{'default': True, 'modified': False, 'name': 'list_element', 'type': 'string', 'value': 'a'}], maps)
+        self.assertEqual([{'default': True, 'modified': False, 'name': 'Spec2/item5[0]', 'type': 'string', 'value': 'a'}], maps)
         maps = self.mcd.get_value_maps("/Spec2/item1")
-        self.assertEqual([{'default': False, 'type': 'integer', 'name': 'item1', 'value': 2, 'modified': False}], maps)
+        self.assertEqual([{'default': False, 'type': 'integer', 'name': 'Spec2/item1', 'value': 2, 'modified': False}], maps)
         maps = self.mcd.get_value_maps("/Spec2/item2")
-        self.assertEqual([{'default': True, 'type': 'real', 'name': 'item2', 'value': 1.1, 'modified': False}], maps)
+        self.assertEqual([{'default': True, 'type': 'real', 'name': 'Spec2/item2', 'value': 1.1, 'modified': False}], maps)
         maps = self.mcd.get_value_maps("/Spec2/item3")
-        self.assertEqual([{'default': False, 'type': 'boolean', 'name': 'item3', 'value': False, 'modified': True}], maps)
+        self.assertEqual([{'default': False, 'type': 'boolean', 'name': 'Spec2/item3', 'value': False, 'modified': True}], maps)
         maps = self.mcd.get_value_maps("/Spec2/item4")
-        self.assertEqual([{'default': True, 'type': 'string', 'name': 'item4', 'value': 'test', 'modified': False}], maps)
+        self.assertEqual([{'default': True, 'type': 'string', 'name': 'Spec2/item4', 'value': 'test', 'modified': False}], maps)
 
         module_spec = isc.config.module_spec_from_file(self.data_path + os.sep + "spec24.spec")
         self.mcd.set_specification(module_spec)
@@ -456,9 +469,30 @@ class TestMultiConfigData(unittest.TestCase):
         self.assertEqual([], maps)
         self.mcd._set_current_config({ "Spec24": { "item": [] } })
         maps = self.mcd.get_value_maps("/Spec24/item")
-        self.assertEqual([], maps)
-
+        self.assertEqual([{'default': False, 'modified': False, 'name': 'Spec24/item', 'type': 'list', 'value': []}], maps)
 
+        module_spec = isc.config.module_spec_from_file(self.data_path + os.sep + "spec22.spec")
+        self.mcd.set_specification(module_spec)
+        expected = [{'default': True,
+                     'modified': False,
+                     'name': 'Spec22/value9/v91',
+                     'type': 'string',
+                     'value': 'def'},
+                    {'default': True,
+                     'modified': False,
+                     'name': 'Spec22/value9/v92/v92a',
+                     'type': 'string',
+                     'value': 'Hello'
+                    },
+                    {'default': True,
+                     'modified': False,
+                     'name': 'Spec22/value9/v92/v92b',
+                     'type': 'integer',
+                     'value': 47806
+                    }
+                   ]
+        maps = self.mcd.get_value_maps("/Spec22/value9")
+        self.assertEqual(expected, maps)
 
     def test_set_value(self):
         module_spec = isc.config.module_spec_from_file(self.data_path + os.sep + "spec2.spec")

src/lib/python/isc/notify/tests/notify_out_test.py

@@ -22,9 +22,45 @@ import socket
 from isc.datasrc import sqlite3_ds
 from isc.notify import notify_out, SOCK_DATA
 
+# our fake socket, where we can read and insert messages
+class MockSocket():
+    def __init__(self, family, type):
+        self.family = family
+        self.type = type
+        self._local_sock, self._remote_sock = socket.socketpair()
+
+    def connect(self, to):
+        pass
+
+    def fileno(self):
+        return self._local_sock.fileno()
+
+    def close(self):
+        self._local_sock.close()
+        self._remote_sock.close()
+
+    def sendto(self, data, flag, dst):
+        return self._local_sock.send(data)
+
+    def recvfrom(self, length):
+        data = self._local_sock.recv(length)
+        return (data, None)
+
+    # provide a remote end which can write data to MockSocket for testing.
+    def remote_end(self):
+        return self._remote_sock
+
+# We subclass the ZoneNotifyInfo class we're testing here, only
+# to override the prepare_notify_out() method.
+class MockZoneNotifyInfo(notify_out.ZoneNotifyInfo):
+    def prepare_notify_out(self):
+        super().prepare_notify_out();
+        self._sock.close()
+        self._sock = MockSocket(socket.AF_INET, socket.SOCK_DGRAM)
+
 class TestZoneNotifyInfo(unittest.TestCase):
     def setUp(self):
-        self.info = notify_out.ZoneNotifyInfo('cn.', 'IN')
+        self.info = notify_out.ZoneNotifyInfo('example.net.', 'IN')
 
     def test_prepare_finish_notify_out(self):
         self.info.prepare_notify_out()
@@ -46,7 +82,7 @@ class TestZoneNotifyInfo(unittest.TestCase):
         self.info.set_next_notify_target()
         self.assertIsNone(self.info.get_current_notify_target())
 
-        temp_info = notify_out.ZoneNotifyInfo('com.', 'IN')
+        temp_info = notify_out.ZoneNotifyInfo('example.com.', 'IN')
         temp_info.prepare_notify_out()
         self.assertIsNone(temp_info.get_current_notify_target())
 
@@ -54,16 +90,16 @@ class TestZoneNotifyInfo(unittest.TestCase):
 class TestNotifyOut(unittest.TestCase):
     def setUp(self):
         self._db_file = tempfile.NamedTemporaryFile(delete=False)
-        sqlite3_ds.load(self._db_file.name, 'cn.', self._cn_data_reader)
-        sqlite3_ds.load(self._db_file.name, 'com.', self._com_data_reader)
+        sqlite3_ds.load(self._db_file.name, 'example.net.', self._example_net_data_reader)
+        sqlite3_ds.load(self._db_file.name, 'example.com.', self._example_com_data_reader)
         self._notify = notify_out.NotifyOut(self._db_file.name)
-        self._notify._notify_infos[('com.', 'IN')] = notify_out.ZoneNotifyInfo('com.', 'IN')
-        self._notify._notify_infos[('com.', 'CH')] = notify_out.ZoneNotifyInfo('com.', 'CH')
-        self._notify._notify_infos[('cn.', 'IN')] = notify_out.ZoneNotifyInfo('cn.', 'IN')
-        self._notify._notify_infos[('org.', 'IN')] = notify_out.ZoneNotifyInfo('org.', 'IN')
-        self._notify._notify_infos[('org.', 'CH')] = notify_out.ZoneNotifyInfo('org.', 'CH')
-        
-        info = self._notify._notify_infos[('cn.', 'IN')]
+        self._notify._notify_infos[('example.com.', 'IN')] = MockZoneNotifyInfo('example.com.', 'IN')
+        self._notify._notify_infos[('example.com.', 'CH')] = MockZoneNotifyInfo('example.com.', 'CH')
+        self._notify._notify_infos[('example.net.', 'IN')] = MockZoneNotifyInfo('example.net.', 'IN')
+        self._notify._notify_infos[('example.org.', 'IN')] = MockZoneNotifyInfo('example.org.', 'IN')
+        self._notify._notify_infos[('example.org.', 'CH')] = MockZoneNotifyInfo('example.org.', 'CH')
+
+        info = self._notify._notify_infos[('example.net.', 'IN')]
         info.notify_slaves.append(('127.0.0.1', 53))
         info.notify_slaves.append(('1.1.1.1', 5353))
 
@@ -72,62 +108,59 @@ class TestNotifyOut(unittest.TestCase):
         os.unlink(self._db_file.name)
 
     def test_send_notify(self):
-        self._notify.send_notify('cn')
+        self._notify.send_notify('example.net')
         self.assertEqual(self._notify.notify_num, 1)
-        self.assertEqual(self._notify._notifying_zones[0], ('cn.','IN'))
+        self.assertEqual(self._notify._notifying_zones[0], ('example.net.','IN'))
 
-        self._notify.send_notify('com')
+        self._notify.send_notify('example.com')
         self.assertEqual(self._notify.notify_num, 2)
-        self.assertEqual(self._notify._notifying_zones[1], ('com.','IN'))
+        self.assertEqual(self._notify._notifying_zones[1], ('example.com.','IN'))
 
         notify_out._MAX_NOTIFY_NUM = 3
-        self._notify.send_notify('com', 'CH')
+        self._notify.send_notify('example.com', 'CH')
         self.assertEqual(self._notify.notify_num, 3)
-        self.assertEqual(self._notify._notifying_zones[2], ('com.','CH'))
-    
-        self._notify.send_notify('org.')
-        self.assertEqual(self._notify._waiting_zones[0], ('org.', 'IN'))
-        self._notify.send_notify('org.')
+        self.assertEqual(self._notify._notifying_zones[2], ('example.com.','CH'))
+
+        self._notify.send_notify('example.org.')
+        self.assertEqual(self._notify._waiting_zones[0], ('example.org.', 'IN'))
+        self._notify.send_notify('example.org.')
         self.assertEqual(1, len(self._notify._waiting_zones))
 
-        self._notify.send_notify('org.', 'CH')
+        self._notify.send_notify('example.org.', 'CH')
         self.assertEqual(2, len(self._notify._waiting_zones))
-        self.assertEqual(self._notify._waiting_zones[1], ('org.', 'CH'))
+        self.assertEqual(self._notify._waiting_zones[1], ('example.org.', 'CH'))
 
     def test_wait_for_notify_reply(self):
-        self._notify.send_notify('cn.')
-        self._notify.send_notify('com.')
-    
+        self._notify.send_notify('example.net.')
+        self._notify.send_notify('example.com.')
+
         notify_out._MAX_NOTIFY_NUM = 2
-        self._notify.send_notify('org.')
+        self._notify.send_notify('example.org.')
         replied_zones, timeout_zones = self._notify._wait_for_notify_reply()
         self.assertEqual(len(replied_zones), 0)
         self.assertEqual(len(timeout_zones), 2)
 
         # Now make one socket be readable
-        addr = ('localhost', 12340)
-        self._notify._notify_infos[('cn.', 'IN')]._sock.bind(addr)
-        self._notify._notify_infos[('cn.', 'IN')].notify_timeout = time.time() + 10
-        self._notify._notify_infos[('com.', 'IN')].notify_timeout = time.time() + 10
-        
-        send_fd = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        self._notify._notify_infos[('example.net.', 'IN')].notify_timeout = time.time() + 10
+        self._notify._notify_infos[('example.com.', 'IN')].notify_timeout = time.time() + 10
+
         #Send some data to socket 12340, to make the target socket be readable
-        send_fd.sendto(b'data', addr)
+        self._notify._notify_infos[('example.net.', 'IN')]._sock.remote_end().send(b'data')
         replied_zones, timeout_zones = self._notify._wait_for_notify_reply()
         self.assertEqual(len(replied_zones), 1)
         self.assertEqual(len(timeout_zones), 1)
-        self.assertTrue(('cn.', 'IN') in replied_zones.keys())
-        self.assertTrue(('com.', 'IN') in timeout_zones.keys())
-        self.assertLess(time.time(), self._notify._notify_infos[('com.', 'IN')].notify_timeout)
-    
+        self.assertTrue(('example.net.', 'IN') in replied_zones.keys())
+        self.assertTrue(('example.com.', 'IN') in timeout_zones.keys())
+        self.assertLess(time.time(), self._notify._notify_infos[('example.com.', 'IN')].notify_timeout)
+
     def test_wait_for_notify_reply_2(self):
         # Test the returned value when the read_side socket is readable.
-        self._notify.send_notify('cn.')
-        self._notify.send_notify('com.')
+        self._notify.send_notify('example.net.')
+        self._notify.send_notify('example.com.')
 
         # Now make one socket be readable
-        self._notify._notify_infos[('cn.', 'IN')].notify_timeout = time.time() + 10
-        self._notify._notify_infos[('com.', 'IN')].notify_timeout = time.time() + 10
+        self._notify._notify_infos[('example.net.', 'IN')].notify_timeout = time.time() + 10
+        self._notify._notify_infos[('example.com.', 'IN')].notify_timeout = time.time() + 10
         self._notify._read_sock, self._notify._write_sock = socket.socketpair()
         self._notify._write_sock.send(SOCK_DATA)
         replied_zones, timeout_zones = self._notify._wait_for_notify_reply()
@@ -135,13 +168,13 @@ class TestNotifyOut(unittest.TestCase):
         self.assertEqual(0, len(timeout_zones))
 
     def test_notify_next_target(self):
-        self._notify.send_notify('cn.')
-        self._notify.send_notify('com.')
+        self._notify.send_notify('example.net.')
+        self._notify.send_notify('example.com.')
         notify_out._MAX_NOTIFY_NUM = 2
-        self._notify.send_notify('org.')
-        self._notify.send_notify('com.', 'CH')
+        self._notify.send_notify('example.org.')
+        self._notify.send_notify('example.com.', 'CH')
 
-        info = self._notify._notify_infos[('cn.', 'IN')]
+        info = self._notify._notify_infos[('example.net.', 'IN')]
         self._notify._notify_next_target(info)
         self.assertEqual(0, info.notify_try_num)
         self.assertEqual(info.get_current_notify_target(), ('1.1.1.1', 5353))
@@ -153,101 +186,101 @@ class TestNotifyOut(unittest.TestCase):
         self.assertEqual(2, self._notify.notify_num)
         self.assertEqual(1, len(self._notify._waiting_zones))
 
-        com_info = self._notify._notify_infos[('com.', 'IN')]
-        self._notify._notify_next_target(com_info)
+        example_com_info = self._notify._notify_infos[('example.com.', 'IN')]
+        self._notify._notify_next_target(example_com_info)
         self.assertEqual(2, self._notify.notify_num)
         self.assertEqual(2, len(self._notify._notifying_zones))
-    
+
     def test_handle_notify_reply(self):
         self.assertEqual(notify_out._BAD_REPLY_PACKET, self._notify._handle_notify_reply(None, b'badmsg'))
-        com_info = self._notify._notify_infos[('com.', 'IN')]
-        com_info.notify_msg_id = 0X2f18
+        example_com_info = self._notify._notify_infos[('example.com.', 'IN')]
+        example_com_info.notify_msg_id = 0X2f18
 
         # test with right notify reply message
-        data = b'\x2f\x18\xa0\x00\x00\x01\x00\x00\x00\x00\x00\x00\x03com\x00\x00\x06\x00\x01'
-        self.assertEqual(notify_out._REPLY_OK, self._notify._handle_notify_reply(com_info, data))
+        data = b'\x2f\x18\xa0\x00\x00\x01\x00\x00\x00\x00\x00\x00\x07example\03com\x00\x00\x06\x00\x01'
+        self.assertEqual(notify_out._REPLY_OK, self._notify._handle_notify_reply(example_com_info, data))
 
         # test with unright query id
-        data = b'\x2e\x18\xa0\x00\x00\x01\x00\x00\x00\x00\x00\x00\x03com\x00\x00\x06\x00\x01'
-        self.assertEqual(notify_out._BAD_QUERY_ID, self._notify._handle_notify_reply(com_info, data))
+        data = b'\x2e\x18\xa0\x00\x00\x01\x00\x00\x00\x00\x00\x00\x07example\03com\x00\x00\x06\x00\x01'
+        self.assertEqual(notify_out._BAD_QUERY_ID, self._notify._handle_notify_reply(example_com_info, data))
 
         # test with unright query name
-        data = b'\x2f\x18\xa0\x00\x00\x01\x00\x00\x00\x00\x00\x00\x02cn\x00\x00\x06\x00\x01'
-        self.assertEqual(notify_out._BAD_QUERY_NAME, self._notify._handle_notify_reply(com_info, data))
+        data = b'\x2f\x18\xa0\x00\x00\x01\x00\x00\x00\x00\x00\x00\x07example\03net\x00\x00\x06\x00\x01'
+        self.assertEqual(notify_out._BAD_QUERY_NAME, self._notify._handle_notify_reply(example_com_info, data))
 
         # test with unright opcode
-        data = b'\x2f\x18\x80\x00\x00\x01\x00\x00\x00\x00\x00\x00\x03com\x00\x00\x06\x00\x01'
-        self.assertEqual(notify_out._BAD_OPCODE, self._notify._handle_notify_reply(com_info, data))
+        data = b'\x2f\x18\x80\x00\x00\x01\x00\x00\x00\x00\x00\x00\x07example\03com\x00\x00\x06\x00\x01'
+        self.assertEqual(notify_out._BAD_OPCODE, self._notify._handle_notify_reply(example_com_info, data))
 
         # test with unright qr
-        data = b'\x2f\x18\x10\x10\x00\x01\x00\x00\x00\x00\x00\x00\x03com\x00\x00\x06\x00\x01'
-        self.assertEqual(notify_out._BAD_QR, self._notify._handle_notify_reply(com_info, data))
+        data = b'\x2f\x18\x10\x10\x00\x01\x00\x00\x00\x00\x00\x00\x07example\03com\x00\x00\x06\x00\x01'
+        self.assertEqual(notify_out._BAD_QR, self._notify._handle_notify_reply(example_com_info, data))
 
     def test_send_notify_message_udp(self):
-        com_info = self._notify._notify_infos[('cn.', 'IN')]
-        com_info.prepare_notify_out()
-        ret = self._notify._send_notify_message_udp(com_info, ('1.1.1.1', 53))
+        example_com_info = self._notify._notify_infos[('example.net.', 'IN')]
+        example_com_info.prepare_notify_out()
+        ret = self._notify._send_notify_message_udp(example_com_info, ('1.1.1.1', 53))
         self.assertTrue(ret)
 
     def test_zone_notify_handler(self):
         old_send_msg = self._notify._send_notify_message_udp
-        def _fake_send_notify_message_udp(va1, va2): 
+        def _fake_send_notify_message_udp(va1, va2):
             pass
         self._notify._send_notify_message_udp = _fake_send_notify_message_udp
-        self._notify.send_notify('cn.')
-        self._notify.send_notify('com.')
+        self._notify.send_notify('example.net.')
+        self._notify.send_notify('example.com.')
         notify_out._MAX_NOTIFY_NUM = 2
-        self._notify.send_notify('org.')
+        self._notify.send_notify('example.org.')
 
-        cn_info = self._notify._notify_infos[('cn.', 'IN')]
-        cn_info.prepare_notify_out()
+        example_net_info = self._notify._notify_infos[('example.net.', 'IN')]
+        example_net_info.prepare_notify_out()
 
-        cn_info.notify_try_num = 2
-        self._notify._zone_notify_handler(cn_info, notify_out._EVENT_TIMEOUT)
-        self.assertEqual(3, cn_info.notify_try_num)
+        example_net_info.notify_try_num = 2
+        self._notify._zone_notify_handler(example_net_info, notify_out._EVENT_TIMEOUT)
+        self.assertEqual(3, example_net_info.notify_try_num)
 
-        time1 = cn_info.notify_timeout
-        self._notify._zone_notify_handler(cn_info, notify_out._EVENT_TIMEOUT)
-        self.assertEqual(4, cn_info.notify_try_num)
-        self.assertGreater(cn_info.notify_timeout, time1 + 2) # bigger than 2 seconds
+        time1 = example_net_info.notify_timeout
+        self._notify._zone_notify_handler(example_net_info, notify_out._EVENT_TIMEOUT)
+        self.assertEqual(4, example_net_info.notify_try_num)
+        self.assertGreater(example_net_info.notify_timeout, time1 + 2) # bigger than 2 seconds
 
-        cur_tgt = cn_info._notify_current
-        cn_info.notify_try_num = notify_out._MAX_NOTIFY_TRY_NUM
-        self._notify._zone_notify_handler(cn_info, notify_out._EVENT_NONE)
-        self.assertNotEqual(cur_tgt, cn_info._notify_current)
+        cur_tgt = example_net_info._notify_current
+        example_net_info.notify_try_num = notify_out._MAX_NOTIFY_TRY_NUM
+        self._notify._zone_notify_handler(example_net_info, notify_out._EVENT_NONE)
+        self.assertNotEqual(cur_tgt, example_net_info._notify_current)
 
-    def _cn_data_reader(self):
+    def _example_net_data_reader(self):
         zone_data = [
-        ('cn.',         '1000',  'IN',  'SOA', 'a.dns.cn. mail.cn. 1 1 1 1 1'),
-        ('cn.',         '1000',  'IN',  'NS',  'a.dns.cn.'),
-        ('cn.',         '1000',  'IN',  'NS',  'b.dns.cn.'),
-        ('cn.',         '1000',  'IN',  'NS',  'c.dns.cn.'),
-        ('a.dns.cn.',   '1000',  'IN',  'A',    '1.1.1.1'),
-        ('a.dns.cn.',   '1000',  'IN',  'AAAA', '2:2::2:2'),
-        ('b.dns.cn.',   '1000',  'IN',  'A',    '3.3.3.3'),
-        ('b.dns.cn.',   '1000',  'IN',  'AAAA', '4:4::4:4'),
-        ('b.dns.cn.',   '1000',  'IN',  'AAAA', '5:5::5:5'),
-        ('c.dns.cn.',   '1000',  'IN',  'A',    '6.6.6.6'),
-        ('c.dns.cn.',   '1000',  'IN',  'A',    '7.7.7.7'),
-        ('c.dns.cn.',   '1000',  'IN',  'AAAA', '8:8::8:8')]
+        ('example.net.',         '1000',  'IN',  'SOA', 'a.dns.example.net. mail.example.net. 1 1 1 1 1'),
+        ('example.net.',         '1000',  'IN',  'NS',  'a.dns.example.net.'),
+        ('example.net.',         '1000',  'IN',  'NS',  'b.dns.example.net.'),
+        ('example.net.',         '1000',  'IN',  'NS',  'c.dns.example.net.'),
+        ('a.dns.example.net.',   '1000',  'IN',  'A',    '1.1.1.1'),
+        ('a.dns.example.net.',   '1000',  'IN',  'AAAA', '2:2::2:2'),
+        ('b.dns.example.net.',   '1000',  'IN',  'A',    '3.3.3.3'),
+        ('b.dns.example.net.',   '1000',  'IN',  'AAAA', '4:4::4:4'),
+        ('b.dns.example.net.',   '1000',  'IN',  'AAAA', '5:5::5:5'),
+        ('c.dns.example.net.',   '1000',  'IN',  'A',    '6.6.6.6'),
+        ('c.dns.example.net.',   '1000',  'IN',  'A',    '7.7.7.7'),
+        ('c.dns.example.net.',   '1000',  'IN',  'AAAA', '8:8::8:8')]
         for item in zone_data:
             yield item
 
-    def _com_data_reader(self):
+    def _example_com_data_reader(self):
         zone_data = [
-        ('com.',         '1000',  'IN',  'SOA', 'a.dns.com. mail.com. 1 1 1 1 1'),
-        ('com.',         '1000',  'IN',  'NS',  'a.dns.com.'),
-        ('com.',         '1000',  'IN',  'NS',  'b.dns.com.'),
-        ('com.',         '1000',  'IN',  'NS',  'c.dns.com.'),
-        ('a.dns.com.',   '1000',  'IN',  'A',    '1.1.1.1'),
-        ('b.dns.com.',   '1000',  'IN',  'A',    '3.3.3.3'),
-        ('b.dns.com.',   '1000',  'IN',  'AAAA', '4:4::4:4'),
-        ('b.dns.com.',   '1000',  'IN',  'AAAA', '5:5::5:5')]
+        ('example.com.',         '1000',  'IN',  'SOA', 'a.dns.example.com. mail.example.com. 1 1 1 1 1'),
+        ('example.com.',         '1000',  'IN',  'NS',  'a.dns.example.com.'),
+        ('example.com.',         '1000',  'IN',  'NS',  'b.dns.example.com.'),
+        ('example.com.',         '1000',  'IN',  'NS',  'c.dns.example.com.'),
+        ('a.dns.example.com.',   '1000',  'IN',  'A',    '1.1.1.1'),
+        ('b.dns.example.com.',   '1000',  'IN',  'A',    '3.3.3.3'),
+        ('b.dns.example.com.',   '1000',  'IN',  'AAAA', '4:4::4:4'),
+        ('b.dns.example.com.',   '1000',  'IN',  'AAAA', '5:5::5:5')]
         for item in zone_data:
             yield item
 
     def test_get_notify_slaves_from_ns(self):
-        records = self._notify._get_notify_slaves_from_ns('cn.')
+        records = self._notify._get_notify_slaves_from_ns('example.net.')
         self.assertEqual(6, len(records))
         self.assertEqual('8:8::8:8', records[5])
         self.assertEqual('7.7.7.7', records[4])
@@ -256,36 +289,36 @@ class TestNotifyOut(unittest.TestCase):
         self.assertEqual('4:4::4:4', records[1])
         self.assertEqual('3.3.3.3', records[0])
 
-        records = self._notify._get_notify_slaves_from_ns('com.')
+        records = self._notify._get_notify_slaves_from_ns('example.com.')
         self.assertEqual(3, len(records))
         self.assertEqual('5:5::5:5', records[2])
         self.assertEqual('4:4::4:4', records[1])
         self.assertEqual('3.3.3.3', records[0])
-    
+
     def test_init_notify_out(self):
         self._notify._init_notify_out(self._db_file.name)
-        self.assertListEqual([('3.3.3.3', 53), ('4:4::4:4', 53), ('5:5::5:5', 53)], 
-                             self._notify._notify_infos[('com.', 'IN')].notify_slaves)
-        
+        self.assertListEqual([('3.3.3.3', 53), ('4:4::4:4', 53), ('5:5::5:5', 53)],
+                             self._notify._notify_infos[('example.com.', 'IN')].notify_slaves)
+
     def test_prepare_select_info(self):
         timeout, valid_fds, notifying_zones = self._notify._prepare_select_info()
         self.assertEqual(notify_out._IDLE_SLEEP_TIME, timeout)
         self.assertListEqual([], valid_fds)
 
-        self._notify._notify_infos[('cn.', 'IN')]._sock = 1
-        self._notify._notify_infos[('cn.', 'IN')].notify_timeout = time.time() + 5
+        self._notify._notify_infos[('example.net.', 'IN')]._sock = 1
+        self._notify._notify_infos[('example.net.', 'IN')].notify_timeout = time.time() + 5
         timeout, valid_fds, notifying_zones = self._notify._prepare_select_info()
         self.assertGreater(timeout, 0)
         self.assertListEqual([1], valid_fds)
 
-        self._notify._notify_infos[('cn.', 'IN')]._sock = 1
-        self._notify._notify_infos[('cn.', 'IN')].notify_timeout = time.time() - 5
+        self._notify._notify_infos[('example.net.', 'IN')]._sock = 1
+        self._notify._notify_infos[('example.net.', 'IN')].notify_timeout = time.time() - 5
         timeout, valid_fds, notifying_zones = self._notify._prepare_select_info()
         self.assertEqual(timeout, 0)
         self.assertListEqual([1], valid_fds)
 
-        self._notify._notify_infos[('com.', 'IN')]._sock = 2
-        self._notify._notify_infos[('com.', 'IN')].notify_timeout = time.time() + 5
+        self._notify._notify_infos[('example.com.', 'IN')]._sock = 2
+        self._notify._notify_infos[('example.com.', 'IN')].notify_timeout = time.time() + 5
         timeout, valid_fds, notifying_zones = self._notify._prepare_select_info()
         self.assertEqual(timeout, 0)
         self.assertListEqual([2, 1], valid_fds)