added option to validate 'partial' data against a definition

added some more dummy specfile entries
creates two simple functions for making and reading answer messages
bindctl will parse command 'config set' values natively (i.e. for "config set my_item 3" the 3 is now read as an integer instead of a string)
added config diff option that shows a dict of the current uncommited changes


git-svn-id: svn://bind10.isc.org/svn/bind10/branches/jelte-configuration@823 e5f2f494-b856-4b98-b285-d166d9295462

src/bin/auth/auth.spec

@@ -18,6 +18,23 @@
             "item_default": ""
           }
       }
+    ],
+    "commands": [
+      {
+        "command_name": "print_message",
+        "command_description": "Print the given message to stdout",
+        "command_args": [ {
+          "item_name": "message",
+          "item_type": "string",
+          "item_optional": False,
+          "item_default": ""
+        } ]
+      },
+      {
+        "command_name": "shutdown",
+        "command_description": "Shut down BIND 10",
+        "command_args": []
+      }
     ]
   }
 }

src/bin/bind10/bind10.py.in

@@ -106,6 +106,7 @@ class BoB:
         self.verbose = verbose
         self.c_channel_port = c_channel_port
         self.cc_session = None
+        self.ccs = None
         self.processes = {}
         self.dead_processes = {}
         self.runnable = False
@@ -114,6 +115,18 @@ class BoB:
         if self.verbose:
             print("[XX] handling new config:")
             print(new_config)
+        errors = []
+        if self.ccs.get_config_data().get_specification().validate(False, new_config, errors):
+            print("[XX] new config validated")
+            self.ccs.set_config(new_config)
+            answer = { "result": [ 0 ] }
+        else:
+            print("[XX] new config validation failure")
+            if len(errors) > 0:
+                answer = { "result": [ 1, errors ] }
+            else:
+                answer = { "result": [ 1, "Unknown error in validation" ] }
+        return answer
         # TODO
 
     def command_handler(self, command):
@@ -121,7 +134,7 @@ class BoB:
         if self.verbose:
             print("[XX] Boss got command:")
             print(command)
-        answer = None
+        answer = [ 1, "Command not implemented" ]
         if type(command) != list or len(command) == 0:
             answer = { "result": [ 1, "bad command" ] }
         else:
@@ -134,6 +147,10 @@ class BoB:
                 if len(command) > 1 and type(command[1]) == dict and "message" in command[1]:
                     print(command[1]["message"])
                 answer = { "result": [ 0 ] }
+            elif cmd == "print_settings":
+                print("Config:")
+                print(self.ccs.get_config())
+                answer = { "result": [ 0 ] }
             else:
                 answer = { "result": [ 1, "Unknown command" ] }
         return answer
@@ -191,6 +208,7 @@ class BoB:
         if self.verbose:
             print("[XX] starting ccsession")
         self.ccs = isc.config.CCSession(SPECFILE_LOCATION, self.config_handler, self.command_handler)
+        self.ccs.start()
         if self.verbose:
             print("[XX] ccsession started")
 

src/bin/bind10/bob.spec

@@ -9,10 +9,10 @@
         "item_default": "Hi, shane!"
       },
       {
-        "item_name": "some_other_string",
+        "item_name": "some_int",
-        "item_type": "string",
+        "item_type": "integer",
         "item_optional": False,
-        "item_default": "Hi, shane!"
+        "item_default": 1
       }
     ],
     "commands": [
@@ -27,6 +27,16 @@
         } ]
       },
       {
+        "command_name": "print_settings",
+        "command_description": "Print some_string and some_int to stdout",
+        "command_args": [ {
+          "item_name": "message",
+          "item_type": "string",
+          "item_optional": True,
+          "item_default": ""
+        } ]
+      },
+      {
         "command_name": "shutdown",
         "command_description": "Shut down BIND 10",
         "command_args": []

src/bin/bindctl/bindcmd.py

@@ -31,6 +31,7 @@ import os, time, random, re
 import getpass
 from hashlib import sha1
 import csv
+import ast
 
 try:
     from collections import OrderedDict
@@ -445,13 +446,21 @@ class BindCmdInterpreter(Cmd):
             elif cmd.command == "remove":
                 self.config_data.remove_value(identifier, cmd.params['value'])
             elif cmd.command == "set":
-                self.config_data.set_value(identifier, cmd.params['value'])
+                parsed_value = None
+                try:
+                    parsed_value = ast.literal_eval(cmd.params['value'])
+                except Exception as exc:
+                    # ok could be an unquoted string, interpret as such
+                    parsed_value = cmd.params['value']
+                self.config_data.set_value(identifier, parsed_value)
             elif cmd.command == "unset":
                 self.config_data.unset(identifier)
             elif cmd.command == "revert":
                 self.config_data.revert()
             elif cmd.command == "commit":
                 self.config_data.commit()
+            elif cmd.command == "diff":
+                print(self.config_data.get_local_changes());
             elif cmd.command == "go":
                 self.go(identifier)
         except isc.cc.data.DataTypeError as dte:

src/bin/bindctl/bindctl.py

@@ -53,6 +53,9 @@ def prepare_config_commands(tool):
     cmd.add_param(param)
     module.add_command(cmd)
 
+    cmd = CommandInfo(name = "diff", desc = "Show all local changes", need_inst_param = False)
+    module.add_command(cmd)
+
     cmd = CommandInfo(name = "revert", desc = "Revert all local changes", need_inst_param = False)
     module.add_command(cmd)
 

src/bin/cmdctl/b10-cmdctl.py.in

@@ -168,8 +168,8 @@ class SecureHTTPRequestHandler(http.server.BaseHTTPRequestHandler):
             param = json.loads(post_str)
             # TODO, need return some proper return code. 
             # currently always OK.
-            reply = self.server.send_command_to_module(mod, cmd, param)
+        reply = self.server.send_command_to_module(mod, cmd, param)
-            print('b10-cmdctl finish send message \'%s\' to module %s' % (cmd, mod))            
+        print('b10-cmdctl finish send message \'%s\' to module %s' % (cmd, mod))            
 
         return rcode, reply
             

src/lib/config/python/isc/config/ccsession.py

@@ -25,10 +25,45 @@
 from isc.cc import Session
 import isc
 
+class CCSessionError(Exception): pass
+
+def parse_answer(msg):
+    """Returns a type (rcode, value), where value depends on the command
+       that was called. If rcode != 0, value is a string containing
+       an error message"""
+    if 'result' not in msg:
+        raise CCSessionError("answer message does not contain 'result' element")
+    elif type(msg['result']) != list:
+        raise CCSessionError("wrong result type in answer message")
+    elif len(msg['result']) < 1:
+        raise CCSessionError("empty result list in answer message")
+    elif type(msg['result'][0]) != int:
+        raise CCSessionError("wrong rcode type in answer message")
+    else:
+        if len(msg['result']) > 1:
+            return msg['result'][0], msg['result'][1]
+        else:
+            return msg['result'][0], None
+
+def create_answer(rcode, arg = None):
+    """Creates an answer packet for config&commands. rcode must be an
+       integer. If rcode == 0, arg is an optional value that depends
+       on what the command or option was. If rcode != 0, arg must be
+       a string containing an error message"""
+    if type(rcode) != int:
+        raise CCSessionError("rcode in create_answer() must be an integer")
+    if rcode != 0 and type(arg) != str:
+        raise CCSessionError("arg in create_answer for rcode != 0 must be a string describing the error")
+    if arg:
+        return { 'result': [ rcode, arg ] }
+    else:
+        return { 'result': [ 0 ] }
+
 class CCSession:
     def __init__(self, spec_file_name, config_handler, command_handler):
-        self._data_definition = isc.config.data_spec_from_file(spec_file_name)
+        data_definition = isc.config.data_spec_from_file(spec_file_name)
-        self._module_name = self._data_definition.get_module_name()
+        self._config_data = isc.config.config_data.ConfigData(data_definition)
+        self._module_name = data_definition.get_module_name()
         
         self.set_config_handler(config_handler)
         self.set_command_handler(command_handler)
@@ -36,8 +71,10 @@ class CCSession:
         self._session = Session()
         self._session.group_subscribe(self._module_name, "*")
 
+    def start(self):
+        print("[XX] SEND SPEC AND REQ CONFIG")
         self.__send_spec()
-        self.__get_full_config()
+        self.__request_config()
 
     def get_socket(self):
         """Returns the socket from the command channel session"""
@@ -48,6 +85,15 @@ class CCSession:
            application can use it directly"""
         return self._session
 
+    def set_config(self, new_config):
+        return self._config_data.set_local_config(new_config)
+
+    def get_config(self):
+        return self._config_data.get_local_config()
+
+    def get_config_data(self):
+        return self._config_data
+
     def close(self):
         self._session.close()
 
@@ -55,13 +101,18 @@ class CCSession:
         """Check whether there is a command on the channel.
            Call the command callback function if so"""
         msg, env = self._session.group_recvmsg(False)
+        # should we default to an answer? success-by-default? unhandled error?
         answer = None
-        if msg:
+        try:
-            if "config_update" in msg and self._config_handler:
+            if msg:
-                self._config_handler(msg["config_update"])
+                print("[XX] got msg: ")
-                answer = { "result": [ 0 ] }
+                print(msg)
-            if "command" in msg and self._command_handler:
+                if "config_update" in msg and self._config_handler:
-                answer = self._command_handler(msg["command"])
+                    answer = self._config_handler(msg["config_update"])
+                if "command" in msg and self._command_handler:
+                    answer = self._command_handler(msg["command"])
+        except Exception as exc:
+            answer = create_answer(1, str(exc))
         if answer:
             self._session.group_reply(env, answer)
 
@@ -69,32 +120,35 @@ class CCSession:
     def set_config_handler(self, config_handler):
         """Set the config handler for this module. The handler is a
            function that takes the full configuration and handles it.
-           It should return either { "result": [ 0 ] } or
+           It should return an answer created with create_answer()"""
-           { "result": [ <error_number>, "error message" ] }"""
         self._config_handler = config_handler
         # should we run this right now since we've changed the handler?
 
     def set_command_handler(self, command_handler):
         """Set the command handler for this module. The handler is a
            function that takes a command as defined in the .spec file
-           and return either { "result": [ 0, (result) ] } or
+           and return an answer created with create_answer()"""
-           { "result": [ <error_number>. "error message" ] }"""
         self._command_handler = command_handler
 
     def __send_spec(self):
         """Sends the data specification to the configuration manager"""
-        self._session.group_sendmsg({ "data_specification": self._data_definition.get_definition() }, "ConfigManager")
+        print("[XX] send spec for " + self._module_name + " to ConfigManager")
+        self._session.group_sendmsg({ "data_specification": self._config_data.get_specification().get_definition() }, "ConfigManager")
         answer, env = self._session.group_recvmsg(False)
+        print("[XX] got answer from cfgmgr:")
+        print(answer)
         
-    def __get_full_config(self):
+    def __request_config(self):
         """Asks the configuration manager for the current configuration, and call the config handler if set"""
         self._session.group_sendmsg({ "command": [ "get_config", { "module_name": self._module_name } ] }, "ConfigManager")
         answer, env = self._session.group_recvmsg(False)
-        if "result" in answer:
+        rcode, value = parse_answer(answer)
-            if answer["result"][0] == 0 and len(answer["result"]) > 1:
+        if rcode == 0:
-                new_config = answer["result"][1]
+            if self._config_data.get_specification().validate(False, value):
-                if self._data_definition.validate(new_config):
+                self._config_data.set_local_config(value);
-                    self._config = new_config;
+                if self._config_handler:
-                    if self._config_handler:
+                    self._config_handler(value)
-                        self._config_handler(answer["result"])
+        else:
+            # log error
+            print("Error requesting configuration: " + value)
     

src/lib/config/python/isc/config/cfgmgr.py

@@ -156,7 +156,6 @@ class ConfigManager:
                 commands[name] = self.data_specs[name].get_commands
         else:
             for module_name in self.data_specs.keys():
-                print("[XX] add commands for " + module_name)
                 commands[module_name] = self.data_specs[module_name].get_commands()
         return commands
 
@@ -218,24 +217,34 @@ class ConfigManager:
             if conf_part:
                 data.merge(conf_part, cmd[2])
                 self.cc.group_sendmsg({ "config_update": conf_part }, module_name)
+                answer, env = self.cc.group_recvmsg(False)
             else:
                 conf_part = data.set(self.config.data, module_name, {})
-                print("[XX] SET CONF PART:")
-                print(conf_part)
                 data.merge(conf_part[module_name], cmd[2])
                 # send out changed info
                 self.cc.group_sendmsg({ "config_update": conf_part[module_name] }, module_name)
-            self.write_config()
+                # replace 'our' answer with that of the module
-            answer["result"] = [ 0 ]
+                answer, env = selc.cc.group_recvmsg(False)
+                print("[XX] module responded with")
+                print(answer)
+            if answer and "result" in answer and answer['result'][0] == 0:
+                self.write_config()
         elif len(cmd) == 2:
             # todo: use api (and check the data against the definition?)
             data.merge(self.config.data, cmd[1])
             # send out changed info
+            got_error = False
             for module in self.config.data:
                 if module != "version":
                     self.cc.group_sendmsg({ "config_update": self.config.data[module] }, module)
-            self.write_config()
+                    answer, env = self.cc.group_recvmsg(False)
-            answer["result"] = [ 0 ]
+                    print("[XX] one module responded with")
+                    print(answer)
+                    if answer and 'result' in answer and answer['result'][0] != 0:
+                        got_error = True
+            if not got_error:
+                self.write_config()
+            # TODO rollback changes that did get through?
         else:
             answer["result"] = [ 1, "Wrong number of arguments" ]
         return answer
@@ -245,9 +254,9 @@ class ConfigManager:
         # todo: use DataDefinition class
         # todo: error checking (like keyerrors)
         answer = {}
-        print("[XX] CFGMGR got spec:")
-        print(spec)
         self.set_data_spec(spec)
+        print("[XX] cfgmgr add spec:")
+        print(spec)
         
         # We should make one general 'spec update for module' that
         # passes both specification and commands at once
@@ -259,23 +268,15 @@ class ConfigManager:
     def handle_msg(self, msg):
         """Handle a direct command"""
         answer = {}
-        print("[XX] cfgmgr got msg:")
-        print(msg)
         if "command" in msg:
             cmd = msg["command"]
             try:
                 if cmd[0] == "get_commands":
                     answer["result"] = [ 0, self.get_commands() ]
-                    print("[XX] get_commands answer:")
-                    print(answer)
                 elif cmd[0] == "get_data_spec":
                     answer = self._handle_get_data_spec(cmd)
-                    print("[XX] get_data_spec answer:")
-                    print(answer)
                 elif cmd[0] == "get_config":
                     answer = self._handle_get_config(cmd)
-                    print("[XX] get_config answer:")
-                    print(answer)
                 elif cmd[0] == "set_config":
                     answer = self._handle_set_config(cmd)
                 elif cmd[0] == "shutdown":
@@ -297,8 +298,6 @@ class ConfigManager:
             answer['result'] = [0]
         else:
             answer["result"] = [ 1, "Unknown message format: " + str(msg) ]
-        print("[XX] cfgmgr sending answer:")
-        print(answer)
         return answer
         
     def run(self):
@@ -307,6 +306,8 @@ class ConfigManager:
             msg, env = self.cc.group_recvmsg(False)
             if msg:
                 answer = self.handle_msg(msg);
+                print("[XX] CFGMGR Sending answer to UI:")
+                print(answer)
                 self.cc.group_reply(env, answer)
             else:
                 self.running = False

src/lib/config/python/isc/config/cfgmgr_test.py

@@ -119,24 +119,6 @@ class TestConfigManager(unittest.TestCase):
         # this one is actually wrong, but 'current status quo'
         self.assertEqual(msg, {"running": "configmanager"})
 
-    #def test_set_config(self):
-        #self.cm.set_config(self.name, self.spec)
-        #self.assertEqual(self.cm.data_definitions[self.name], self.spec)
-
-    #def test_remove_config(self):
-        #self.assertRaises(KeyError, self.cm.remove_config, self.name)
-        #self.cm.set_config(self.name, self.spec)
-        #self.cm.remove_config(self.name)
-
-    #def test_set_commands(self):
-    #    self.cm.set_commands(self.name, self.commands)
-    #    self.assertEqual(self.cm.commands[self.name], self.commands)
-
-    #def test_write_config(self):
-    #    self.assertRaises(KeyError, self.cm.remove_commands, self.name)
-    #    self.cm.set_commands(self.name, self.commands)
-    #    self.cm.remove_commands(self.name)
-
     def _handle_msg_helper(self, msg, expected_answer):
         answer = self.cm.handle_msg(msg)
         self.assertEqual(expected_answer, answer)

src/lib/config/python/isc/config/config_data.py

@@ -137,6 +137,25 @@ class ConfigData:
             return spec['item_default'], True
         return None, False
 
+    def get_specification(self):
+        """Returns the datadefinition"""
+        print(self.specification)
+        return self.specification
+
+    def set_local_config(self, data):
+        """Set the non-default config values, as passed by cfgmgr"""
+        self.data = data
+
+    def get_local_config(self):
+        """Returns the non-default config values in a dict"""
+        return self.config();
+
+    #def get_identifiers(self):
+    # Returns a list containing all identifiers
+
+    #def 
+
+
 class MultiConfigData:
     """This class stores the datadefinitions, current non-default
        configuration values and 'local' (uncommitted) changes."""
@@ -308,7 +327,7 @@ class MultiConfigData:
         """Set the local value at the given identifier to value"""
         # todo: validate
         isc.cc.data.set(self._local_changes, identifier, value)
-
+ 
     def get_config_item_list(self, identifier = None):
         """Returns a list of strings containing the item_names of
            the child items at the given identifier. If no identifier is
@@ -392,6 +411,9 @@ class UIConfigData():
     def get_value_maps(self, identifier = None):
         return self._data.get_value_maps(identifier)
 
+    def get_local_changes(self):
+        return self._data.get_local_changes()
+
     def commit(self):
         self._conn.send_POST('/ConfigManager/set_config', self._data.get_local_changes())
         # todo: check result

src/lib/config/python/isc/config/datadefinition.py

@@ -50,20 +50,23 @@ class DataDefinition:
             _check(data_spec)
         self._data_spec = data_spec
 
-    def validate(self, data, errors = None):
+    def validate(self, full, data, errors = None):
         """Check whether the given piece of data conforms to this
            data definition. If so, it returns True. If not, it will
            return false. If errors is given, and is an array, a string
            describing the error will be appended to it. The current
            version stops as soon as there is one error so this list
-           will not be exhaustive."""
+           will not be exhaustive. If 'full' is true, it also errors on
+           non-optional missing values. Set this to False if you want to
+           validate only a part of a configuration tree (like a list of
+           non-default values)"""
         data_def = self.get_definition()
         if 'config_data' not in data_def:
             if errors:
                 errors.append("The is no config_data for this specification")
             return False
         errors = []
-        return _validate_spec_list(data_def['config_data'], data, errors)
+        return _validate_spec_list(data_def['config_data'], full, data, errors)
 
 
     def get_module_name(self):
@@ -89,7 +92,7 @@ class DataDefinition:
             return self._data_spec['config_data']
         else:
             return None
-    
+
     def __str__(self):
         return self._data_spec.__str__()
 
@@ -246,21 +249,21 @@ def _validate_item(spec, data, errors):
             return False
     return True
 
-def _validate_spec(spec, data, errors):
+def _validate_spec(spec, full, data, errors):
     item_name = spec['item_name']
     item_optional = spec['item_optional']
 
     if item_name in data:
         return _validate_item(spec, data[item_name], errors)
-    elif not item_optional:
+    elif full and not item_optional:
         if errors:
             errors.append("non-optional item " + item_name + " missing")
         return False
     else:
         return True
 
-def _validate_spec_list(data_spec, data, errors):
+def _validate_spec_list(data_spec, full, data, errors):
     for spec_item in data_spec:
-        if not _validate_spec(spec_item, data, errors):
+        if not _validate_spec(spec_item, full, data, errors):
             return False
     return True