[trac384] make the cmdparse remove whitespace

In order for it to distinguish between separate arguments, it should now
remove whitespace in lists and maps (i.e. between [] and {}), unless
data is quoted

src/bin/bindctl/cmdparse.py

@@ -33,6 +33,7 @@ param_value_str  = "(?P<param_value>[^\'\" ][^, ]+)"
 param_value_with_quota_str  = "[\"\'](?P<param_value>.+?)(?<!\\\)[\"\']"
 next_params_str = "(?P<blank>\s*)(?P<comma>,?)(?P<next_params>.*)$"
 
+
 PARAM_WITH_QUOTA_PATTERN = re.compile(param_name_str + 
                                       param_value_with_quota_str + 
                                       next_params_str)
@@ -40,6 +41,56 @@ PARAM_PATTERN = re.compile(param_name_str + param_value_str + next_params_str)
 # Used for module and command name
 NAME_PATTERN = re.compile("^\s*(?P<name>[\w]+)(?P<blank>\s*)(?P<others>.*)$")
 
+# this removes all whitespace inthe given string, except when
+# between " quotes
+_remove_unquoted_whitespace = \
+    lambda text:'"'.join( it if i%2 else ''.join(it.split())
+        for i,it in enumerate(text.split('"'))  )
+
+
+def _remove_list_and_map_whitespace(text):
+    """Returns a string where the whitespace between matching [ and ]
+       is removed, unless quoted"""
+    # regular expression aren't really the right tool, since we may have
+    # nested structures
+    result = []
+    start_pos = 0
+    pos = 0
+    list_count = 0
+    map_count = 0
+    cur_start_list_pos = None
+    cur_start_map_pos = None
+    for i in text:
+        if i == '[' and map_count == 0:
+            if list_count == 0:
+                result.append(text[start_pos:pos + 1])
+                cur_start_list_pos = pos + 1
+            list_count = list_count + 1
+        elif i == ']' and map_count == 0:
+            if list_count > 0:
+                list_count = list_count - 1
+                if list_count == 0:
+                    result.append(_remove_unquoted_whitespace(text[cur_start_list_pos:pos + 1]))
+                    start_pos = pos + 1
+        if i == '{' and list_count == 0:
+            if map_count == 0:
+                result.append(text[start_pos:pos + 1])
+                cur_start_map_pos = pos + 1
+            map_count = map_count + 1
+        elif i == '}' and list_count == 0:
+            if map_count > 0:
+                map_count = map_count - 1
+                if map_count == 0:
+                    result.append(_remove_unquoted_whitespace(text[cur_start_map_pos:pos + 1]))
+                    start_pos = pos + 1
+        
+
+        pos = pos + 1
+    if start_pos <= len(text):
+        result.append(text[start_pos:len(text)])
+    return "".join(result)
+    
+    
 class BindCmdParse:
     """ This class will parse the command line usr input into three part
     module name, command, parameters
@@ -86,9 +137,12 @@ class BindCmdParse:
 
             self._parse_params(param_str)
 
+    def _remove_list_whitespace(self, text):
+        return ""
 
     def _parse_params(self, param_text):
         """convert a=b,c=d into one hash """
+        param_text = _remove_list_and_map_whitespace(param_text)
         
         # Check parameter name "help"
         param = NAME_PATTERN.match(param_text)

src/bin/bindctl/tests/Makefile.am

@@ -1,5 +1,5 @@
 PYCOVERAGE_RUN = @PYCOVERAGE_RUN@
-PYTESTS = bindctl_test.py
+PYTESTS = bindctl_test.py cmdparse_test.py
 EXTRA_DIST = $(PYTESTS)
 
 # test using command-line arguments, so use check-local target instead of TESTS

src/bin/bindctl/tests/bindctl_test.py

@@ -299,16 +299,31 @@ class TestConfigCommands(unittest.TestCase):
         cmd = cmdparse.BindCmdParse("config set identifier=\"foo/an_int\" value=\"[]\"")
         self.assertRaises(isc.cc.data.DataTypeError, self.tool.apply_config_cmd, cmd)
 
+    # this is a very specific one for use with a set of list tests
+    # to try out the flexibility of the parser (only in the next test)
+    def clt(self, full_cmd_string, item_value):
+        cmd = cmdparse.BindCmdParse(full_cmd_string)
+        self.tool.apply_config_cmd(cmd)
+        self.assertEqual(([item_value], MultiConfigData.LOCAL),
+                         self.tool.config_data.get_value("/foo/a_list"))
+
     def test_apply_cfg_command_list(self):
         self.tool.location = '/'
 
         self.assertEqual(([], MultiConfigData.DEFAULT),
                          self.tool.config_data.get_value("/foo/a_list"))
 
-        cmd = cmdparse.BindCmdParse("config set identifier=\"foo/a_list\" value=[\"a\"]")
-        self.tool.apply_config_cmd(cmd)
-        self.assertEqual((["a"], MultiConfigData.LOCAL),
-                         self.tool.config_data.get_value("/foo/a_list"))
+        self.clt("config set identifier=\"foo/a_list\" value=[\"a\"]", "a")
+        self.clt("config set identifier=\"foo/a_list\" value =[\"b\"]", "b")
+        self.clt("config set identifier=\"foo/a_list\" value= [\"c\"]", "c")
+        self.clt("config set identifier=\"foo/a_list\" value = [\"d\"]", "d")
+        self.clt("config set identifier =\"foo/a_list\" value=[\"e\"]", "e")
+        self.clt("config set identifier= \"foo/a_list\" value=[\"f\"]", "f")
+        self.clt("config set identifier = \"foo/a_list\" value=[\"g\"]", "g")
+        self.clt("config set identifier = \"foo/a_list\" value = [\"h\"]", "h")
+        self.clt("config set identifier = \"foo/a_list\" value=[\"i\" ]", "i")
+        self.clt("config set identifier = \"foo/a_list\" value=[ \"j\"]", "j")
+        self.clt("config set identifier = \"foo/a_list\" value=[ \"k\" ]", "k")
 
         # this should raise a TypeError
         cmd = cmdparse.BindCmdParse("config set identifier=\"foo/a_list\" value=\"a\"")
@@ -317,6 +332,7 @@ class TestConfigCommands(unittest.TestCase):
         cmd = cmdparse.BindCmdParse("config set identifier=\"foo/a_list\" value=[1]")
         self.assertRaises(isc.cc.data.DataTypeError, self.tool.apply_config_cmd, cmd)
 
+
     
 
 class FakeBindCmdInterpreter(bindcmd.BindCmdInterpreter):

src/bin/bindctl/tests/cmdparse_test.py

@@ -0,0 +1,88 @@
+# Copyright (C) 2009  Internet Systems Consortium.
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
+# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
+# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+
+import unittest
+from bindctl import cmdparse
+
+class TestCmdParse(unittest.TestCase):
+
+    def test_remove_unquoted_whitespace(self):
+        self.assertEqual(cmdparse._remove_unquoted_whitespace("a"), "a")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace(" a"), "a")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace("a "), "a")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace(" a "), "a")
+        self.assertNotEqual(cmdparse._remove_unquoted_whitespace("a"), "a ")
+        self.assertNotEqual(cmdparse._remove_unquoted_whitespace(" a"), " a")
+        self.assertNotEqual(cmdparse._remove_unquoted_whitespace("a "), "a ")
+        self.assertNotEqual(cmdparse._remove_unquoted_whitespace(" a "), " a ")
+        self.assertNotEqual(cmdparse._remove_unquoted_whitespace(" a "), "b")
+
+        self.assertEqual(cmdparse._remove_unquoted_whitespace("\"abc\""), "\"abc\"")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace(" \"abc\""), "\"abc\"")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace("\"abc\" "), "\"abc\"")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace(" \"abc\" "), "\"abc\"")
+        
+        self.assertEqual(cmdparse._remove_unquoted_whitespace("\" abc\""), "\" abc\"")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace(" \"a bc\""), "\"a bc\"")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace("\"ab c\" "), "\"ab c\"")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace(" \"abc \" "), "\"abc \"")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace(" \" a b c \" "), "\" a b c \"")
+        
+        self.assertEqual(cmdparse._remove_unquoted_whitespace("a\" abc\"a"), "a\" abc\"a")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace("a \"a bc\"a"), "a\"a bc\"a")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace("a\"ab c\" a"), "a\"ab c\"a")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace("a \"abc \" a"), "a\"abc \"a")
+        self.assertEqual(cmdparse._remove_unquoted_whitespace("a \" a b c \" a"), "a\" a b c \"a")
+
+    # short-hand function to make the set of tests more readable
+    def rws(self, a, b):
+        self.assertEqual(cmdparse._remove_list_and_map_whitespace(a), b)
+
+    def test_remove_list_whitespace(self):
+        self.rws("a", "a")
+        self.rws(" a ", " a ")
+        self.rws(" [a] ", " [a] ")
+        self.rws(" [ a] ", " [a] ")
+        self.rws(" [ a ] ", " [a] ")
+        self.rws(" [ a b c ] ", " [abc] ")
+        self.rws(" [ a \"b c\" ] ", " [a\"b c\"] ")
+        self.rws("a [ a \"b c\" ] a", "a [a\"b c\"] a")
+        self.rws("a] [ a \"b c\" ] a", "a] [a\"b c\"] a")
+        self.rws(" [ a [b c] ] ", " [a[bc]] ")
+        self.rws(" [ a b][ c d ] ", " [ab][cd] ")
+        self.rws(" [ a b] [ c d ] ", " [ab] [cd] ")
+        
+        self.rws("a", "a")
+        self.rws(" a ", " a ")
+        self.rws(" {a} ", " {a} ")
+        self.rws(" { a} ", " {a} ")
+        self.rws(" { a } ", " {a} ")
+        self.rws(" { a b c } ", " {abc} ")
+        self.rws(" { a \"b c\" } ", " {a\"b c\"} ")
+        self.rws("a { a \"b c\" } a", "a {a\"b c\"} a")
+        self.rws("a} { a \"b c\" } a", "a} {a\"b c\"} a")
+        self.rws(" { a {b c} } ", " {a{bc}} ")
+        self.rws(" { a b}{ c d } ", " {ab}{cd} ")
+        self.rws(" { a b} { c d } ", " {ab} {cd} ")
+
+        self.rws(" [ a b]{ c d } ", " [ab]{cd} ")
+        self.rws(" [ a b{ c d }] ", " [ab{cd}] ")
+        self.rws(" [ a b{ \"c d\" }] ", " [ab{\"c d\"}] ")
+        
+
+if __name__== "__main__":
+    unittest.main()
+