[5105] Fixed the \\ in tokens and improved doc (guides and examples)

doc/examples/kea4/multiple-options.json

@@ -47,13 +47,21 @@
          },
          {
              // String options that have a comma in their values need to have
-             // it escaped (i.e. each comma is predeced by two backslashes). That's
-             // because commas are reserved for separating fields in compound
-             // options. At the same time, we need to be conformant with JSON spec,
-             // that does not allow "\,". Therefore the slightly uncommon double
-             // backslashes natation is needed.
+             // it escaped (i.e. each comma is predeced by two backslashes).
+             // That's because commas are reserved for separating fields in
+             // compound options. At the same time, we need to be conformant
+             // with JSON spec, that does not allow "\,". Therefore the
+             // slightly uncommon double backslashes notation is needed.
              "name": "boot-file-name",
              "data": "EST5EDT4\\,M3.2.0/02:00\\,M11.1.0/02:00"
+
+             // Legal JSON escapes are \ followed by "\/bfnrt character
+             // or \u followed by 4 hexa-decimal numbers (currently Kea
+             // supports only \u0000 to \u00ff code points).
+             // CSV processing translates '\\' into '\' and '\,' into ','
+             // only so for instance '\x' is translated into '\x'. But
+             // as it works on a JSON string value each of these '\'
+             // characters must be doubled on JSON input.
          }
        ]
     } 

doc/examples/kea6/multiple-options.json

@@ -52,13 +52,21 @@
         },
         {
             // String options that have a comma in their values need to have
-            // it escaped (i.e. each comma is predeced by two backslashes). That's
-            // because commas are reserved for separating fields in compound
-            // options. At the same time, we need to be conformant with JSON spec,
-            // that does not allow "\,". Therefore the slightly uncommon double
-            // backslashes natation is needed.
+            // it escaped (i.e. each comma is predeced by two backslashes).
+            // That's because commas are reserved for separating fields in
+            // compound options. At the same time, we need to be conformant
+            // with JSON spec, that does not allow "\,". Therefore the
+            // slightly uncommon double backslashes notation is needed.
             "name": "new-posix-timezone",
             "data": "EST5EDT4\\,M3.2.0/02:00\\,M11.1.0/02:00"
+
+            // Legal JSON escapes are \ followed by "\/bfnrt character
+            // or \u followed by 4 hexa-decimal numbers (currently Kea
+            // supports only \u0000 to \u00ff code points).
+            // CSV processing translates '\\' into '\' and '\,' into ','
+            // only so for instance '\x' is translated into '\x'. But
+            // as it works on a JSON string value each of these '\'
+            // characters must be doubled on JSON input.
         }
       ],
       "pools": [

doc/guide/dhcp4-srv.xml

@@ -1017,11 +1017,12 @@ temporarily override a list of interface names and listen on all interfaces.
         the various types is given in <xref linkend="dhcp-types"/>.
       </para>
 
-      <para>When the data field is a string, and that string contains the comma
+      <para>When a data field is a string, and that string contains the comma
       (,; U+002C) character, the comma must be escaped with a double reverse solidus
-      character (\; U+005C). This double escape is required, because a
-      single escape (\,) would make the JSON invalid. For example, the string
-      &quot;foo,bar&quot; would be represented as:
+      character (\; U+005C). This double escape is required, because both the
+      routine splitting CSV data into fields and JSON use the same escape
+      character: a single escape (\,) would make the JSON invalid.
+      For example, the string &quot;foo,bar&quot; would be represented as:
       <screen>
 "Dhcp4": {
     "subnet4": [

doc/guide/dhcp6-srv.xml

@@ -1050,11 +1050,12 @@ temporarily override a list of interface names and listen on all interfaces.
       which was not assigned by IANA) are listed in
       <xref linkend="dhcp6-exp-options-list"/>.
     </para>
-    <para>When the data field is a string, and that string contains
+    <para>When a data field is a string, and that string contains
     the comma (,; U+002C) character, the comma must be escaped with a
     reverse solidus character (\; U+005C). This double escape is
-    required, because a single escape (\,) would make the JSON
-    invalid. For example, the string
+    required, because both the routine splitting CSV data into fields
+    and JSON use the same escape character: a single escape (\,) would
+    make the JSON invalid. For example, the string
     &quot;EST5EDT4,M3.2.0/02:00,M11.1.0/02:00&quot; would be
     represented as:
 <screen>

src/lib/util/strutil.cc

@@ -91,9 +91,8 @@ tokens(const std::string& text, const std::string& delim, bool escape) {
                 in_token = true;
             }
             if (escaped) {
-                // Escaped escape: reset escaped and keep both characters
+                // Escaped escape: reset escaped and keep one character
                 escaped = false;
-                token.push_back('\\');
                 token.push_back(*c);
             } else {
                 // Remember to keep the next character
@@ -106,7 +105,7 @@ tokens(const std::string& text, const std::string& delim, bool escape) {
                 in_token = true;
             }
             if (escaped) {
-                // Escaped common character: as there was no escape
+                // Escaped common character: as escape was false
                 escaped = false;
                 token.push_back('\\');
                 token.push_back(*c);

src/lib/util/strutil.h

@@ -69,8 +69,8 @@ std::string trim(const std::string& instring);
 /// invisible leading and trailing delimiter characters.  Therefore both cases
 /// reduce to a set of contiguous delimiters, which are considered a single
 /// delimiter (so getting rid of the string).
-/// Optional escape allows to escape delimiter characters (and *only* them)
-/// using backslash.
+/// Optional escape allows to escape delimiter characters (and *only* them
+/// and the escape character itself) using backslash.
 ///
 /// We could use Boost for this, but this (simple) function eliminates one
 /// dependency in the code.

src/lib/util/tests/strutil_unittest.cc

@@ -152,9 +152,15 @@ TEST(StringUtilTest, Tokens) {
     // Escaped escape
     result = isc::util::str::tokens("foo\\\\,bar", ",", true);
     ASSERT_EQ(2, result.size());
-    EXPECT_EQ(string("foo\\\\"), result[0]);
+    EXPECT_EQ(string("foo\\"), result[0]);
     EXPECT_EQ(string("bar"), result[1]);
 
+    // Double escapes
+    result = isc::util::str::tokens("foo\\\\\\\\,\\bar", ",", true);
+    ASSERT_EQ(2, result.size());
+    EXPECT_EQ(string("foo\\\\"), result[0]);
+    EXPECT_EQ(string("\\bar"), result[1]);
+
     // Escaped standard character
     result = isc::util::str::tokens("fo\\o,bar", ",", true);
     ASSERT_EQ(2, result.size());