[master] Merge branch 'trac2374'

src/lib/dns/master_lexer.cc

@@ -212,7 +212,7 @@ const char* const error_text[] = {
     "unexpected end of input",  // UNEXPECTED_END
     "unbalanced quotes",        // UNBALANCED_QUOTES
     "no token produced",        // NO_TOKEN_PRODUCED
-    "number out of range"       // NUMBER_RANGE
+    "number out of range"       // NUMBER_OUT_OF_RANGE
 };
 const size_t error_text_max_count = sizeof(error_text) / sizeof(error_text[0]);
 } // end unnamed namespace
@@ -379,7 +379,7 @@ State::start(MasterLexer& lexer, MasterLexer::Options options) {
                 return (NULL);
             }
             --paren_count;
-        } else if (isdigit(c)) {
+        } else if ((options & MasterLexer::NUMBER) != 0 &&isdigit(c)) {
             lexerimpl.last_was_eol_ = false;
             // this character will be handled in the number state
             lexerimpl.source_->ungetChar();
@@ -469,12 +469,12 @@ Number::handle(MasterLexer& lexer) const {
                 data.push_back('\0');
                 try {
                     const uint32_t number32 =
-                        boost::lexical_cast<uint32_t, const char*>(&data.at(0));
+                        boost::lexical_cast<uint32_t, const char*>(&data[0]);
                     token = MasterLexer::Token(number32);
                 } catch (const boost::bad_lexical_cast&) {
                     // Since we already know we have only digits,
                     // range should be the only possible problem.
-                    token = Token(Token::NUMBER_RANGE);
+                    token = Token(Token::NUMBER_OUT_OF_RANGE);
                 }
             } else {
                 token = MasterLexer::Token(&data.at(0),

src/lib/dns/master_lexer.h

@@ -292,7 +292,7 @@ public:
         UNBALANCED_QUOTES,      ///< Unbalanced quotations detected
         NO_TOKEN_PRODUCED, ///< No token was produced. This means programmer
                            /// error and should never get out of the lexer.
-        NUMBER_RANGE,    ///< Number was out of range
+        NUMBER_OUT_OF_RANGE, ///< Number was out of range
         MAX_ERROR_CODE ///< Max integer corresponding to valid error codes.
                        /// (excluding this one). Mainly for internal use.
     };

src/lib/dns/tests/master_lexer_state_unittest.cc

@@ -467,45 +467,48 @@ TEST_F(MasterLexerStateTest, basicNumbers) {
                          // the tokenizer
     lexer.pushSource(ss);
 
-    EXPECT_EQ(&s_number, State::start(lexer, common_options));
+    // Ask the lexer to recognize numbers as well
+    const MasterLexer::Options options = common_options | MasterLexer::NUMBER;
+
+    EXPECT_EQ(&s_number, State::start(lexer, options));
     s_number.handle(lexer);
     EXPECT_EQ(0, s_number.getToken(lexer).getNumber());
 
-    EXPECT_EQ(&s_number, State::start(lexer, common_options));
+    EXPECT_EQ(&s_number, State::start(lexer, options));
     s_number.handle(lexer);
     EXPECT_EQ(1, s_number.getToken(lexer).getNumber());
 
-    EXPECT_EQ(&s_number, State::start(lexer, common_options));
+    EXPECT_EQ(&s_number, State::start(lexer, options));
     s_number.handle(lexer);
     EXPECT_EQ(12345, s_number.getToken(lexer).getNumber());
 
-    EXPECT_EQ(&s_number, State::start(lexer, common_options));
+    EXPECT_EQ(&s_number, State::start(lexer, options));
     s_number.handle(lexer);
     EXPECT_EQ(4294967295u, s_number.getToken(lexer).getNumber());
 
-    EXPECT_EQ(&s_number, State::start(lexer, common_options));
+    EXPECT_EQ(&s_number, State::start(lexer, options));
     s_number.handle(lexer);
-    EXPECT_EQ(Token::NUMBER_RANGE,
+    EXPECT_EQ(Token::NUMBER_OUT_OF_RANGE,
               s_number.getToken(lexer).getErrorCode());
 
-    EXPECT_EQ(&s_number, State::start(lexer, common_options));
+    EXPECT_EQ(&s_number, State::start(lexer, options));
     s_number.handle(lexer);
-    EXPECT_EQ(Token::NUMBER_RANGE,
+    EXPECT_EQ(Token::NUMBER_OUT_OF_RANGE,
               s_number.getToken(lexer).getErrorCode());
 
-    EXPECT_EQ(&s_number, State::start(lexer, common_options));
+    EXPECT_EQ(&s_number, State::start(lexer, options));
     s_number.handle(lexer);
     EXPECT_EQ(5, s_number.getToken(lexer).getNumber());
 
-    EXPECT_EQ(&s_number, State::start(lexer, common_options));
+    EXPECT_EQ(&s_number, State::start(lexer, options));
     s_number.handle(lexer);
     EXPECT_EQ(42, s_number.getToken(lexer).getNumber());
 
-    EXPECT_EQ(s_null, State::start(lexer, common_options));
+    EXPECT_EQ(s_null, State::start(lexer, options));
     EXPECT_TRUE(s_crlf.wasLastEOL(lexer));
     EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
 
-    EXPECT_EQ(&s_number, State::start(lexer, common_options));
+    EXPECT_EQ(&s_number, State::start(lexer, options));
     s_number.handle(lexer);
     EXPECT_EQ(37, s_number.getToken(lexer).getNumber());
 
@@ -517,6 +520,8 @@ TEST_F(MasterLexerStateTest, basicNumbers) {
 // Test tokens that look like (or start out as) numbers,
 // but turn out to be strings. Tests include escaped characters.
 TEST_F(MasterLexerStateTest, stringNumbers) {
+    ss << "123 ";        // Should be read as a string if the
+                         // NUMBER option is not given
     ss << "-1 ";         // Negative numbers are interpreted
                          // as strings (unsigned integers only)
     ss << "123abc456 ";  // 'Numbers' containing non-digits should
@@ -532,45 +537,54 @@ TEST_F(MasterLexerStateTest, stringNumbers) {
 
     lexer.pushSource(ss);
 
+    // Note that common_options does not include MasterLexer::NUMBER,
+    // so the token should be recognized as a string
     EXPECT_EQ(&s_string, State::start(lexer, common_options));
     s_string.handle(lexer);
+    stringTokenCheck("123", s_string.getToken(lexer), false);
+
+    // Ask the lexer to recognize numbers as well
+    const MasterLexer::Options options = common_options | MasterLexer::NUMBER;
+
+    EXPECT_EQ(&s_string, State::start(lexer, options));
+    s_string.handle(lexer);
     stringTokenCheck("-1", s_string.getToken(lexer), false);
 
     // Starts out as a number, but ends up being a string
-    EXPECT_EQ(&s_number, State::start(lexer, common_options));
+    EXPECT_EQ(&s_number, State::start(lexer, options));
     s_number.handle(lexer);
     stringTokenCheck("123abc456", s_number.getToken(lexer), false);
 
-    EXPECT_EQ(&s_number, State::start(lexer, common_options));
+    EXPECT_EQ(&s_number, State::start(lexer, options));
     s_number.handle(lexer);
     stringTokenCheck("123\\456", s_number.getToken(lexer), false);
 
-    EXPECT_EQ(&s_number, State::start(lexer, common_options));
+    EXPECT_EQ(&s_number, State::start(lexer, options));
     s_number.handle(lexer); // recognize str, see ' ' at end
     stringTokenCheck("3scaped\\ space", s_number.getToken(lexer));
 
-    EXPECT_EQ(&s_number, State::start(lexer, common_options));
+    EXPECT_EQ(&s_number, State::start(lexer, options));
     s_number.handle(lexer); // recognize str, see ' ' at end
     stringTokenCheck("3scaped\\\ttab", s_number.getToken(lexer));
 
-    EXPECT_EQ(&s_number, State::start(lexer, common_options));
+    EXPECT_EQ(&s_number, State::start(lexer, options));
     s_number.handle(lexer); // recognize str, see ' ' at end
     stringTokenCheck("3scaped\\(paren", s_number.getToken(lexer));
 
-    EXPECT_EQ(&s_number, State::start(lexer, common_options));
+    EXPECT_EQ(&s_number, State::start(lexer, options));
     s_number.handle(lexer); // recognize str, see ' ' at end
     stringTokenCheck("3scaped\\)close", s_number.getToken(lexer));
 
-    EXPECT_EQ(&s_number, State::start(lexer, common_options));
+    EXPECT_EQ(&s_number, State::start(lexer, options));
     s_number.handle(lexer); // recognize str, see ' ' at end
     stringTokenCheck("3scaped\\;comment", s_number.getToken(lexer));
 
-    EXPECT_EQ(&s_number, State::start(lexer, common_options));
+    EXPECT_EQ(&s_number, State::start(lexer, options));
     s_number.handle(lexer); // recognize str, see ' ' in mid
     stringTokenCheck("3scaped\\\\", s_number.getToken(lexer));
 
     // Confirm the word that follows the escaped '\' is correctly recognized.
-    EXPECT_EQ(&s_number, State::start(lexer, common_options));
+    EXPECT_EQ(&s_number, State::start(lexer, options));
     s_number.handle(lexer); // recognize str, see ' ' at end
     stringTokenCheck("8ackslash", s_number.getToken(lexer));
 

src/lib/dns/tests/master_lexer_token_unittest.cc

@@ -146,7 +146,7 @@ TEST_F(MasterLexerTokenTest, errors) {
               MasterLexer::Token(MasterLexer::Token::NO_TOKEN_PRODUCED).
               getErrorText());
     EXPECT_EQ("number out of range",
-              MasterLexer::Token(MasterLexer::Token::NUMBER_RANGE).
+              MasterLexer::Token(MasterLexer::Token::NUMBER_OUT_OF_RANGE).
               getErrorText());
 
     // getErrorCode/Text() isn't allowed for non number types