[2375] Add a test a token is actually produced

This is a debug test. It checks the state machine produced a token.
There are no tests for this test, since such condition would be a
programmer error, not something expected.

src/lib/dns/master_lexer.cc

@@ -154,6 +154,9 @@ MasterLexer::getSourceLine() const {
 
 MasterLexer::Token
 MasterLexer::getNextToken(Options options) {
+    // Reset the token now. This is to check a token was actually produced.
+    // This is debugging aid.
+    impl_->token_ = Token(Token::NO_TOKEN_PRODUCED);
     if (impl_->source_ == NULL) {
         isc_throw(isc::InvalidOperation, "No source to read tokens from");
     }
@@ -161,7 +164,10 @@ MasterLexer::getNextToken(Options options) {
          state = state->handle(*this)) {
         // Do nothing here. All is handled in the for cycle header itself.
     }
-    // TODO load the token
+    // Make sure a token was produced. Since this Can Not Happen, we assert
+    // here instead of throwing.
+    assert(impl_->token_.getType() != Token::ERROR ||
+           impl_->token_.getErrorCode() != Token::NO_TOKEN_PRODUCED);
     return (impl_->token_);
 }
 
@@ -180,7 +186,8 @@ const char* const error_text[] = {
     "lexer not started",        // NOT_STARTED
     "unbalanced parentheses",   // UNBALANCED_PAREN
     "unexpected end of input",  // UNEXPECTED_END
-    "unbalanced quotes"         // UNBALANCED_QUOTES
+    "unbalanced quotes",        // UNBALANCED_QUOTES
+    "no token produced"
 };
 const size_t error_text_max_count = sizeof(error_text) / sizeof(error_text[0]);
 }

src/lib/dns/master_lexer.h

@@ -277,8 +277,10 @@ public:
         NOT_STARTED, ///< The lexer is just initialized and has no token
         UNBALANCED_PAREN,       ///< Unbalanced parentheses detected
         UNEXPECTED_END, ///< The lexer reaches the end of line or file
-                       /// unexpectedly
+                        /// unexpectedly
         UNBALANCED_QUOTES,      ///< Unbalanced quotations detected
+        NO_TOKEN_PRODUCED, ///< No token was produced. This means programmer
+                           /// error and should never get out of the lexer.
         MAX_ERROR_CODE ///< Max integer corresponding to valid error codes.
                        /// (excluding this one). Mainly for internal use.
     };

src/lib/dns/tests/master_lexer_token_unittest.cc

@@ -142,15 +142,18 @@ TEST_F(MasterLexerTokenTest, errors) {
     EXPECT_EQ("unbalanced quotes",
               MasterLexer::Token(MasterLexer::Token::UNBALANCED_QUOTES).
               getErrorText());
+    EXPECT_EQ("no token produced",
+              MasterLexer::Token(MasterLexer::Token::NO_TOKEN_PRODUCED).
+              getErrorText());
 
     // getErrorCode/Text() isn't allowed for non number types
     EXPECT_THROW(token_num.getErrorCode(), isc::InvalidOperation);
     EXPECT_THROW(token_num.getErrorText(), isc::InvalidOperation);
 
-    // Only the pre-defined error code is accepted.  Hardcoding '4' (max code
+    // Only the pre-defined error code is accepted.  Hardcoding '5' (max code
     // + 1) is intentional; it'd be actually better if we notice it when we
     // update the enum list (which shouldn't happen too often).
-    EXPECT_THROW(MasterLexer::Token(MasterLexer::Token::ErrorCode(4)),
+    EXPECT_THROW(MasterLexer::Token(MasterLexer::Token::ErrorCode(5)),
                  isc::InvalidParameter);
 
     // Check the coexistence of "from number" and "from error-code"