[2375] Handle EOF after unbalanced parentheses

If there's an error that manifests at the end of a file, like an
unbalanced parenthesis or a quoted string that does not end, report the
error. But also, return EOF as the next token, not an exception.

src/lib/dns/master_lexer.cc

@@ -167,7 +167,7 @@ MasterLexer::getSourceLine() const {
 MasterLexer::Token
 MasterLexer::getNextToken(Options options) {
     // If the source is not available
-    if (impl_->source_ == NULL || impl_->source_->atEOF()) {
+    if (impl_->source_ == NULL) {
         isc_throw(isc::InvalidOperation, "No source to read tokens from");
     }
     // Store the current state so we can restore it in ungetToken
@@ -179,6 +179,8 @@ MasterLexer::getNextToken(Options options) {
     // This is debugging aid.
     impl_->token_ = Token(Token::NO_TOKEN_PRODUCED);
     // And get the token
+
+    // This actually handles EOF internally too.
     for (const State *state = start(options); state != NULL;
          state = state->handle(*this)) {
         // Do nothing here. All is handled in the for cycle header itself.

src/lib/dns/tests/master_lexer_unittest.cc

@@ -266,16 +266,16 @@ TEST_F(MasterLexerTest, eof) {
 
     // The first one is found to be EOF
     EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
-    // And it is not allowed to use this one any more.
-    EXPECT_THROW(lexer.getNextToken(), isc::InvalidOperation);
-    // But if we unget a step back, we should get the EOF again
+    // And it stays on EOF for any following attempts
+    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    // And we can step back one token, but that is the EOF too.
     lexer.ungetToken();
     EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
 }
 
 // Check we properly return error when there's an opened parentheses and no
 // closing one
-TEST_F(MasterLexerTest, getUnbalanced) {
+TEST_F(MasterLexerTest, getUnbalancedParen) {
     ss << "(\"string\"";
     lexer.pushSource(ss);
 
@@ -284,6 +284,21 @@ TEST_F(MasterLexerTest, getUnbalanced) {
     // Then an unbalanced parenthesis
     EXPECT_EQ(MasterLexer::Token::UNBALANCED_PAREN,
               lexer.getNextToken().getErrorCode());
+    // And then EOF
+    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+}
+
+// Check we properly return error when there's an opened parentheses and no
+// closing one
+TEST_F(MasterLexerTest, getUnbalancedString) {
+    ss << "\"string";
+    lexer.pushSource(ss);
+
+    // Then an unbalanced parenthesis
+    EXPECT_EQ(MasterLexer::Token::UNEXPECTED_END,
+              lexer.getNextToken(MasterLexer::QSTRING).getErrorCode());
+    // And then EOF
+    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
 }
 
 void