[2375] Don't restore internal token on ungetToken

It was not safe, because it may refer to some data elsewhere. Luckily,
it seems it is not needed anyway, since the only way that a token can
get out of the lexer is through the getNextToken, which would get a new
one, and through a testing method not meant for public use.

src/lib/dns/master_lexer.cc

@@ -38,7 +38,6 @@ struct MasterLexer::MasterLexerImpl {
     MasterLexerImpl() : source_(NULL), token_(Token::NOT_STARTED),
                         paren_count_(0), last_was_eol_(false),
                         has_previous_(false),
-                        previous_token_(Token::NOT_STARTED),
                         previous_paren_count_(0),
                         previous_was_eol_(false)
     {
@@ -99,7 +98,6 @@ struct MasterLexer::MasterLexerImpl {
 
     // These are to allow restoring state before previous token.
     bool has_previous_;
-    Token previous_token_;
     size_t previous_paren_count_;
     bool previous_was_eol_;
 };
@@ -176,7 +174,6 @@ MasterLexer::getNextToken(Options options) {
     impl_->has_previous_ = false;
     // If thrown in this section, nothing observable except for not having
     // previous will happen.
-    impl_->previous_token_ = impl_->token_;
     impl_->previous_paren_count_ = impl_->paren_count_;
     impl_->previous_was_eol_ = impl_->last_was_eol_;
     impl_->source_->mark();
@@ -206,9 +203,6 @@ MasterLexer::getNextToken(Options options) {
 void
 MasterLexer::ungetToken() {
     if (impl_->has_previous_) {
-        // The one that could fail due to bad_alloc first
-        impl_->token_ = impl_->previous_token_;
-        // The rest should not throw.
         impl_->has_previous_ = false;
         impl_->source_->ungetAll();
         impl_->last_was_eol_ = impl_->previous_was_eol_;

src/lib/dns/master_lexer.h

@@ -202,7 +202,10 @@ public:
     ///     by this parameter. Multiple options can be passed at once by
     ///     bitwise or (eg. option1 | option 2). See description of available
     ///     options.
-    /// \return Next token found in the input.
+    /// \return Next token found in the input. Note that the token refers to
+    ///     some internal data in in the lexer. It is valid only until
+    ///     getNextToken or ungetToken is called. Also, the token becomes
+    ///     invalid when the lexer is destroyed.
     /// \throw isc::InvalidOperation in case the source is not available. This
     ///     may mean the pushSource() has not been called yet, or that the
     ///     current source has been read past the end.

src/lib/dns/tests/master_lexer_unittest.cc

@@ -303,30 +303,22 @@ TEST_F(MasterLexerTest, ungetSimple) {
     // We access the lexer through any state, so use the one we have.
     EXPECT_EQ(1, state->getParenCount(lexer));
     EXPECT_TRUE(state->wasLastEOL(lexer));
-    EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
-              state->getToken(lexer).getType());
 
     // Now get the token and check the state changed
     EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
     EXPECT_EQ(2, state->getParenCount(lexer));
     EXPECT_FALSE(state->wasLastEOL(lexer));
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE,
-              state->getToken(lexer).getType());
 
     // Return the token back. Check the state is as it was before.
     lexer.ungetToken();
     EXPECT_EQ(1, state->getParenCount(lexer));
     EXPECT_TRUE(state->wasLastEOL(lexer));
-    EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
-              state->getToken(lexer).getType());
     // By calling getToken again, we verify even the source got back to
     // original. We must push it as a fake start again so it is picked.
     lexer.pushFakeStart(state.get());
     EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
     EXPECT_EQ(2, state->getParenCount(lexer));
     EXPECT_FALSE(state->wasLastEOL(lexer));
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE,
-              state->getToken(lexer).getType());
 }
 
 // Check ungetting token without overriding the start method. We also
@@ -413,8 +405,6 @@ TEST_F(MasterLexerTest, getTokenExceptions) {
     EXPECT_THROW(lexer.getNextToken(), TestException);
     EXPECT_EQ(0, s1->getParenCount(lexer));
     EXPECT_FALSE(s1->wasLastEOL(lexer));
-    EXPECT_EQ(MasterLexer::Token::NOT_STARTED,
-              s1->getToken(lexer).getErrorCode());
 
     // It gets back to the original state, so getting the newline works.
     EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());