Parcourir la source

[2506] rename: MasterLexer::Token to MasterToken.

it's a kind of hack for the convenience of the planned change of this branch,
but the revised name doesn't seem really bad anyway.
JINMEI Tatuya il y a 12 ans
Parent
commit
c6c7cdee40

+ 23 - 26
src/lib/dns/master_lexer.cc

@@ -36,7 +36,7 @@ using namespace master_lexer_internal;
 
 
 struct MasterLexer::MasterLexerImpl {
-    MasterLexerImpl() : source_(NULL), token_(Token::NOT_STARTED),
+    MasterLexerImpl() : source_(NULL), token_(MasterToken::NOT_STARTED),
                         paren_count_(0), last_was_eol_(false),
                         has_previous_(false),
                         previous_paren_count_(0),
@@ -82,7 +82,7 @@ struct MasterLexer::MasterLexerImpl {
 
     std::vector<InputSourcePtr> sources_;
     InputSource* source_;       // current source (NULL if sources_ is empty)
-    Token token_;               // currently recognized token (set by a state)
+    MasterToken token_;         // currently recognized token (set by a state)
     std::vector<char> data_;    // placeholder for string data
 
     // These are used in states, and defined here only as a placeholder.
@@ -165,7 +165,7 @@ MasterLexer::getSourceLine() const {
     return (impl_->sources_.back()->getCurrentLine());
 }
 
-const MasterLexer::Token&
+const MasterToken&
 MasterLexer::getNextToken(Options options) {
     if (impl_->source_ == NULL) {
         isc_throw(isc::InvalidOperation, "No source to read tokens from");
@@ -177,7 +177,7 @@ MasterLexer::getNextToken(Options options) {
     impl_->has_previous_ = true;
     // Reset the token now. This is to check a token was actually produced.
     // This is debugging aid.
-    impl_->token_ = Token(Token::NO_TOKEN_PRODUCED);
+    impl_->token_ = MasterToken(MasterToken::NO_TOKEN_PRODUCED);
     // And get the token
 
     // This actually handles EOF internally too.
@@ -187,8 +187,8 @@ MasterLexer::getNextToken(Options options) {
     }
     // Make sure a token was produced. Since this Can Not Happen, we assert
     // here instead of throwing.
-    assert(impl_->token_.getType() != Token::ERROR ||
-           impl_->token_.getErrorCode() != Token::NO_TOKEN_PRODUCED);
+    assert(impl_->token_.getType() != MasterToken::ERROR ||
+           impl_->token_.getErrorCode() != MasterToken::NO_TOKEN_PRODUCED);
     return (impl_->token_);
 }
 
@@ -217,10 +217,10 @@ const size_t error_text_max_count = sizeof(error_text) / sizeof(error_text[0]);
 } // end unnamed namespace
 
 std::string
-MasterLexer::Token::getErrorText() const {
+MasterToken::getErrorText() const {
     if (type_ != ERROR) {
         isc_throw(InvalidOperation,
-                  "Token::getErrorText() for non error type");
+                  "MasterToken::getErrorText() for non error type");
     }
 
     // The class integrity ensures the following:
@@ -233,14 +233,12 @@ namespace master_lexer_internal {
 // Note that these need to be defined here so that they can refer to
 // the details of MasterLexerImpl.
 
-typedef MasterLexer::Token Token; // convenience shortcut
-
 bool
 State::wasLastEOL(const MasterLexer& lexer) const {
     return (lexer.impl_->last_was_eol_);
 }
 
-const MasterLexer::Token&
+const MasterToken&
 State::getToken(const MasterLexer& lexer) const {
     return (lexer.impl_->token_);
 }
@@ -270,7 +268,7 @@ public:
         if (c != '\n') {
             getLexerImpl(lexer)->source_->ungetChar();
         }
-        getLexerImpl(lexer)->token_ = Token(Token::END_OF_LINE);
+        getLexerImpl(lexer)->token_ = MasterToken(MasterToken::END_OF_LINE);
         getLexerImpl(lexer)->last_was_eol_ = true;
     }
 };
@@ -341,24 +339,24 @@ State::start(MasterLexer& lexer, MasterLexer::Options options) {
         if (c == InputSource::END_OF_STREAM) {
             lexerimpl.last_was_eol_ = false;
             if (paren_count != 0) {
-                lexerimpl.token_ = Token(Token::UNBALANCED_PAREN);
+                lexerimpl.token_ = MasterToken(MasterToken::UNBALANCED_PAREN);
                 paren_count = 0; // reset to 0; this helps in lenient mode.
                 return (NULL);
             }
-            lexerimpl.token_ = Token(Token::END_OF_FILE);
+            lexerimpl.token_ = MasterToken(MasterToken::END_OF_FILE);
             return (NULL);
         } else if (c == ' ' || c == '\t') {
             // If requested and we are not in (), recognize the initial space.
             if (lexerimpl.last_was_eol_ && paren_count == 0 &&
                 (options & MasterLexer::INITIAL_WS) != 0) {
                 lexerimpl.last_was_eol_ = false;
-                lexerimpl.token_ = Token(Token::INITIAL_WS);
+                lexerimpl.token_ = MasterToken(MasterToken::INITIAL_WS);
                 return (NULL);
             }
         } else if (c == '\n') {
             lexerimpl.last_was_eol_ = true;
             if (paren_count == 0) { // we don't recognize EOL if we are in ()
-                lexerimpl.token_ = Token(Token::END_OF_LINE);
+                lexerimpl.token_ = MasterToken(MasterToken::END_OF_LINE);
                 return (NULL);
             }
         } else if (c == '\r') {
@@ -374,7 +372,7 @@ State::start(MasterLexer& lexer, MasterLexer::Options options) {
         } else if (c == ')') {
             lexerimpl.last_was_eol_ = false;
             if (paren_count == 0) {
-                lexerimpl.token_ = Token(Token::UNBALANCED_PAREN);
+                lexerimpl.token_ = MasterToken(MasterToken::UNBALANCED_PAREN);
                 return (NULL);
             }
             --paren_count;
@@ -406,7 +404,7 @@ String::handle(MasterLexer& lexer) const {
         if (getLexerImpl(lexer)->isTokenEnd(c, escaped)) {
             getLexerImpl(lexer)->source_->ungetChar();
             getLexerImpl(lexer)->token_ =
-                MasterLexer::Token(&data.at(0), data.size());
+                MasterToken(&data.at(0), data.size());
             return;
         }
         escaped = (c == '\\' && !escaped);
@@ -416,7 +414,7 @@ String::handle(MasterLexer& lexer) const {
 
 void
 QString::handle(MasterLexer& lexer) const {
-    MasterLexer::Token& token = getLexerImpl(lexer)->token_;
+    MasterToken& token = getLexerImpl(lexer)->token_;
     std::vector<char>& data = getLexerImpl(lexer)->data_;
     data.clear();
 
@@ -424,7 +422,7 @@ QString::handle(MasterLexer& lexer) const {
     while (true) {
         const int c = getLexerImpl(lexer)->source_->getChar();
         if (c == InputSource::END_OF_STREAM) {
-            token = Token(Token::UNEXPECTED_END);
+            token = MasterToken(MasterToken::UNEXPECTED_END);
             return;
         } else if (c == '"') {
             if (escaped) {
@@ -433,12 +431,12 @@ QString::handle(MasterLexer& lexer) const {
                 escaped = false;
                 data.back() = '"';
             } else {
-                token = MasterLexer::Token(&data.at(0), data.size(), true);
+                token = MasterToken(&data.at(0), data.size(), true);
                 return;
             }
         } else if (c == '\n' && !escaped) {
             getLexerImpl(lexer)->source_->ungetChar();
-            token = Token(Token::UNBALANCED_QUOTES);
+            token = MasterToken(MasterToken::UNBALANCED_QUOTES);
             return;
         } else {
             escaped = (c == '\\' && !escaped);
@@ -449,7 +447,7 @@ QString::handle(MasterLexer& lexer) const {
 
 void
 Number::handle(MasterLexer& lexer) const {
-    MasterLexer::Token& token = getLexerImpl(lexer)->token_;
+    MasterToken& token = getLexerImpl(lexer)->token_;
 
     // It may yet turn out to be a string, so we first
     // collect all the data
@@ -473,11 +471,10 @@ Number::handle(MasterLexer& lexer) const {
                 } catch (const boost::bad_lexical_cast&) {
                     // Since we already know we have only digits,
                     // range should be the only possible problem.
-                    token = Token(Token::NUMBER_OUT_OF_RANGE);
+                    token = Token(MasterToken::NUMBER_OUT_OF_RANGE);
                 }
             } else {
-                token = MasterLexer::Token(&data.at(0),
-                                           data.size());
+                token = MasterToken(&data.at(0), data.size());
             }
             return;
         }

+ 222 - 224
src/lib/dns/master_lexer.h

@@ -28,225 +28,6 @@ namespace master_lexer_internal {
 class State;
 }
 
-/// \brief Tokenizer for parsing DNS master files.
-///
-/// The \c MasterLexer class provides tokenize interfaces for parsing DNS
-/// master files.  It understands some special rules of master files as
-/// defined in RFC 1035, such as comments, character escaping, and multi-line
-/// data, and provides the user application with the actual data in a
-/// more convenient form such as a std::string object.
-///
-/// In order to support the $INCLUDE notation, this class is designed to be
-/// able to operate on multiple files or input streams in the nested way.
-/// The \c pushSource() and \c popSource() methods correspond to the push
-/// and pop operations.
-///
-/// While this class is public, it is less likely to be used by normal
-/// applications; it's mainly expected to be used within this library,
-/// specifically by the \c MasterLoader class and \c Rdata implementation
-/// classes.
-///
-/// \note The error handling policy of this class is slightly different from
-/// that of other classes of this library.  We generally throw an exception
-/// for an invalid input, whether it's more likely to be a program error or
-/// a "user error", which means an invalid input that comes from outside of
-/// the library.  But, this class returns an error code for some certain
-/// types of user errors instead of throwing an exception.  Such cases include
-/// a syntax error identified by the lexer or a misspelled file name that
-/// causes a system error at the time of open.  This is based on the assumption
-/// that the main user of this class is a parser of master files, where
-/// we want to give an option to ignore some non fatal errors and continue
-/// the parsing.  This will be useful if it just performs overall error
-/// checks on a master file.  When the (immediate) caller needs to do explicit
-/// error handling, exceptions are not that a useful tool for error reporting
-/// because we cannot separate the normal and error cases anyway, which would
-/// be one major advantage when we use exceptions.  And, exceptions are
-/// generally more expensive, either when it happens or just by being able
-/// to handle with \c try and \c catch (depending on the underlying
-/// implementation of the exception handling).  For these reasons, some of
-/// this class does not throw for an error that would be reported as an
-/// exception in other classes.
-class MasterLexer {
-    friend class master_lexer_internal::State;
-public:
-    /// \brief Exception thrown when we fail to read from the input
-    /// stream or file.
-    struct ReadError : public Unexpected {
-        ReadError(const char* file, size_t line, const char* what) :
-            Unexpected(file, line, what)
-        {}
-    };
-
-    class Token;       // we define it separately for better readability
-
-    /// \brief Options for getNextToken.
-    ///
-    /// A compound option, indicating multiple options are set, can be
-    /// specified using the logical OR operator (operator|()).
-    enum Options {
-        NONE = 0,               ///< No option
-        INITIAL_WS = 1, ///< recognize begin-of-line spaces after an
-                        ///< end-of-line
-        QSTRING = 2,    ///< recognize quoted string
-        NUMBER = 4   ///< recognize numeric text as integer
-    };
-
-    /// \brief The constructor.
-    ///
-    /// \throw std::bad_alloc Internal resource allocation fails (rare case).
-    MasterLexer();
-
-    /// \brief The destructor.
-    ///
-    /// It internally closes any remaining input sources.
-    ~MasterLexer();
-
-    /// \brief Open a file and make it the current input source of MasterLexer.
-    ///
-    /// The opened file can be explicitly closed by the \c popSource() method;
-    /// if \c popSource() is not called within the lifetime of the
-    /// \c MasterLexer, it will be closed in the destructor.
-    ///
-    /// In the case possible system errors in opening the file (most likely
-    /// because of specifying a non-existent or unreadable file), it returns
-    /// false, and if the optional \c error parameter is non NULL, it will be
-    /// set to a description of the error (any existing content of the string
-    /// will be discarded).  If opening the file succeeds, the given
-    /// \c error parameter will be intact.
-    ///
-    /// Note that this method has two styles of error reporting: one by
-    /// returning \c false (and setting \c error optionally) and the other
-    /// by throwing an exception.  See the note for the class description
-    /// about the distinction.
-    ///
-    /// \throw InvalidParameter filename is NULL
-    /// \param filename A non NULL string specifying a master file
-    /// \param error If non null, a placeholder to set error description in
-    /// case of failure.
-    ///
-    /// \return true if pushing the file succeeds; false otherwise.
-    bool pushSource(const char* filename, std::string* error = NULL);
-
-    /// \brief Make the given stream the current input source of MasterLexer.
-    ///
-    /// The caller still holds the ownership of the passed stream; it's the
-    /// caller's responsibility to keep it valid as long as it's used in
-    /// \c MasterLexer or to release any resource for the stream after that.
-    /// The caller can explicitly tell \c MasterLexer to stop using the
-    /// stream by calling the \c popSource() method.
-    ///
-    /// \param input An input stream object that produces textual
-    /// representation of DNS RRs.
-    void pushSource(std::istream& input);
-
-    /// \brief Stop using the most recently opened input source (file or
-    /// stream).
-    ///
-    /// If it's a file, the previously opened file will be closed internally.
-    /// If it's a stream, \c MasterLexer will simply stop using
-    /// the stream; the caller can assume it will be never used in
-    /// \c MasterLexer thereafter.
-    ///
-    /// This method must not be called when there is no source pushed for
-    /// \c MasterLexer.  This method is otherwise exception free.
-    ///
-    /// \throw isc::InvalidOperation Called with no pushed source.
-    void popSource();
-
-    /// \brief Return the name of the current input source name.
-    ///
-    /// If it's a file, it will be the C string given at the corresponding
-    /// \c pushSource() call, that is, its filename.  If it's a stream, it will
-    /// be formatted as \c "stream-%p" where \c %p is hex representation
-    /// of the address of the stream object.
-    ///
-    /// If there is no opened source at the time of the call, this method
-    /// returns an empty string.
-    ///
-    /// \throw std::bad_alloc Resource allocation failed for string
-    /// construction (rare case)
-    ///
-    /// \return A string representation of the current source (see the
-    /// description)
-    std::string getSourceName() const;
-
-    /// \brief Return the input source line number.
-    ///
-    /// If there is an opened source, the return value will be a non-0
-    /// integer indicating the line number of the current source where
-    /// the \c MasterLexer is currently working.  The expected usage of
-    /// this value is to print a helpful error message when parsing fails
-    /// by specifically identifying the position of the error.
-    ///
-    /// If there is no opened source at the time of the call, this method
-    /// returns 0.
-    ///
-    /// \throw None
-    ///
-    /// \return The current line number of the source (see the description)
-    size_t getSourceLine() const;
-
-    /// \brief Parse and return another token from the input.
-    ///
-    /// It reads a bit of the last opened source and produces another token
-    /// found in it.
-    ///
-    /// This method does not provide the strong exception guarantee. Generally,
-    /// if it throws, the object should not be used any more and should be
-    /// discarded. It was decided all the exceptions thrown from here are
-    /// serious enough that aborting the loading process is the only reasonable
-    /// recovery anyway, so the strong exception guarantee is not needed.
-    ///
-    /// \param options The options can be used to modify the tokenization.
-    ///     The method can be made reporting things which are usually ignored
-    ///     by this parameter. Multiple options can be passed at once by
-    ///     bitwise or (eg. option1 | option 2). See description of available
-    ///     options.
-    /// \return Next token found in the input. Note that the token refers to
-    ///     some internal data in the lexer. It is valid only until
-    ///     getNextToken or ungetToken is called. Also, the token becomes
-    ///     invalid when the lexer is destroyed.
-    /// \throw isc::InvalidOperation in case the source is not available. This
-    ///     may mean the pushSource() has not been called yet, or that the
-    ///     current source has been read past the end.
-    /// \throw ReadError in case there's problem reading from the underlying
-    ///     source (eg. I/O error in the file on the disk).
-    /// \throw std::bad_alloc in case allocation of some internal resources
-    ///     or the token fail.
-    const Token& getNextToken(Options options = NONE);
-
-    /// \brief Return the last token back to the lexer.
-    ///
-    /// The method undoes the lasts call to getNextToken(). If you call the
-    /// getNextToken() again with the same options, it'll return the same
-    /// token. If the options are different, it may return a different token,
-    /// but it acts as if the previous getNextToken() was never called.
-    ///
-    /// It is possible to return only one token back in time (you can't call
-    /// ungetToken() twice in a row without calling getNextToken() in between
-    /// successfully).
-    ///
-    /// It does not work after change of source (by pushSource or popSource).
-    ///
-    /// \throw isc::InvalidOperation If called second time in a row or if
-    ///     getNextToken() was not called since the last change of the source.
-    void ungetToken();
-
-private:
-    struct MasterLexerImpl;
-    MasterLexerImpl* impl_;
-};
-
-/// \brief Operator to combine \c MasterLexer options
-///
-/// This is a trivial shortcut so that compound options can be specified
-/// in an intuitive way.
-inline MasterLexer::Options
-operator|(MasterLexer::Options o1, MasterLexer::Options o2) {
-    return (static_cast<MasterLexer::Options>(
-                static_cast<unsigned>(o1) | static_cast<unsigned>(o2)));
-}
-
 /// \brief Tokens for \c MasterLexer
 ///
 /// This is a simple value-class encapsulating a type of a lexer token and
@@ -261,7 +42,7 @@ operator|(MasterLexer::Options o1, MasterLexer::Options o2) {
 /// (using the default version of copy constructor and assignment operator),
 /// but it's mainly for internal implementation convenience.  Applications will
 /// simply refer to Token object as a reference via the \c MasterLexer class.
-class MasterLexer::Token {
+class MasterToken {
 public:
     /// \brief Enumeration for token types
     ///
@@ -318,7 +99,7 @@ public:
     /// \throw InvalidParameter A value type token is specified.
     /// \param type The type of the token.  It must indicate a non-value
     /// type (not larger than \c NOVALUE_TYPE_MAX).
-    explicit Token(Type type) : type_(type) {
+    explicit MasterToken(Type type) : type_(type) {
         if (type > NOVALUE_TYPE_MAX) {
             isc_throw(InvalidParameter, "Token per-type constructor "
                       "called with invalid type: " << type);
@@ -340,7 +121,7 @@ public:
     /// \param str_beg The start address of the string
     /// \param str_len The size of the string in bytes
     /// \param quoted true if it's a quoted string; false otherwise.
-    Token(const char* str_beg, size_t str_len, bool quoted = false) :
+    MasterToken(const char* str_beg, size_t str_len, bool quoted = false) :
         type_(quoted ? QSTRING : STRING)
     {
         val_.str_region_.beg = str_beg;
@@ -351,7 +132,7 @@ public:
     ///
     /// \brief number An unsigned 32-bit integer corresponding to the token
     /// value.
-    explicit Token(uint32_t number) : type_(NUMBER) {
+    explicit MasterToken(uint32_t number) : type_(NUMBER) {
         val_.number_ = number;
     }
 
@@ -359,7 +140,7 @@ public:
     ///
     /// \throw InvalidParameter Invalid error code value is specified.
     /// \brief error_code A pre-defined constant of \c ErrorCode.
-    explicit Token(ErrorCode error_code) : type_(ERROR) {
+    explicit MasterToken(ErrorCode error_code) : type_(ERROR) {
         if (!(error_code < MAX_ERROR_CODE)) {
             isc_throw(InvalidParameter, "Invalid master lexer error code: "
                       << error_code);
@@ -476,6 +257,223 @@ private:
     } val_;
 };
 
+/// \brief Tokenizer for parsing DNS master files.
+///
+/// The \c MasterLexer class provides tokenize interfaces for parsing DNS
+/// master files.  It understands some special rules of master files as
+/// defined in RFC 1035, such as comments, character escaping, and multi-line
+/// data, and provides the user application with the actual data in a
+/// more convenient form such as a std::string object.
+///
+/// In order to support the $INCLUDE notation, this class is designed to be
+/// able to operate on multiple files or input streams in the nested way.
+/// The \c pushSource() and \c popSource() methods correspond to the push
+/// and pop operations.
+///
+/// While this class is public, it is less likely to be used by normal
+/// applications; it's mainly expected to be used within this library,
+/// specifically by the \c MasterLoader class and \c Rdata implementation
+/// classes.
+///
+/// \note The error handling policy of this class is slightly different from
+/// that of other classes of this library.  We generally throw an exception
+/// for an invalid input, whether it's more likely to be a program error or
+/// a "user error", which means an invalid input that comes from outside of
+/// the library.  But, this class returns an error code for some certain
+/// types of user errors instead of throwing an exception.  Such cases include
+/// a syntax error identified by the lexer or a misspelled file name that
+/// causes a system error at the time of open.  This is based on the assumption
+/// that the main user of this class is a parser of master files, where
+/// we want to give an option to ignore some non fatal errors and continue
+/// the parsing.  This will be useful if it just performs overall error
+/// checks on a master file.  When the (immediate) caller needs to do explicit
+/// error handling, exceptions are not that a useful tool for error reporting
+/// because we cannot separate the normal and error cases anyway, which would
+/// be one major advantage when we use exceptions.  And, exceptions are
+/// generally more expensive, either when it happens or just by being able
+/// to handle with \c try and \c catch (depending on the underlying
+/// implementation of the exception handling).  For these reasons, some of
+/// this class does not throw for an error that would be reported as an
+/// exception in other classes.
+class MasterLexer {
+    friend class master_lexer_internal::State;
+public:
+    /// \brief Exception thrown when we fail to read from the input
+    /// stream or file.
+    struct ReadError : public Unexpected {
+        ReadError(const char* file, size_t line, const char* what) :
+            Unexpected(file, line, what)
+        {}
+    };
+
+    /// \brief Options for getNextToken.
+    ///
+    /// A compound option, indicating multiple options are set, can be
+    /// specified using the logical OR operator (operator|()).
+    enum Options {
+        NONE = 0,               ///< No option
+        INITIAL_WS = 1, ///< recognize begin-of-line spaces after an
+                        ///< end-of-line
+        QSTRING = 2,    ///< recognize quoted string
+        NUMBER = 4   ///< recognize numeric text as integer
+    };
+
+    /// \brief The constructor.
+    ///
+    /// \throw std::bad_alloc Internal resource allocation fails (rare case).
+    MasterLexer();
+
+    /// \brief The destructor.
+    ///
+    /// It internally closes any remaining input sources.
+    ~MasterLexer();
+
+    /// \brief Open a file and make it the current input source of MasterLexer.
+    ///
+    /// The opened file can be explicitly closed by the \c popSource() method;
+    /// if \c popSource() is not called within the lifetime of the
+    /// \c MasterLexer, it will be closed in the destructor.
+    ///
+    /// In the case possible system errors in opening the file (most likely
+    /// because of specifying a non-existent or unreadable file), it returns
+    /// false, and if the optional \c error parameter is non NULL, it will be
+    /// set to a description of the error (any existing content of the string
+    /// will be discarded).  If opening the file succeeds, the given
+    /// \c error parameter will be intact.
+    ///
+    /// Note that this method has two styles of error reporting: one by
+    /// returning \c false (and setting \c error optionally) and the other
+    /// by throwing an exception.  See the note for the class description
+    /// about the distinction.
+    ///
+    /// \throw InvalidParameter filename is NULL
+    /// \param filename A non NULL string specifying a master file
+    /// \param error If non null, a placeholder to set error description in
+    /// case of failure.
+    ///
+    /// \return true if pushing the file succeeds; false otherwise.
+    bool pushSource(const char* filename, std::string* error = NULL);
+
+    /// \brief Make the given stream the current input source of MasterLexer.
+    ///
+    /// The caller still holds the ownership of the passed stream; it's the
+    /// caller's responsibility to keep it valid as long as it's used in
+    /// \c MasterLexer or to release any resource for the stream after that.
+    /// The caller can explicitly tell \c MasterLexer to stop using the
+    /// stream by calling the \c popSource() method.
+    ///
+    /// \param input An input stream object that produces textual
+    /// representation of DNS RRs.
+    void pushSource(std::istream& input);
+
+    /// \brief Stop using the most recently opened input source (file or
+    /// stream).
+    ///
+    /// If it's a file, the previously opened file will be closed internally.
+    /// If it's a stream, \c MasterLexer will simply stop using
+    /// the stream; the caller can assume it will be never used in
+    /// \c MasterLexer thereafter.
+    ///
+    /// This method must not be called when there is no source pushed for
+    /// \c MasterLexer.  This method is otherwise exception free.
+    ///
+    /// \throw isc::InvalidOperation Called with no pushed source.
+    void popSource();
+
+    /// \brief Return the name of the current input source name.
+    ///
+    /// If it's a file, it will be the C string given at the corresponding
+    /// \c pushSource() call, that is, its filename.  If it's a stream, it will
+    /// be formatted as \c "stream-%p" where \c %p is hex representation
+    /// of the address of the stream object.
+    ///
+    /// If there is no opened source at the time of the call, this method
+    /// returns an empty string.
+    ///
+    /// \throw std::bad_alloc Resource allocation failed for string
+    /// construction (rare case)
+    ///
+    /// \return A string representation of the current source (see the
+    /// description)
+    std::string getSourceName() const;
+
+    /// \brief Return the input source line number.
+    ///
+    /// If there is an opened source, the return value will be a non-0
+    /// integer indicating the line number of the current source where
+    /// the \c MasterLexer is currently working.  The expected usage of
+    /// this value is to print a helpful error message when parsing fails
+    /// by specifically identifying the position of the error.
+    ///
+    /// If there is no opened source at the time of the call, this method
+    /// returns 0.
+    ///
+    /// \throw None
+    ///
+    /// \return The current line number of the source (see the description)
+    size_t getSourceLine() const;
+
+    /// \brief Parse and return another token from the input.
+    ///
+    /// It reads a bit of the last opened source and produces another token
+    /// found in it.
+    ///
+    /// This method does not provide the strong exception guarantee. Generally,
+    /// if it throws, the object should not be used any more and should be
+    /// discarded. It was decided all the exceptions thrown from here are
+    /// serious enough that aborting the loading process is the only reasonable
+    /// recovery anyway, so the strong exception guarantee is not needed.
+    ///
+    /// \param options The options can be used to modify the tokenization.
+    ///     The method can be made reporting things which are usually ignored
+    ///     by this parameter. Multiple options can be passed at once by
+    ///     bitwise or (eg. option1 | option 2). See description of available
+    ///     options.
+    /// \return Next token found in the input. Note that the token refers to
+    ///     some internal data in the lexer. It is valid only until
+    ///     getNextToken or ungetToken is called. Also, the token becomes
+    ///     invalid when the lexer is destroyed.
+    /// \throw isc::InvalidOperation in case the source is not available. This
+    ///     may mean the pushSource() has not been called yet, or that the
+    ///     current source has been read past the end.
+    /// \throw ReadError in case there's problem reading from the underlying
+    ///     source (eg. I/O error in the file on the disk).
+    /// \throw std::bad_alloc in case allocation of some internal resources
+    ///     or the token fail.
+    const MasterToken& getNextToken(Options options = NONE);
+
+    /// \brief Return the last token back to the lexer.
+    ///
+    /// The method undoes the lasts call to getNextToken(). If you call the
+    /// getNextToken() again with the same options, it'll return the same
+    /// token. If the options are different, it may return a different token,
+    /// but it acts as if the previous getNextToken() was never called.
+    ///
+    /// It is possible to return only one token back in time (you can't call
+    /// ungetToken() twice in a row without calling getNextToken() in between
+    /// successfully).
+    ///
+    /// It does not work after change of source (by pushSource or popSource).
+    ///
+    /// \throw isc::InvalidOperation If called second time in a row or if
+    ///     getNextToken() was not called since the last change of the source.
+    void ungetToken();
+
+private:
+    struct MasterLexerImpl;
+    MasterLexerImpl* impl_;
+};
+
+/// \brief Operator to combine \c MasterLexer options
+///
+/// This is a trivial shortcut so that compound options can be specified
+/// in an intuitive way.
+inline MasterLexer::Options
+operator|(MasterLexer::Options o1, MasterLexer::Options o2) {
+    return (static_cast<MasterLexer::Options>(
+                static_cast<unsigned>(o1) | static_cast<unsigned>(o2)));
+}
+
 } // namespace dns
 } // namespace isc
 #endif  // MASTER_LEXER_H

+ 1 - 1
src/lib/dns/master_lexer_state.h

@@ -119,7 +119,7 @@ public:
     /// purposes.
     ///@{
     bool wasLastEOL(const MasterLexer& lexer) const;
-    const MasterLexer::Token& getToken(const MasterLexer& lexer) const;
+    const MasterToken& getToken(const MasterLexer& lexer) const;
     size_t getParenCount(const MasterLexer& lexer) const;
     ///@}
 

+ 2 - 2
src/lib/dns/tests/master_lexer_state_unittest.cc

@@ -24,7 +24,7 @@ using namespace isc::dns;
 using namespace master_lexer_internal;
 
 namespace {
-typedef MasterLexer::Token Token; // shortcut
+typedef MasterToken Token; // shortcut
 
 class MasterLexerStateTest : public ::testing::Test {
 protected:
@@ -260,7 +260,7 @@ TEST_F(MasterLexerStateTest, crlf) {
 // Commonly used check for string related test cases, checking if the given
 // token has expected values.
 void
-stringTokenCheck(const std::string& expected, const MasterLexer::Token& token,
+stringTokenCheck(const std::string& expected, const MasterToken& token,
                  bool quoted = false)
 {
     EXPECT_EQ(quoted ? Token::QSTRING : Token::STRING, token.getType());

+ 39 - 47
src/lib/dns/tests/master_lexer_token_unittest.cc

@@ -31,27 +31,27 @@ const size_t TEST_STRING_LEN = sizeof(TEST_STRING) - 1;
 class MasterLexerTokenTest : public ::testing::Test {
 protected:
     MasterLexerTokenTest() :
-        token_eof(MasterLexer::Token::END_OF_FILE),
+        token_eof(MasterToken::END_OF_FILE),
         token_str(TEST_STRING, TEST_STRING_LEN),
         token_num(42),
-        token_err(MasterLexer::Token::UNEXPECTED_END)
+        token_err(MasterToken::UNEXPECTED_END)
     {}
 
-    const MasterLexer::Token token_eof; // an example of non-value type token
-    const MasterLexer::Token token_str;
-    const MasterLexer::Token token_num;
-    const MasterLexer::Token token_err;
+    const MasterToken token_eof; // an example of non-value type token
+    const MasterToken token_str;
+    const MasterToken token_num;
+    const MasterToken token_err;
 };
 
 
 TEST_F(MasterLexerTokenTest, strings) {
     // basic construction and getter checks
-    EXPECT_EQ(MasterLexer::Token::STRING, token_str.getType());
+    EXPECT_EQ(MasterToken::STRING, token_str.getType());
     EXPECT_EQ(std::string("string token"), token_str.getString());
     std::string strval = "dummy"; // this should be replaced
     token_str.getString(strval);
     EXPECT_EQ(std::string("string token"), strval);
-    const MasterLexer::Token::StringRegion str_region =
+    const MasterToken::StringRegion str_region =
         token_str.getStringRegion();
     EXPECT_EQ(TEST_STRING, str_region.beg);
     EXPECT_EQ(TEST_STRING_LEN, str_region.len);
@@ -62,17 +62,17 @@ TEST_F(MasterLexerTokenTest, strings) {
     std::string expected_str("string token");
     expected_str.push_back('\0');
     EXPECT_EQ(expected_str,
-              MasterLexer::Token(TEST_STRING, TEST_STRING_LEN + 1).getString());
-    MasterLexer::Token(TEST_STRING, TEST_STRING_LEN + 1).getString(strval);
+              MasterToken(TEST_STRING, TEST_STRING_LEN + 1).getString());
+    MasterToken(TEST_STRING, TEST_STRING_LEN + 1).getString(strval);
     EXPECT_EQ(expected_str, strval);
 
     // Construct type of qstring
-    EXPECT_EQ(MasterLexer::Token::QSTRING,
-              MasterLexer::Token(TEST_STRING, sizeof(TEST_STRING), true).
+    EXPECT_EQ(MasterToken::QSTRING,
+              MasterToken(TEST_STRING, sizeof(TEST_STRING), true).
               getType());
     // if we explicitly set 'quoted' to false, it should be normal string
-    EXPECT_EQ(MasterLexer::Token::STRING,
-              MasterLexer::Token(TEST_STRING, sizeof(TEST_STRING), false).
+    EXPECT_EQ(MasterToken::STRING,
+              MasterToken(TEST_STRING, sizeof(TEST_STRING), false).
               getType());
 
     // getString/StringRegion() aren't allowed for non string(-variant) types
@@ -86,23 +86,23 @@ TEST_F(MasterLexerTokenTest, strings) {
 
 TEST_F(MasterLexerTokenTest, numbers) {
     EXPECT_EQ(42, token_num.getNumber());
-    EXPECT_EQ(MasterLexer::Token::NUMBER, token_num.getType());
+    EXPECT_EQ(MasterToken::NUMBER, token_num.getType());
 
     // It's copyable and assignable.
-    MasterLexer::Token token(token_num);
+    MasterToken token(token_num);
     EXPECT_EQ(42, token.getNumber());
-    EXPECT_EQ(MasterLexer::Token::NUMBER, token.getType());
+    EXPECT_EQ(MasterToken::NUMBER, token.getType());
 
     token = token_num;
     EXPECT_EQ(42, token.getNumber());
-    EXPECT_EQ(MasterLexer::Token::NUMBER, token.getType());
+    EXPECT_EQ(MasterToken::NUMBER, token.getType());
 
     // it's okay to replace it with a different type of token
     token = token_eof;
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, token.getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, token.getType());
 
     // Possible max value
-    token = MasterLexer::Token(0xffffffff);
+    token = MasterToken(0xffffffff);
     EXPECT_EQ(4294967295u, token.getNumber());
 
     // getNumber() isn't allowed for non number types
@@ -112,41 +112,34 @@ TEST_F(MasterLexerTokenTest, numbers) {
 
 TEST_F(MasterLexerTokenTest, novalues) {
     // Just checking we can construct them and getType() returns correct value.
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, token_eof.getType());
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE,
-              MasterLexer::Token(MasterLexer::Token::END_OF_LINE).getType());
-    EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
-              MasterLexer::Token(MasterLexer::Token::INITIAL_WS).getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, token_eof.getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE,
+              MasterToken(MasterToken::END_OF_LINE).getType());
+    EXPECT_EQ(MasterToken::INITIAL_WS,
+              MasterToken(MasterToken::INITIAL_WS).getType());
 
     // Special types of tokens cannot have value-based types
-    EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::STRING),
-                 isc::InvalidParameter);
-    EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::QSTRING),
-                 isc::InvalidParameter);
-    EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::NUMBER),
-                 isc::InvalidParameter);
-    EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::ERROR),
-                 isc::InvalidParameter);
+    EXPECT_THROW(MasterToken t(MasterToken::STRING), isc::InvalidParameter);
+    EXPECT_THROW(MasterToken t(MasterToken::QSTRING), isc::InvalidParameter);
+    EXPECT_THROW(MasterToken t(MasterToken::NUMBER), isc::InvalidParameter);
+    EXPECT_THROW(MasterToken t(MasterToken::ERROR), isc::InvalidParameter);
 }
 
 TEST_F(MasterLexerTokenTest, errors) {
-    EXPECT_EQ(MasterLexer::Token::ERROR, token_err.getType());
-    EXPECT_EQ(MasterLexer::Token::UNEXPECTED_END, token_err.getErrorCode());
+    EXPECT_EQ(MasterToken::ERROR, token_err.getType());
+    EXPECT_EQ(MasterToken::UNEXPECTED_END, token_err.getErrorCode());
     EXPECT_EQ("unexpected end of input", token_err.getErrorText());
-    EXPECT_EQ("lexer not started",
-              MasterLexer::Token(MasterLexer::Token::NOT_STARTED).
+    EXPECT_EQ("lexer not started", MasterToken(MasterToken::NOT_STARTED).
               getErrorText());
     EXPECT_EQ("unbalanced parentheses",
-              MasterLexer::Token(MasterLexer::Token::UNBALANCED_PAREN).
+              MasterToken(MasterToken::UNBALANCED_PAREN).
               getErrorText());
-    EXPECT_EQ("unbalanced quotes",
-              MasterLexer::Token(MasterLexer::Token::UNBALANCED_QUOTES).
+    EXPECT_EQ("unbalanced quotes", MasterToken(MasterToken::UNBALANCED_QUOTES).
               getErrorText());
-    EXPECT_EQ("no token produced",
-              MasterLexer::Token(MasterLexer::Token::NO_TOKEN_PRODUCED).
+    EXPECT_EQ("no token produced", MasterToken(MasterToken::NO_TOKEN_PRODUCED).
               getErrorText());
     EXPECT_EQ("number out of range",
-              MasterLexer::Token(MasterLexer::Token::NUMBER_OUT_OF_RANGE).
+              MasterLexer::Token(MasterToken::NUMBER_OUT_OF_RANGE).
               getErrorText());
 
     // getErrorCode/Text() isn't allowed for non number types
@@ -156,14 +149,13 @@ TEST_F(MasterLexerTokenTest, errors) {
     // Only the pre-defined error code is accepted.  Hardcoding '6' (max code
     // + 1) is intentional; it'd be actually better if we notice it when we
     // update the enum list (which shouldn't happen too often).
-    EXPECT_THROW(MasterLexer::Token(MasterLexer::Token::ErrorCode(6)),
+    EXPECT_THROW(MasterToken(MasterToken::ErrorCode(6)),
                  isc::InvalidParameter);
 
     // Check the coexistence of "from number" and "from error-code"
     // constructors won't cause confusion.
-    EXPECT_EQ(MasterLexer::Token::NUMBER,
-              MasterLexer::Token(static_cast<uint32_t>(
-                                     MasterLexer::Token::NOT_STARTED)).
+    EXPECT_EQ(MasterToken::NUMBER,
+              MasterToken(static_cast<uint32_t>(MasterToken::NOT_STARTED)).
               getType());
 }
 }

+ 27 - 27
src/lib/dns/tests/master_lexer_unittest.cc

@@ -141,19 +141,19 @@ TEST_F(MasterLexerTest, getNextToken) {
     lexer.pushSource(ss);
 
     // First, the newline should get out.
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // Then the whitespace, if we specify the option.
-    EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
+    EXPECT_EQ(MasterToken::INITIAL_WS,
               lexer.getNextToken(MasterLexer::INITIAL_WS).getType());
     // The newline
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // The (quoted) string
-    EXPECT_EQ(MasterLexer::Token::QSTRING,
+    EXPECT_EQ(MasterToken::QSTRING,
               lexer.getNextToken(MasterLexer::QSTRING).getType());
 
     // And the end of line and file
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
 }
 
 // Test we correctly find end of file.
@@ -162,12 +162,12 @@ TEST_F(MasterLexerTest, eof) {
     lexer.pushSource(ss);
 
     // The first one is found to be EOF
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
     // And it stays on EOF for any following attempts
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
     // And we can step back one token, but that is the EOF too.
     lexer.ungetToken();
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
 }
 
 // Check we properly return error when there's an opened parentheses and no
@@ -177,12 +177,12 @@ TEST_F(MasterLexerTest, getUnbalancedParen) {
     lexer.pushSource(ss);
 
     // The string gets out first
-    EXPECT_EQ(MasterLexer::Token::STRING, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::STRING, lexer.getNextToken().getType());
     // Then an unbalanced parenthesis
-    EXPECT_EQ(MasterLexer::Token::UNBALANCED_PAREN,
+    EXPECT_EQ(MasterToken::UNBALANCED_PAREN,
               lexer.getNextToken().getErrorCode());
     // And then EOF
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
 }
 
 // Check we properly return error when there's an opened quoted string and no
@@ -192,10 +192,10 @@ TEST_F(MasterLexerTest, getUnbalancedString) {
     lexer.pushSource(ss);
 
     // Then an unbalanced qstring (reported as an unexpected end)
-    EXPECT_EQ(MasterLexer::Token::UNEXPECTED_END,
+    EXPECT_EQ(MasterToken::UNEXPECTED_END,
               lexer.getNextToken(MasterLexer::QSTRING).getErrorCode());
     // And then EOF
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
 }
 
 // Test ungetting tokens works
@@ -204,28 +204,28 @@ TEST_F(MasterLexerTest, ungetToken) {
     lexer.pushSource(ss);
 
     // Try getting the newline
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // Return it and get again
     lexer.ungetToken();
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // Get the string and return it back
-    EXPECT_EQ(MasterLexer::Token::QSTRING,
+    EXPECT_EQ(MasterToken::QSTRING,
               lexer.getNextToken(MasterLexer::QSTRING).getType());
     lexer.ungetToken();
     // But if we change the options, it honors them
-    EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
+    EXPECT_EQ(MasterToken::INITIAL_WS,
               lexer.getNextToken(MasterLexer::QSTRING |
                                  MasterLexer::INITIAL_WS).getType());
     // Get to the "more" string
-    EXPECT_EQ(MasterLexer::Token::QSTRING,
+    EXPECT_EQ(MasterToken::QSTRING,
               lexer.getNextToken(MasterLexer::QSTRING).getType());
-    EXPECT_EQ(MasterLexer::Token::STRING,
+    EXPECT_EQ(MasterToken::STRING,
               lexer.getNextToken(MasterLexer::QSTRING).getType());
     // Return it back. It should get inside the parentheses.
     // Upon next attempt to get it again, the newline inside the parentheses
     // should be still ignored.
     lexer.ungetToken();
-    EXPECT_EQ(MasterLexer::Token::STRING,
+    EXPECT_EQ(MasterToken::STRING,
               lexer.getNextToken(MasterLexer::QSTRING).getType());
 }
 
@@ -235,16 +235,16 @@ TEST_F(MasterLexerTest, ungetRealOptions) {
     ss << "\n    \n";
     lexer.pushSource(ss);
     // Skip the first newline
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
 
     // If we call it the usual way, it skips up to the newline and returns
     // it
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
 
     // Now we return it. If we call it again, but with different options,
     // we get the initial whitespace.
     lexer.ungetToken();
-    EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
+    EXPECT_EQ(MasterToken::INITIAL_WS,
               lexer.getNextToken(MasterLexer::INITIAL_WS).getType());
 }
 
@@ -253,7 +253,7 @@ TEST_F(MasterLexerTest, ungetTwice) {
     ss << "\n";
     lexer.pushSource(ss);
 
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // Unget the token. It can be done once
     lexer.ungetToken();
     // But not twice
@@ -271,14 +271,14 @@ TEST_F(MasterLexerTest, ungetBeforeGet) {
 TEST_F(MasterLexerTest, ungetAfterSwitch) {
     ss << "\n\n";
     lexer.pushSource(ss);
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // Switch the source
     std::stringstream ss2;
     ss2 << "\n\n";
     lexer.pushSource(ss2);
     EXPECT_THROW(lexer.ungetToken(), isc::InvalidOperation);
     // We can get from the new source
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // And when we drop the current source, we can't unget again
     lexer.popSource();
     EXPECT_THROW(lexer.ungetToken(), isc::InvalidOperation);