[2372] overall documentation updates

src/lib/dns/master_lexer.cc

@@ -34,10 +34,14 @@ typedef boost::shared_ptr<master_lexer_internal::InputSource> InputSourcePtr;
 using namespace master_lexer_internal;
 
 struct MasterLexer::MasterLexerImpl {
-    MasterLexerImpl() : source_(NULL), paren_count_(0), last_was_eol_(false),
-                        token_(Token::NOT_STARTED)
+    MasterLexerImpl() : source_(NULL), token_(Token::NOT_STARTED),
+                        paren_count_(0), last_was_eol_(false)
     {}
 
+    // A helper method to skip possible comments toward the end of EOL or EOF.
+    // commonly used by state classes.  It returns the corresponding "end-of"
+    // character in case it's a comment; otherwise it simply returns the
+    // current character.
     int skipComment(int c) {
         if (c == ';') {
             while (true) {
@@ -51,10 +55,13 @@ struct MasterLexer::MasterLexerImpl {
     }
 
     std::vector<InputSourcePtr> sources_;
-    InputSource* source_;       // current source
-    size_t paren_count_;
-    bool last_was_eol_;
-    Token token_;
+    InputSource* source_;       // current source (NULL if sources_ is empty)
+    Token token_;               // currently recognized token (set by a state)
+
+    // These are used in states, and defined here only as a placeholder.
+    // The main lexer class does not need these members.
+    size_t paren_count_;        // nest count of the parentheses
+    bool last_was_eol_; // whether the lexer just passed an end-of-line
 };
 
 MasterLexer::MasterLexer() : impl_(new MasterLexerImpl) {
@@ -139,6 +146,10 @@ MasterLexer::Token::getErrorText() const {
 }
 
 namespace master_lexer_internal {
+// Below we implement state classes for state transitions of MasterLexer.
+// Note that these need to be defined here so that they can refer to
+// the details of of MasterLexerImpl.
+
 typedef MasterLexer::Token Token; // convenience shortcut
 
 bool
@@ -184,13 +195,17 @@ public:
 class String : public State {
 public:
     String() {}
-    virtual const State* handle(MasterLexer& /*lexer*/) const
-    {
+    virtual const State* handle(MasterLexer& /*lexer*/) const {
         return (NULL);
     }
 };
 
 namespace {
+// We use a common instance of a each state in a singleton-like way to save
+// construction overhead.  They are not singletons in its strict sense as
+// we don't prohibit direct construction of these objects.  But that doesn't
+// matter much anyway, because the definitions are completely hidden within
+// this file.
 const CRLF CRLF_STATE;
 const String STRING_STATE;
 }

src/lib/dns/master_lexer_state.h

@@ -21,29 +21,108 @@ namespace isc {
 namespace dns {
 
 namespace master_lexer_internal {
-class InputSource;
 
+/// \brief Tokenization state for \c MasterLexer.
+///
+/// This is a base class of classes that represent various states of a single
+/// tokenization session of \c MasterLexer, i.e., the states used for a
+/// single call to \c MasterLexer::getNextToken().
+///
+/// It follows the convention of the state design pattern: each derived class
+/// corresponds to a specific state, and the state transition takes place
+/// through the virtual method named \c handle().  The \c handle() method
+/// takes the main \c MasterLexer object that holds all necessary internal
+/// context, and updates it as necessary; each \c State derived class is
+/// completely stateless.
+///
+/// The initial transition takes place a static method of the base class,
+/// \c start().  This is mainly for implementation convenience; we need to
+/// pass options given to \c MasterLexer::getNextToken() for the initial
+/// state, so it makes more sense to separate the interface for the transition
+/// from the initial state.
+///
+/// When an object of a specific state class completes the session, it
+/// normally sets the identified token in the lexer, and returns NULL;
+/// if more transition is necessary, it returns a pointer to the next state
+/// object.
+///
+/// As is usual in the state design pattern, the \c State class is made
+/// a friend class of \c MasterLexer and can refer to its internal details.
+/// This is intentional; essentially its a part of \c MasterLexer and
+/// is defined as a separate class only for implementation clarity and better
+/// testability.  It's defined in a publicly visible header, but that's only
+/// for testing purposes.  No normal application or even no other classes of
+/// this library are expected to use this class.
 class State {
 public:
+    /// \brief Begin state transitions to get the next token.
+    ///
+    /// This is the first method that \c MasterLexer needs to call for a
+    /// tokenization session.  The lexer passes a reference to itself
+    /// and options given in \c getNextToken().
+    ///
+    /// \throw InputSource::ReadError Unexpected I/O error
+    /// \throw std::bad_alloc Internal resource allocation failure
+    ///
+    /// \param lexer The lexer object that holds the main context.
+    /// \param options The options passed to getNextToken().
+    /// \return A pointer to the next state object or NULL if the transition
+    /// is completed.
     static const State* start(MasterLexer& lexer,
                               MasterLexer::Options options);
+
+    /// \brief Handle the process of one specific state.
+    ///
+    /// This method is expected to be called on the object returned by
+    /// start(), and keep called on the returned object until NULL is
+    /// returned.  The call chain will form the complete state transition.
+    ///
+    /// \throw InputSource::ReadError Unexpected I/O error
+    /// \throw std::bad_alloc Internal resource allocation failure
+    ///
+    /// \param lexer The lexer object that holds the main context.
+    /// \return A pointer to the next state object or NULL if the transition
+    /// is completed.
     virtual const State* handle(MasterLexer& lexer) const = 0;
 
+    /// \brief Types of states.
+    ///
     /// Specific states are basically hidden within the implementation,
     /// but we'd like to allow tests to examine them, so we provide
     /// a way to get an instance of a specific state.
     enum ID {
-        CRLF,                  ///< TBD
-        String
+        CRLF,                  ///< Just seen a carriage-return character
+        String                 ///< Handling a string token
     };
+
+    /// \brief Returns a \c State instance of the given state.
+    ///
+    /// This is provided only for testing purposes so tests can check
+    /// the behavior of each state separately.  \c MasterLexer shouldn't
+    /// need this method.
     static const State& getInstance(ID state_id);
 
     /// \name Read-only accessors for testing purposes.
+    ///
+    /// These allow tests to inspect some selected portion of the internal
+    /// states of \c MasterLexer.  These shouldn't be used except for testing
+    /// purposes.
+    ///@{
     bool wasLastEOL(const MasterLexer& lexer) const;
     const MasterLexer::Token& getToken(const MasterLexer& lexer) const;
     size_t getParenCount(const MasterLexer& lexer) const;
+    ///@}
 
 protected:
+    /// \brief An accessor to the internal implementation class of
+    /// \c MasterLexer.
+    ///
+    /// This is provided for specific derived classes as they are not direct
+    /// friends of \c MasterLexer.
+    ///
+    /// \param lexer The lexer object that holds the main context.
+    /// \return A pointer to the implementation class object of the given
+    /// lexer.  This is never NULL.
     MasterLexer::MasterLexerImpl* getLexerImpl(MasterLexer& lexer) const {
         return (lexer.impl_);
     }