[master] Merge branch 'trac2371'

src/lib/dns/master_lexer.cc

@@ -12,10 +12,86 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
+#include <exceptions/exceptions.h>
+
 #include <dns/master_lexer.h>
+#include <dns/master_lexer_inputsource.h>
+
+#include <boost/shared_ptr.hpp>
 
 #include <cassert>
 #include <string>
+#include <vector>
+
+namespace isc {
+namespace dns {
+
+namespace {
+typedef boost::shared_ptr<master_lexer_internal::InputSource> InputSourcePtr;
+}
+using namespace master_lexer_internal;
+
+struct MasterLexer::MasterLexerImpl {
+    MasterLexerImpl() : token_(Token::NOT_STARTED) {}
+
+    std::vector<InputSourcePtr> sources_;
+    Token token_;
+};
+
+MasterLexer::MasterLexer() : impl_(new MasterLexerImpl) {
+}
+
+MasterLexer::~MasterLexer() {
+    delete impl_;
+}
+
+bool
+MasterLexer::pushSource(const char* filename, std::string* error) {
+    if (filename == NULL) {
+        isc_throw(InvalidParameter,
+                  "NULL filename for MasterLexer::pushSource");
+    }
+    try {
+        impl_->sources_.push_back(InputSourcePtr(new InputSource(filename)));
+    } catch (const InputSource::OpenError& ex) {
+        if (error != NULL) {
+            *error = ex.what();
+        }
+        return (false);
+    }
+
+    return (true);
+}
+
+void
+MasterLexer::pushSource(std::istream& input) {
+    impl_->sources_.push_back(InputSourcePtr(new InputSource(input)));
+}
+
+void
+MasterLexer::popSource() {
+    if (impl_->sources_.empty()) {
+        isc_throw(InvalidOperation,
+                  "MasterLexer::popSource on an empty source");
+    }
+    impl_->sources_.pop_back();
+}
+
+std::string
+MasterLexer::getSourceName() const {
+    if (impl_->sources_.empty()) {
+        return (std::string());
+    }
+    return (impl_->sources_.back()->getName());
+}
+
+size_t
+MasterLexer::getSourceLine() const {
+    if (impl_->sources_.empty()) {
+        return (0);
+    }
+    return (impl_->sources_.back()->getCurrentLine());
+}
 
 namespace {
 const char* const error_text[] = {
@@ -27,9 +103,6 @@ const char* const error_text[] = {
 const size_t error_text_max_count = sizeof(error_text) / sizeof(error_text[0]);
 }
 
-namespace isc {
-namespace dns {
-
 std::string
 MasterLexer::Token::getErrorText() const {
     if (type_ != ERROR) {
@@ -42,6 +115,5 @@ MasterLexer::Token::getErrorText() const {
     return (error_text[val_.error_code_]);
 }
 
-
 } // end of namespace dns
 } // end of namespace isc

src/lib/dns/master_lexer.h

@@ -17,6 +17,7 @@
 
 #include <exceptions/exceptions.h>
 
+#include <istream>
 #include <string>
 
 #include <stdint.h>
@@ -24,9 +25,146 @@
 namespace isc {
 namespace dns {
 
+/// \brief Tokenizer for parsing DNS master files.
+///
+/// The \c MasterLexer class provides tokenize interfaces for parsing DNS
+/// master files.  It understands some special rules of master files as
+/// defined in RFC 1035, such as comments, character escaping, and multi-line
+/// data, and provides the user application with the actual data in a
+/// more convenient form such as a std::string object.
+///
+/// In order to support the $INCLUDE notation, this class is designed to be
+/// able to operate on multiple files or input streams in the nested way.
+/// The \c pushSource() and \c popSource() methods correspond to the push
+/// and pop operations.
+///
+/// While this class is public, it is less likely to be used by normal
+/// applications; it's mainly expected to be used within this library,
+/// specifically by the \c MasterLoader class and \c Rdata implementation
+/// classes.
+///
+/// \note The error handling policy of this class is slightly different from
+/// that of other classes of this library.  We generally throw an exception
+/// for an invalid input, whether it's more likely to be a program error or
+/// a "user error", which means an invalid input that comes from outside of
+/// the library.  But, this class returns an error code for some certain
+/// types of user errors instead of throwing an exception.  Such cases include
+/// a syntax error identified by the lexer or a misspelled file name that
+/// causes a system error at the time of open.  This is based on the assumption
+/// that the main user of this class is a parser of master files, where
+/// we want to give an option to ignore some non fatal errors and continue
+/// the parsing.  This will be useful if it just performs overall error
+/// checks on a master file.  When the (immediate) caller needs to do explicit
+/// error handling, exceptions are not that a useful tool for error reporting
+/// because we cannot separate the normal and error cases anyway, which would
+/// be one major advantage when we use exceptions.  And, exceptions are
+/// generally more expensive, either when it happens or just by being able
+/// to handle with \c try and \c catch (depending on the underlying
+/// implementation of the exception handling).  For these reasons, some of
+/// this class does not throw for an error that would be reported as an
+/// exception in other classes.
 class MasterLexer {
 public:
     class Token;       // we define it separately for better readability
+
+    /// \brief The constructor.
+    ///
+    /// \throw std::bad_alloc Internal resource allocation fails (rare case).
+    MasterLexer();
+
+    /// \brief The destructor.
+    ///
+    /// It internally closes any remaining input sources.
+    ~MasterLexer();
+
+    /// \brief Open a file and make it the current input source of MasterLexer.
+    ///
+    /// The opened file can be explicitly closed by the \c popSource() method;
+    /// if \c popSource() is not called within the lifetime of the
+    /// \c MasterLexer, it will be closed in the destructor.
+    ///
+    /// In the case possible system errors in opening the file (most likely
+    /// because of specifying a non-existent or unreadable file), it returns
+    /// false, and if the optional \c error parameter is non NULL, it will be
+    /// set to a description of the error (any existing content of the string
+    /// will be discarded).  If opening the file succeeds, the given
+    /// \c error parameter will be intact.
+    ///
+    /// Note that this method has two styles of error reporting: one by
+    /// returning \c false (and setting \c error optionally) and the other
+    /// by throwing an exception.  See the note for the class description
+    /// about the distinction.
+    ///
+    /// \throw InvalidParameter filename is NULL
+    /// \param filename A non NULL string specifying a master file
+    /// \param error If non null, a placeholder to set error description in
+    /// case of failure.
+    ///
+    /// \return true if pushing the file succeeds; false otherwise.
+    bool pushSource(const char* filename, std::string* error = NULL);
+
+    /// \brief Make the given stream the current input source of MasterLexer.
+    ///
+    /// The caller still holds the ownership of the passed stream; it's the
+    /// caller's responsibility to keep it valid as long as it's used in
+    /// \c MasterLexer or to release any resource for the stream after that.
+    /// The caller can explicitly tell \c MasterLexer to stop using the
+    /// stream by calling the \c popSource() method.
+    ///
+    /// \param input An input stream object that produces textual
+    /// representation of DNS RRs.
+    void pushSource(std::istream& input);
+
+    /// \brief Stop using the most recently opened input source (file or
+    /// stream).
+    ///
+    /// If it's a file, the previously opened file will be closed internally.
+    /// If it's a stream, \c MasterLexer will simply stop using
+    /// the stream; the caller can assume it will be never used in
+    /// \c MasterLexer thereafter.
+    ///
+    /// This method must not be called when there is no source pushed for
+    /// \c MasterLexer.  This method is otherwise exception free.
+    ///
+    /// \throw isc::InvalidOperation Called with no pushed source.
+    void popSource();
+
+    /// \brief Return the name of the current input source name.
+    ///
+    /// If it's a file, it will be the C string given at the corresponding
+    /// \c pushSource() call, that is, its filename.  If it's a stream, it will
+    /// be formatted as \c "stream-%p" where \c %p is hex representation
+    /// of the address of the stream object.
+    ///
+    /// If there is no opened source at the time of the call, this method
+    /// returns an empty string.
+    ///
+    /// \throw std::bad_alloc Resource allocation failed for string
+    /// construction (rare case)
+    ///
+    /// \return A string representation of the current source (see the
+    /// description)
+    std::string getSourceName() const;
+
+    /// \brief Return the input source line number.
+    ///
+    /// If there is an opened source, the return value will be a non-0
+    /// integer indicating the line number of the current source where
+    /// the \c MasterLexer is currently working.  The expected usage of
+    /// this value is to print a helpful error message when parsing fails
+    /// by specifically identifying the position of the error.
+    ///
+    /// If there is no opened source at the time of the call, this method
+    /// returns 0.
+    ///
+    /// \throw None
+    ///
+    /// \return The current line number of the source (see the description)
+    size_t getSourceLine() const;
+
+private:
+    struct MasterLexerImpl;
+    MasterLexerImpl* impl_;
 };
 
 /// \brief Tokens for \c MasterLexer

src/lib/dns/master_lexer_inputsource.cc

@@ -32,6 +32,10 @@ createStreamName(const std::istream& input_stream) {
 
 } // end of unnamed namespace
 
+// Explicit definition of class static constant.  The value is given in the
+// declaration so it's not needed here.
+const int InputSource::END_OF_STREAM;
+
 InputSource::InputSource(std::istream& input_stream) :
     at_eof_(false),
     line_(1),

src/lib/dns/master_lexer_inputsource.h

@@ -41,7 +41,12 @@ namespace master_lexer_internal {
 class InputSource : boost::noncopyable {
 public:
     /// \brief Returned by getChar() when end of stream is reached.
-    static const int END_OF_STREAM;
+    ///
+    /// \note C++ allows a static const class member of an integral type to
+    /// be used without explicit definition as long as its address isn't
+    /// required.  But, since this is a public member variable and we cannot
+    /// assume how it's used, we give a definition in the implementation.
+    static const int END_OF_STREAM = -1;
 
     /// \brief Exception thrown when ungetChar() is made to go before
     /// the start of buffer.
@@ -151,8 +156,6 @@ private:
     std::istream& input_;
 };
 
-const int InputSource::END_OF_STREAM = -1;
-
 } // namespace master_lexer_internal
 } // namespace dns
 } // namespace isc

src/lib/dns/tests/Makefile.am

@@ -4,7 +4,7 @@ AM_CPPFLAGS = -I$(top_builddir)/src/lib -I$(top_srcdir)/src/lib
 AM_CPPFLAGS += $(BOOST_INCLUDES)
 AM_CPPFLAGS += -I$(top_srcdir)/src/lib/dns -I$(top_builddir)/src/lib/dns
 AM_CPPFLAGS += -I$(top_srcdir)/src/lib/util -I$(top_builddir)/src/lib/util
-AM_CPPFLAGS += -DTEST_DATA_SRCDIR=\"$(srcdir)/testdata\"
+AM_CPPFLAGS += -DTEST_DATA_SRCDIR=\"$(abs_top_srcdir)/src/lib/dns/tests/testdata\"
 AM_CPPFLAGS += -DTEST_DATA_BUILDDIR=\"$(abs_top_builddir)/src/lib/dns/tests/testdata\"
 AM_CXXFLAGS = $(B10_CXXFLAGS)
 
@@ -26,6 +26,7 @@ run_unittests_SOURCES += master_lexer_inputsource_unittest.cc
 run_unittests_SOURCES += labelsequence_unittest.cc
 run_unittests_SOURCES += messagerenderer_unittest.cc
 run_unittests_SOURCES += master_lexer_token_unittest.cc
+run_unittests_SOURCES += master_lexer_unittest.cc
 run_unittests_SOURCES += name_unittest.cc
 run_unittests_SOURCES += nsec3hash_unittest.cc
 run_unittests_SOURCES += rrclass_unittest.cc rrtype_unittest.cc

src/lib/dns/tests/master_lexer_unittest.cc

@@ -0,0 +1,127 @@
+// Copyright (C) 2012  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <exceptions/exceptions.h>
+
+#include <dns/master_lexer.h>
+
+#include <gtest/gtest.h>
+
+#include <boost/lexical_cast.hpp>
+
+#include <string>
+#include <sstream>
+
+using namespace isc::dns;
+using std::string;
+using std::stringstream;
+using boost::lexical_cast;
+
+namespace {
+
+class MasterLexerTest : public ::testing::Test {
+protected:
+    MasterLexerTest() :
+        expected_stream_name("stream-" + lexical_cast<string>(&ss))
+    {}
+
+    MasterLexer lexer;
+    stringstream ss;
+    const string expected_stream_name;
+};
+
+// Commonly used check case where the input sources stack is empty.
+void
+checkEmptySource(const MasterLexer& lexer) {
+    EXPECT_TRUE(lexer.getSourceName().empty());
+    EXPECT_EQ(0, lexer.getSourceLine());
+}
+
+TEST_F(MasterLexerTest, preOpen) {
+    // Initially sources stack is empty.
+    checkEmptySource(lexer);
+}
+
+TEST_F(MasterLexerTest, pushStream) {
+    lexer.pushSource(ss);
+    EXPECT_EQ(expected_stream_name, lexer.getSourceName());
+
+    // From the point of view of this test, we only have to check (though
+    // indirectly) getSourceLine calls InputSource::getCurrentLine.  It should
+    // return 1 initially.
+    EXPECT_EQ(1, lexer.getSourceLine());
+
+    // By popping it the stack will be empty again.
+    lexer.popSource();
+    checkEmptySource(lexer);
+}
+
+TEST_F(MasterLexerTest, pushFile) {
+    // We use zone file (-like) data, but in this test that actually doesn't
+    // matter.
+    EXPECT_TRUE(lexer.pushSource(TEST_DATA_SRCDIR "/masterload.txt"));
+    EXPECT_EQ(TEST_DATA_SRCDIR "/masterload.txt", lexer.getSourceName());
+    EXPECT_EQ(1, lexer.getSourceLine());
+
+    lexer.popSource();
+    checkEmptySource(lexer);
+
+    // If we give a non NULL string pointer, its content will be intact
+    // if pushSource succeeds.
+    std::string error_txt = "dummy";
+    EXPECT_TRUE(lexer.pushSource(TEST_DATA_SRCDIR "/masterload.txt",
+                                 &error_txt));
+    EXPECT_EQ("dummy", error_txt);
+}
+
+TEST_F(MasterLexerTest, pushBadFileName) {
+    EXPECT_THROW(lexer.pushSource(NULL), isc::InvalidParameter);
+}
+
+TEST_F(MasterLexerTest, pushFileFail) {
+    // The file to be pushed doesn't exist.  pushSource() fails and
+    // some non empty error string should be set.
+    std::string error_txt;
+    EXPECT_TRUE(error_txt.empty());
+    EXPECT_FALSE(lexer.pushSource("no-such-file", &error_txt));
+    EXPECT_FALSE(error_txt.empty());
+
+    // It's safe to pass NULL error_txt (either explicitly or implicitly as
+    // the default)
+    EXPECT_FALSE(lexer.pushSource("no-such-file", NULL));
+    EXPECT_FALSE(lexer.pushSource("no-such-file"));
+}
+
+TEST_F(MasterLexerTest, nestedPush) {
+    lexer.pushSource(ss);
+    EXPECT_EQ(expected_stream_name, lexer.getSourceName());
+
+    // We can push another source without popping the previous one.
+    lexer.pushSource(TEST_DATA_SRCDIR "/masterload.txt");
+    EXPECT_EQ(TEST_DATA_SRCDIR "/masterload.txt", lexer.getSourceName());
+
+    // popSource() works on the "topmost" (last-pushed) source
+    lexer.popSource();
+    EXPECT_EQ(expected_stream_name, lexer.getSourceName());
+
+    lexer.popSource();
+    EXPECT_TRUE(lexer.getSourceName().empty());
+}
+
+TEST_F(MasterLexerTest, invalidPop) {
+    // popSource() cannot be called if the sources stack is empty.
+    EXPECT_THROW(lexer.popSource(), isc::InvalidOperation);
+}
+
+}