[2372] Merge branch 'trac2371' into trac2372

src/lib/dns/Makefile.am

@@ -93,6 +93,7 @@ libb10_dns___la_LDFLAGS = -no-undefined -version-info 2:0:0
 libb10_dns___la_SOURCES =
 libb10_dns___la_SOURCES += edns.h edns.cc
 libb10_dns___la_SOURCES += exceptions.h exceptions.cc
+libb10_dns___la_SOURCES += master_lexer_inputsource.h master_lexer_inputsource.cc
 libb10_dns___la_SOURCES += labelsequence.h labelsequence.cc
 libb10_dns___la_SOURCES += masterload.h masterload.cc
 libb10_dns___la_SOURCES += master_lexer.h master_lexer.cc

src/lib/dns/master_lexer.cc

@@ -12,10 +12,87 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
+#include <exceptions/exceptions.h>
+
 #include <dns/master_lexer.h>
+#include <dns/master_lexer_inputsource.h>
+
+#include <boost/shared_ptr.hpp>
 
 #include <cassert>
 #include <string>
+#include <sstream>
+#include <vector>
+
+namespace isc {
+namespace dns {
+
+namespace {
+typedef boost::shared_ptr<master_lexer_internal::InputSource> InputSourcePtr;
+}
+using namespace master_lexer_internal;
+
+struct MasterLexer::MasterLexerImpl {
+    MasterLexerImpl() : token_(Token::NOT_STARTED) {}
+
+    std::vector<InputSourcePtr> sources_;
+    Token token_;
+};
+
+MasterLexer::MasterLexer() : impl_(new MasterLexerImpl) {
+}
+
+MasterLexer::~MasterLexer() {
+    delete impl_;
+}
+
+bool
+MasterLexer::pushSource(const char* filename, std::string* error) {
+    if (filename == NULL) {
+        isc_throw(InvalidParameter,
+                  "NULL filename for MasterLexer::pushSource");
+    }
+    try {
+        impl_->sources_.push_back(InputSourcePtr(new InputSource(filename)));
+    } catch (const InputSource::OpenError& ex) {
+        if (error != NULL) {
+            *error = ex.what();
+        }
+        return (false);
+    }
+
+    return (true);
+}
+
+void
+MasterLexer::pushSource(std::istream& input) {
+    impl_->sources_.push_back(InputSourcePtr(new InputSource(input)));
+}
+
+void
+MasterLexer::popSource() {
+    if (impl_->sources_.empty()) {
+        isc_throw(InvalidOperation,
+                  "MasterLexer::popSource on an empty source");
+    }
+    impl_->sources_.pop_back();
+}
+
+std::string
+MasterLexer::getSourceName() const {
+    if (impl_->sources_.empty()) {
+        return (std::string());
+    }
+    return (impl_->sources_.back()->getName());
+}
+
+size_t
+MasterLexer::getSourceLine() const {
+    if (impl_->sources_.empty()) {
+        return (0);
+    }
+    return (impl_->sources_.back()->getCurrentLine());
+}
 
 namespace {
 const char* const error_text[] = {
@@ -27,9 +104,6 @@ const char* const error_text[] = {
 const size_t error_text_max_count = sizeof(error_text) / sizeof(error_text[0]);
 }
 
-namespace isc {
-namespace dns {
-
 std::string
 MasterLexer::Token::getErrorText() const {
     if (type_ != ERROR) {
@@ -42,6 +116,5 @@ MasterLexer::Token::getErrorText() const {
     return (error_text[val_.error_code_]);
 }
 
-
 } // end of namespace dns
 } // end of namespace isc

src/lib/dns/master_lexer.h

@@ -17,6 +17,7 @@
 
 #include <exceptions/exceptions.h>
 
+#include <istream>
 #include <string>
 
 #include <stdint.h>
@@ -24,9 +25,120 @@
 namespace isc {
 namespace dns {
 
+/// \brief Tokenizer for parsing DNS master files.
+///
+/// The \c MasterLexer class provides tokenize interfaces for parsing DNS
+/// master files.  It understands some special rules of master files as
+/// defined in RFC 1035, such as comments, character escaping, and multi-line
+/// data, and provides the user application with the actual data in a
+/// more convenient form such as a std::string object.
+///
+/// In order to support the $INCLUDE notation, this class is designed to be
+/// able to operate on multiple files or input streams in the nested way.
+/// The \c pushSource() and \c popSource() methods correspond to the push
+/// and pop operations.
+///
+/// While this class is public, it is less likely to be used by normal
+/// applications; it's mainly expected to be used within this library,
+/// specifically by the \c MasterLoader class and \c Rdata implementation
+/// classes.
 class MasterLexer {
 public:
     class Token;       // we define it separately for better readability
+
+    /// \brief The constructor.
+    ///
+    /// \throw std::bad_alloc Internal resource allocation fails (rare case).
+    MasterLexer();
+
+    /// \brief The destructor.
+    ///
+    /// It internally closes any remaining input sources.
+    ~MasterLexer();
+
+    /// \brief Open a file and make it the current input source of MasterLexer.
+    ///
+    /// The opened file can be explicitly closed by the \c popSource() method;
+    /// if \c popSource() is not called within the lifetime of the
+    /// \c MasterLexer, it will be closed in the destructor.
+    ///
+    /// In the case possible system errors in opening the file (most likely
+    /// because of specifying a non-existent or unreadable file), it returns
+    /// false, and if the optional \c error parameter is non NULL, it will be
+    /// set to a description of the error (any existing content of the string
+    /// will be discarded).  If opening the file succeeds, the given
+    /// \c error parameter will be intact.
+    ///
+    /// \throw InvalidParameter filename is NULL
+    /// \param filename A non NULL string specifying a master file
+    /// \param error If non null, a placeholder to set error description in
+    /// case of failure.
+    ///
+    /// \return true if pushing the file succeeds; false otherwise.
+    bool pushSource(const char* filename, std::string* error = NULL);
+
+    /// \brief Make the given stream the current input source of MasterLexer.
+    ///
+    /// The caller still holds the ownership of the passed stream; it's the
+    /// caller's responsibility to keep it valid as long as it's used in
+    /// \c MasterLexer or to release any resource for the stream after that.
+    /// The caller can explicitly tell \c MasterLexer to stop using the
+    /// stream by calling the \c popSource() method.
+    ///
+    /// \param input An input stream object that produces textual
+    /// representation of DNS RRs.
+    void pushSource(std::istream& input);
+
+    /// \brief Stop using the most recently opened input source (file or
+    /// stream).
+    ///
+    /// If it's a file, the previously opened file will be closed internally.
+    /// If it's a stream, \c MasterLexer will simply stop using
+    /// the stream; the caller can assume it will be never used in
+    /// \c MasterLexer thereafter.
+    ///
+    /// This method must not be called when there is no source pushed for
+    /// \c MasterLexer.  This method is otherwise exception free.
+    ///
+    /// \throw isc::InvalidOperation Called with no pushed source.
+    void popSource();
+
+    /// \brief Return the name of the current input source name.
+    ///
+    /// If it's a file, it will be the C string given at the corresponding
+    /// \c pushSource() call, that is, its filename.  If it's a stream, it will
+    /// be formatted as \c "stream-%p" where \c %p is hex representation
+    /// of the address of the stream object.
+    ///
+    /// If there is no opened source at the time of the call, this method
+    /// returns an empty string.
+    ///
+    /// \throw std::bad_alloc Resource allocation failed for string
+    /// construction (rare case)
+    ///
+    /// \return A string representation of the current source (see the
+    /// description)
+    std::string getSourceName() const;
+
+    /// \brief Return the input source line number.
+    ///
+    /// If there is an opened source, the return value will be a non-0
+    /// integer indicating the line number of the current source where
+    /// the \c MasterLexer is currently working.  The expected usage of
+    /// this value is to print a helpful error message when parsing fails
+    /// by specifically identifying the position of the error.
+    ///
+    /// If there is no opened source at the time of the call, this method
+    /// returns 0.
+    ///
+    /// \throw None
+    ///
+    /// \return The current line number of the source (see the description)
+    size_t getSourceLine() const;
+
+private:
+    struct MasterLexerImpl;
+    MasterLexerImpl* impl_;
 };
 
 /// \brief Tokens for \c MasterLexer

src/lib/dns/master_lexer_inputsource.cc

@@ -0,0 +1,133 @@
+// Copyright (C) 2012  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <dns/master_lexer_inputsource.h>
+
+namespace isc {
+namespace dns {
+namespace master_lexer_internal {
+
+namespace { // unnamed namespace
+
+std::string
+createStreamName(std::istream& input_stream) {
+     std::stringstream ss;
+     ss << "stream-" << &input_stream;
+     return (ss.str());
+}
+
+} // end of unnamed namespace
+
+InputSource::InputSource(std::istream& input_stream) :
+    at_eof_(false),
+    line_(1),
+    saved_line_(line_),
+    buffer_pos_(0),
+    name_(createStreamName(input_stream)),
+    input_(input_stream)
+{}
+
+InputSource::InputSource(const char* filename) :
+    at_eof_(false),
+    line_(1),
+    saved_line_(line_),
+    buffer_pos_(0),
+    name_(filename),
+    input_(file_stream_)
+{
+    file_stream_.open(filename, std::fstream::in);
+    if (file_stream_.fail()) {
+        isc_throw(OpenError,
+                  "Error opening the input source file: " << filename);
+    }
+}
+
+InputSource::~InputSource()
+{
+    if (file_stream_.is_open()) {
+        file_stream_.close();
+    }
+}
+
+int
+InputSource::getChar() {
+    if (buffer_pos_ == buffer_.size()) {
+        // We may have reached EOF at the last call to
+        // getChar(). at_eof_ will be set then. We then simply return
+        // early.
+        if (at_eof_) {
+            return (END_OF_STREAM);
+        }
+        // We are not yet at EOF. Read from the stream.
+        int c = input_.get();
+        // Have we reached EOF now? If so, set at_eof_ and return early,
+        // but don't modify buffer_pos_ (which should still be equal to
+        // the size of buffer_).
+        if (input_.eof()) {
+            at_eof_ = true;
+            return (END_OF_STREAM);
+        }
+        // This has to come after the .eof() check as some
+        // implementations seem to check the eofbit also in .fail().
+        if (input_.fail()) {
+            isc_throw(ReadError,
+                      "Error reading from the input stream: " << getName());
+        }
+        buffer_.push_back(c);
+    }
+
+    int c = buffer_[buffer_pos_++];
+    if (c == '\n') {
+        line_++;
+    }
+
+    return (c);
+}
+
+void
+InputSource::ungetChar() {
+    if (at_eof_) {
+        at_eof_ = false;
+    } else if (buffer_pos_ == 0) {
+        isc_throw(UngetBeforeBeginning,
+                  "Cannot skip before the start of buffer");
+    } else {
+        buffer_pos_--;
+        if (buffer_[buffer_pos_] == '\n') {
+            line_--;
+        }
+    }
+}
+
+void
+InputSource::ungetAll() {
+    buffer_pos_ = 0;
+    line_ = saved_line_;
+    at_eof_ = false;
+}
+
+void
+InputSource::compact() {
+    if (buffer_pos_ == buffer_.size()) {
+        buffer_.clear();
+    } else {
+        buffer_.erase(buffer_.begin(), buffer_.begin() + buffer_pos_);
+    }
+
+    buffer_pos_ = 0;
+}
+
+} // namespace master_lexer_internal
+} // namespace dns
+} // namespace isc

src/lib/dns/master_lexer_inputsource.h

@@ -0,0 +1,150 @@
+// Copyright (C) 2012  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#ifndef DNS_INPUTSOURCE_H
+#define DNS_INPUTSOURCE_H 1
+
+#include <exceptions/exceptions.h>
+
+#include <iostream>
+#include <fstream>
+#include <string>
+#include <vector>
+
+namespace isc {
+namespace dns {
+namespace master_lexer_internal {
+
+/// \brief An input source that is used internally by MasterLexer.
+///
+/// This is a helper internal class for MasterLexer, and represents
+/// state of a single source of the entire zone data to be
+/// parsed. Normally this means the master zone file, but MasterLexer
+/// can have multiple InputSources if $INCLUDE is used. The source can
+/// also be generic input stream (std::istream).
+///
+/// This class is not meant for public use.
+class InputSource {
+public:
+    /// \brief Returned by getChar() when end of stream is reached.
+    static const int END_OF_STREAM = -1;
+
+    /// \brief Exception thrown when ungetChar() is made to go before
+    /// the start of buffer.
+    struct UngetBeforeBeginning : public OutOfRange {
+        UngetBeforeBeginning(const char* file, size_t line, const char* what) :
+            OutOfRange(file, line, what)
+        {}
+    };
+
+    /// \brief Exception thrown when we fail to read from the input
+    /// stream or file.
+    struct ReadError : public Unexpected {
+        ReadError(const char* file, size_t line, const char* what) :
+            Unexpected(file, line, what)
+        {}
+    };
+
+    /// \brief Exception thrown when we fail to open the input file.
+    struct OpenError : public Unexpected {
+        OpenError(const char* file, size_t line, const char* what) :
+            Unexpected(file, line, what)
+        {}
+    };
+
+    /// \brief Constructor which takes an input stream. The stream is
+    /// read-from, but it is not closed.
+    InputSource(std::istream& input_stream);
+
+    /// \brief Constructor which takes a filename to read from. The
+    /// associated file stream is managed internally.
+    ///
+    /// \throws OpenError when opening the input file fails.
+    InputSource(const char* filename);
+
+    /// \brief Destructor
+    ~InputSource();
+
+    /// \brief Returns a name for the InputSource. Typically this is the
+    /// filename, but if the InputSource was constructed for an
+    /// \c std::istream, it returns a name in the format "stream-%p".
+    const std::string& getName() const {
+        return (name_);
+    }
+
+    /// \brief Returns if the input source is at end of file.
+    bool atEOF() const {
+        return (at_eof_);
+    }
+
+    /// \brief Returns the current line number being read.
+    size_t getCurrentLine() const {
+        return (line_);
+    }
+
+    /// \brief Saves the current line being read. Later, when
+    /// \c ungetAll() is called, it skips back to the last-saved line.
+    void saveLine() {
+        saved_line_ = line_;
+    }
+
+    /// \brief Returns a single character from the input source. If end
+    /// of file is reached, \c END_OF_STREAM is returned.
+    ///
+    /// \throws ReadError when reading from the input stream or file
+    /// fails.
+    int getChar();
+
+    /// \brief Skips backward a single character in the input
+    /// source. The last-read character is unget.
+    ///
+    /// \throws UngetBeforeBeginning if we go backwards past the start
+    /// of reading, or backwards past the last time compact() was
+    /// called.
+    void ungetChar();
+
+    /// Forgets what was read, and skips back to the position where
+    /// \c compact() was last called. If \c compact() was not called, it
+    /// skips back to where reading started. If \c saveLine() was called
+    /// previously, it sets the current line number to the line number
+    /// saved.
+    void ungetAll();
+
+    /// Removes buffered content before the current location in the
+    /// \c InputSource. It's not possible to \c ungetChar() after this,
+    /// unless we read more data using \c getChar().
+    void compact();
+
+private:
+    bool at_eof_;
+    size_t line_;
+    size_t saved_line_;
+
+    std::vector<char> buffer_;
+    size_t buffer_pos_;
+
+    const std::string name_;
+    std::fstream file_stream_;
+    std::istream& input_;
+};
+
+} // namespace master_lexer_internal
+} // namespace dns
+} // namespace isc
+
+#endif  // DNS_INPUTSOURCE_H
+
+// Local Variables:
+// mode: c++
+// End:

src/lib/dns/tests/Makefile.am

@@ -4,7 +4,7 @@ AM_CPPFLAGS = -I$(top_builddir)/src/lib -I$(top_srcdir)/src/lib
 AM_CPPFLAGS += $(BOOST_INCLUDES)
 AM_CPPFLAGS += -I$(top_srcdir)/src/lib/dns -I$(top_builddir)/src/lib/dns
 AM_CPPFLAGS += -I$(top_srcdir)/src/lib/util -I$(top_builddir)/src/lib/util
-AM_CPPFLAGS += -DTEST_DATA_SRCDIR=\"$(srcdir)/testdata\"
+AM_CPPFLAGS += -DTEST_DATA_SRCDIR=\"$(abs_top_srcdir)/src/lib/dns/tests/testdata\"
 AM_CPPFLAGS += -DTEST_DATA_BUILDDIR=\"$(abs_top_builddir)/src/lib/dns/tests/testdata\"
 AM_CXXFLAGS = $(B10_CXXFLAGS)
 
@@ -22,9 +22,11 @@ if HAVE_GTEST
 TESTS += run_unittests
 run_unittests_SOURCES = unittest_util.h unittest_util.cc
 run_unittests_SOURCES += edns_unittest.cc
+run_unittests_SOURCES += master_lexer_inputsource_unittest.cc
 run_unittests_SOURCES += labelsequence_unittest.cc
 run_unittests_SOURCES += messagerenderer_unittest.cc
 run_unittests_SOURCES += master_lexer_token_unittest.cc
+run_unittests_SOURCES += master_lexer_unittest.cc
 run_unittests_SOURCES += name_unittest.cc
 run_unittests_SOURCES += nsec3hash_unittest.cc
 run_unittests_SOURCES += rrclass_unittest.cc rrtype_unittest.cc

src/lib/dns/tests/master_lexer_inputsource_unittest.cc

@@ -0,0 +1,333 @@
+// Copyright (C) 2012  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <dns/master_lexer_inputsource.h>
+#include <exceptions/exceptions.h>
+
+#include <gtest/gtest.h>
+
+#include <iostream>
+#include <sstream>
+#include <string>
+
+#include <string.h>
+
+using namespace std;
+using namespace isc::dns;
+using namespace isc::dns::master_lexer_internal;
+
+// Some compilers cannot find symbols of class constants when used in the
+// EXPECT_xxx macros, so we need explicit declaration.
+const int InputSource::END_OF_STREAM;
+
+namespace {
+
+class InputSourceTest : public ::testing::Test {
+protected:
+    InputSourceTest() :
+        str_("Line1 to scan.\nLine2 to scan.\nLine3 to scan.\n"),
+        str_length_(strlen(str_)),
+        iss_(str_),
+        source_(iss_)
+    {}
+
+    const char* str_;
+    const size_t str_length_;
+    stringstream iss_;
+    InputSource source_;
+};
+
+// Test the default return values set during InputSource construction.
+TEST_F(InputSourceTest, defaults) {
+    EXPECT_EQ(1, source_.getCurrentLine());
+    EXPECT_FALSE(source_.atEOF());
+}
+
+// getName() on file and stream sources
+TEST_F(InputSourceTest, getName) {
+    EXPECT_EQ(0, source_.getName().find("stream-"));
+
+    // Use some file; doesn't really matter what.
+    InputSource source2(TEST_DATA_SRCDIR "/masterload.txt");
+    EXPECT_EQ(TEST_DATA_SRCDIR "/masterload.txt", source2.getName());
+}
+
+TEST_F(InputSourceTest, nonExistentFile) {
+    EXPECT_THROW({
+        InputSource source(TEST_DATA_SRCDIR "/videokilledtheradiostar");
+    }, InputSource::OpenError);
+}
+
+// getChar() should return characters from the input stream in
+// sequence. ungetChar() should skip backwards.
+void
+checkGetAndUngetChar(InputSource& source, const char* str, size_t str_length)
+{
+    for (size_t i = 0; i < str_length; i++) {
+        EXPECT_EQ(str[i], source.getChar());
+        EXPECT_FALSE(source.atEOF());
+    }
+
+    // At this point, we still have not reached EOF.
+    EXPECT_FALSE(source.atEOF());
+
+    // This should cause EOF to be set.
+    EXPECT_EQ(InputSource::END_OF_STREAM, source.getChar());
+
+    // Now, EOF should be set.
+    EXPECT_TRUE(source.atEOF());
+
+    // Now, let's go backwards. This should cause the EOF to be set to
+    // false.
+    source.ungetChar();
+
+    // Now, EOF should be false.
+    EXPECT_FALSE(source.atEOF());
+
+    // This should cause EOF to be set again.
+    EXPECT_EQ(InputSource::END_OF_STREAM, source.getChar());
+
+    // Now, EOF should be set.
+    EXPECT_TRUE(source.atEOF());
+
+    // Now, let's go backwards in a loop. Start by skipping the EOF.
+    source.ungetChar();
+
+    for (size_t i = 0; i < str_length; i++) {
+        size_t index = str_length - 1 - i;
+        // Skip one character.
+        source.ungetChar();
+        EXPECT_EQ(str[index], source.getChar());
+        // Skip the character we received again.
+        source.ungetChar();
+    }
+
+    // Skipping past the start of buffer should throw.
+    EXPECT_THROW(source.ungetChar(), InputSource::UngetBeforeBeginning);
+}
+
+TEST_F(InputSourceTest, stream) {
+    checkGetAndUngetChar(source_, str_, str_length_);
+}
+
+TEST_F(InputSourceTest, file) {
+    const char* str =
+        ";; a simple (incomplete) zone file\n"
+        "\n"
+        "example.com. 3600 IN TXT \"test data\"\n"
+        "www.example.com. 60 IN A 192.0.2.1\n"
+        "www.example.com. 60 IN A 192.0.2.2\n";
+    size_t str_length = strlen(str);
+
+    InputSource source(TEST_DATA_SRCDIR "/masterload.txt");
+    checkGetAndUngetChar(source, str, str_length);
+}
+
+// ungetAll() should skip back to the place where the InputSource
+// started at construction, or the last saved start of line.
+TEST_F(InputSourceTest, ungetAll) {
+    while (!source_.atEOF()) {
+        source_.getChar();
+    }
+
+    // Now, we are at EOF.
+    EXPECT_TRUE(source_.atEOF());
+    EXPECT_EQ(4, source_.getCurrentLine());
+
+    source_.ungetAll();
+
+    // Now we are back to where we started.
+    EXPECT_EQ(1, source_.getCurrentLine());
+    EXPECT_FALSE(source_.atEOF());
+}
+
+TEST_F(InputSourceTest, compact) {
+    // Compact at the start
+    source_.compact();
+
+    // Ungetting here must throw.
+    EXPECT_THROW(source_.ungetChar(), InputSource::UngetBeforeBeginning);
+
+    for (size_t i = 0; i < str_length_; i++) {
+        EXPECT_EQ(str_[i], source_.getChar());
+        EXPECT_FALSE(source_.atEOF());
+    }
+
+    // At this point, we still have not reached EOF.
+    EXPECT_FALSE(source_.atEOF());
+
+    // This should cause EOF to be set.
+    EXPECT_EQ(InputSource::END_OF_STREAM, source_.getChar());
+
+    // Now, EOF should be set.
+    EXPECT_TRUE(source_.atEOF());
+    EXPECT_EQ(4, source_.getCurrentLine());
+
+    // Compact again
+    source_.compact();
+
+    // We are still at EOF.
+    EXPECT_TRUE(source_.atEOF());
+    EXPECT_EQ(4, source_.getCurrentLine());
+
+    // Skip the EOF.
+    source_.ungetChar();
+
+    // Ungetting here must throw.
+    EXPECT_THROW(source_.ungetChar(), InputSource::UngetBeforeBeginning);
+
+    EXPECT_EQ(InputSource::END_OF_STREAM, source_.getChar());
+    EXPECT_TRUE(source_.atEOF());
+}
+
+TEST_F(InputSourceTest, compactDuring) {
+    // First, skip to line 2.
+    while (!source_.atEOF() &&
+           (source_.getCurrentLine() != 2)) {
+        source_.getChar();
+    }
+    EXPECT_FALSE(source_.atEOF());
+    size_t line = source_.getCurrentLine();
+    EXPECT_EQ(2, line);
+
+    // Now, unget a couple of characters. This should cause the
+    // buffer_pos_ to be not equal to the size of the buffer.
+    source_.ungetChar();
+    source_.ungetChar();
+
+    source_.saveLine();
+    source_.compact();
+
+    // Ungetting here must throw.
+    EXPECT_THROW(source_.ungetChar(), InputSource::UngetBeforeBeginning);
+
+    for (size_t i = 13; i < str_length_; i++) {
+        EXPECT_EQ(str_[i], source_.getChar());
+        EXPECT_FALSE(source_.atEOF());
+    }
+
+    // At this point, we still have not reached EOF.
+    EXPECT_FALSE(source_.atEOF());
+
+    // This should cause EOF to be set.
+    EXPECT_EQ(InputSource::END_OF_STREAM, source_.getChar());
+
+    // Now, EOF should be set.
+    EXPECT_TRUE(source_.atEOF());
+
+    // Now, ungetAll() and check where it goes back.
+    source_.ungetAll();
+
+    // Ungetting here must throw.
+    EXPECT_THROW(source_.ungetChar(), InputSource::UngetBeforeBeginning);
+
+    for (size_t i = 13; i < str_length_; i++) {
+        EXPECT_EQ(str_[i], source_.getChar());
+        EXPECT_FALSE(source_.atEOF());
+    }
+
+    // At this point, we still have not reached EOF.
+    EXPECT_FALSE(source_.atEOF());
+
+    // This should cause EOF to be set.
+    EXPECT_EQ(InputSource::END_OF_STREAM, source_.getChar());
+
+    // Now, EOF should be set.
+    EXPECT_TRUE(source_.atEOF());
+}
+
+// Test line counters.
+TEST_F(InputSourceTest, lines) {
+    size_t line = 1;
+    while (!source_.atEOF()) {
+        if (source_.getChar() == '\n') {
+            line++;
+        }
+        EXPECT_EQ(line, source_.getCurrentLine());
+    }
+
+    // Now, we are at EOF.
+    EXPECT_TRUE(source_.atEOF());
+    EXPECT_EQ(4, source_.getCurrentLine());
+
+    // Go backwards 2 characters, skipping the last EOF and '\n'.
+    source_.ungetChar();
+    source_.ungetChar();
+
+    EXPECT_FALSE(source_.atEOF());
+    EXPECT_EQ(3, source_.getCurrentLine());
+
+    source_.ungetAll();
+
+    // Now we are back to where we started.
+    EXPECT_EQ(1, source_.getCurrentLine());
+    EXPECT_FALSE(source_.atEOF());
+
+    // Now check that line numbers are decremented properly (as much as
+    // possible using the available API).
+    while (!source_.atEOF()) {
+        source_.getChar();
+    }
+    line = source_.getCurrentLine();
+
+    // Now, we are at EOF.
+    EXPECT_TRUE(source_.atEOF());
+    EXPECT_EQ(4, line);
+
+    EXPECT_THROW({
+        while (true) {
+            source_.ungetChar();
+            EXPECT_TRUE(((line == source_.getCurrentLine()) ||
+                         ((line - 1) == source_.getCurrentLine())));
+            line = source_.getCurrentLine();
+        }
+    }, InputSource::UngetBeforeBeginning);
+
+    // Now we are back to where we started.
+    EXPECT_EQ(1, source_.getCurrentLine());
+}
+
+// ungetAll() after saveLine() should skip back to the last-saved place.
+TEST_F(InputSourceTest, saveLine) {
+    // First, skip to line 2.
+    while (!source_.atEOF() &&
+           (source_.getCurrentLine() != 2)) {
+        source_.getChar();
+    }
+    EXPECT_FALSE(source_.atEOF());
+    size_t line = source_.getCurrentLine();
+    EXPECT_EQ(2, line);
+
+    // Now, save the line.
+    source_.saveLine();
+
+    // Now, go to EOF
+    while (!source_.atEOF()) {
+        source_.getChar();
+    }
+    line = source_.getCurrentLine();
+
+    // Now, we are at EOF.
+    EXPECT_TRUE(source_.atEOF());
+    EXPECT_EQ(4, line);
+
+    // Now, ungetAll() and check where it goes back.
+    source_.ungetAll();
+
+    // Now we are back to where we last-saved.
+    EXPECT_EQ(2, source_.getCurrentLine());
+    EXPECT_FALSE(source_.atEOF());
+}
+
+} // end namespace

src/lib/dns/tests/master_lexer_unittest.cc

@@ -0,0 +1,127 @@
+// Copyright (C) 2012  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <exceptions/exceptions.h>
+
+#include <dns/master_lexer.h>
+
+#include <gtest/gtest.h>
+
+#include <boost/lexical_cast.hpp>
+
+#include <string>
+#include <sstream>
+
+using namespace isc::dns;
+using std::string;
+using std::stringstream;
+using boost::lexical_cast;
+
+namespace {
+
+class MasterLexerTest : public ::testing::Test {
+protected:
+    MasterLexerTest() :
+        expected_stream_name("stream-" + lexical_cast<string>(&ss))
+    {}
+
+    MasterLexer lexer;
+    stringstream ss;
+    const string expected_stream_name;
+};
+
+// Commonly used check case where the input sources stack is empty.
+void
+checkEmptySource(const MasterLexer& lexer) {
+    EXPECT_TRUE(lexer.getSourceName().empty());
+    EXPECT_EQ(0, lexer.getSourceLine());
+}
+
+TEST_F(MasterLexerTest, preOpen) {
+    // Initially sources stack is empty.
+    checkEmptySource(lexer);
+}
+
+TEST_F(MasterLexerTest, pushStream) {
+    lexer.pushSource(ss);
+    EXPECT_EQ(expected_stream_name, lexer.getSourceName());
+
+    // From the point of view of this test, we only have to check (though
+    // indirectly) getSourceLine calls InputSource::getCurrentLine.  It should
+    // return 1 initially.
+    EXPECT_EQ(1, lexer.getSourceLine());
+
+    // By closing it the stack will be empty again.
+    lexer.popSource();
+    checkEmptySource(lexer);
+}
+
+TEST_F(MasterLexerTest, pushFile) {
+    // We use zone file (-like) data, but in this test that actually doesn't
+    // matter.
+    EXPECT_TRUE(lexer.pushSource(TEST_DATA_SRCDIR "/masterload.txt"));
+    EXPECT_EQ(TEST_DATA_SRCDIR "/masterload.txt", lexer.getSourceName());
+    EXPECT_EQ(1, lexer.getSourceLine());
+
+    lexer.popSource();
+    checkEmptySource(lexer);
+
+    // If we give a non NULL string pointer, its content will be intact
+    // if pushSource succeeds.
+    std::string error_txt = "dummy";
+    EXPECT_TRUE(lexer.pushSource(TEST_DATA_SRCDIR "/masterload.txt",
+                                 &error_txt));
+    EXPECT_EQ("dummy", error_txt);
+}
+
+TEST_F(MasterLexerTest, pushBadFileName) {
+    EXPECT_THROW(lexer.pushSource(NULL), isc::InvalidParameter);
+}
+
+TEST_F(MasterLexerTest, pushFileFail) {
+    // The file to be pushed doesn't exist.  pushSource() fails and
+    // some non empty error string should be set.
+    std::string error_txt;
+    EXPECT_TRUE(error_txt.empty());
+    EXPECT_FALSE(lexer.pushSource("no-such-file", &error_txt));
+    EXPECT_FALSE(error_txt.empty());
+
+    // It's safe to pass NULL error_txt (either explicitly or implicitly as
+    // the default)
+    EXPECT_FALSE(lexer.pushSource("no-such-file", NULL));
+    EXPECT_FALSE(lexer.pushSource("no-such-file"));
+}
+
+TEST_F(MasterLexerTest, nestedPush) {
+    lexer.pushSource(ss);
+    EXPECT_EQ(expected_stream_name, lexer.getSourceName());
+
+    // We can push another source without popping the previous one.
+    lexer.pushSource(TEST_DATA_SRCDIR "/masterload.txt");
+    EXPECT_EQ(TEST_DATA_SRCDIR "/masterload.txt", lexer.getSourceName());
+
+    // Close works on the "topmost" (last-pushed) source
+    lexer.popSource();
+    EXPECT_EQ(expected_stream_name, lexer.getSourceName());
+
+    lexer.popSource();
+    EXPECT_TRUE(lexer.getSourceName().empty());
+}
+
+TEST_F(MasterLexerTest, invalidClose) {
+    // popSource() cannot be called if the sources stack is empty.
+    EXPECT_THROW(lexer.popSource(), isc::InvalidOperation);
+}
+
+}