[2396] Merge branch 'master' into trac2396

ChangeLog

@@ -1,3 +1,10 @@
+516.	[bug]		marcin
+	Fixed 'make distcheck' failure when running perfdhcp unit tests.
+	The unit tests used to read files from the folder specified
+	with the path relative to current folder, thus when the test was
+	run from a different folder the files could not be found.
+	(Trac #2479, git 4e8325e1b309f1d388a3055ec1e1df98c377f383)
+
 515.	[bug]		jinmei
 	The in-memory data source now accepts an RRSIG provided without
 	a covered RRset in loading.  A subsequent query for its owner name

configure.ac

@@ -1308,7 +1308,7 @@ AC_CONFIG_FILES([Makefile
                  tests/tools/badpacket/tests/Makefile
                  tests/tools/perfdhcp/Makefile
                  tests/tools/perfdhcp/tests/Makefile
-                 tests/tools/perfdhcp/templates/Makefile
+                 tests/tools/perfdhcp/tests/testdata/Makefile
                  dns++.pc
                ])
 AC_OUTPUT([doc/version.ent

doc/guide/bind10-guide.xml

@@ -472,7 +472,7 @@ var/
       <title>Packages</title>
 
       <para>
-        Some operating systems or softare package vendors may
+        Some operating systems or software package vendors may
         provide ready-to-use, pre-built software packages for
         the BIND 10 suite.
         Installing a pre-built package means you do not need to
@@ -2157,7 +2157,7 @@ AND_MATCH := "ALL": [ RULE_RAW, RULE_RAW, ... ]
 	you indicate that the system is not usable without the
 	component and if such component fails, the system shuts
 	down no matter when the failure happened.  This is the
-	behaviour of the core components (the ones you can't turn
+	behavior of the core components (the ones you can't turn
 	off), but you can declare any other components as core as
 	well if you wish (but you can turn these off, they just
 	can't fail).

src/bin/auth/tests/auth_srv_unittest.cc

@@ -225,7 +225,7 @@ createBuiltinVersionResponse(const qid_t qid, vector<uint8_t>& data) {
     message.setHeaderFlag(Message::HEADERFLAG_AA);
     RRsetPtr rrset_version = RRsetPtr(new RRset(version_name, RRClass::CH(),
                                                 RRType::TXT(), RRTTL(0)));
-    rrset_version->addRdata(generic::TXT(PACKAGE_STRING));
+    rrset_version->addRdata(generic::TXT("\"" PACKAGE_STRING "\""));
     message.addRRset(Message::SECTION_ANSWER, rrset_version);
 
     RRsetPtr rrset_version_ns = RRsetPtr(new RRset(apex_name, RRClass::CH(),

src/bin/zonemgr/zonemgr.py.in

@@ -193,7 +193,8 @@ class ZonemgrRefresh:
     def zone_handle_notify(self, zone_name_class, master):
         """Handle zone notify"""
         if (self._zone_not_exist(zone_name_class)):
-            logger.error(ZONEMGR_UNKNOWN_ZONE_NOTIFIED, zone_name_class[0], zone_name_class[1])
+            logger.error(ZONEMGR_UNKNOWN_ZONE_NOTIFIED, zone_name_class[0],
+                         zone_name_class[1], master)
             raise ZonemgrException("[b10-zonemgr] Notified zone (%s, %s) "
                                    "doesn't belong to zonemgr" % zone_name_class)
         self._set_zone_notifier_master(zone_name_class, master)

src/bin/zonemgr/zonemgr_messages.mes

@@ -138,7 +138,7 @@ zone, or, if this error appears without the administrator giving transfer
 commands, it can indicate an error in the program, as it should not have
 initiated transfers of unknown zones on its own.
 
-% ZONEMGR_UNKNOWN_ZONE_NOTIFIED notified zone %1 (class %2) is not known to the zone manager
+% ZONEMGR_UNKNOWN_ZONE_NOTIFIED notified zone %1/%2 from %3 is not known to the zone manager
 A NOTIFY was received but the zone that was the subject of the operation
 is not being managed by the zone manager.  This may indicate an error
 in the program (as the operation should not have been initiated if this

src/lib/dhcp/Makefile.am

@@ -40,7 +40,7 @@ libb10_dhcp___la_LIBADD   = $(top_builddir)/src/lib/asiolink/libb10-asiolink.la
 libb10_dhcp___la_LIBADD  += $(top_builddir)/src/lib/util/libb10-util.la
 libb10_dhcp___la_LDFLAGS  = -no-undefined -version-info 2:0:0
 
-EXTRA_DIST  = README
+EXTRA_DIST  = README libdhcp++.dox
 
 if USE_CLANGPP
 # Disable unused parameter warning caused by some of the

src/lib/dhcpsrv/Makefile.am

@@ -48,5 +48,5 @@ libb10_dhcpsrv_la_CXXFLAGS += -Wno-unused-parameter
 endif
 
 # Distribute MySQL schema creation script and backend documentation
-EXTRA_DIST = dhcpdb_create.mysql database_backends.dox
+EXTRA_DIST = dhcpdb_create.mysql database_backends.dox libdhcpsrv.dox
 dist_pkgdata_DATA = dhcpdb_create.mysql

src/lib/dns/Makefile.am

@@ -21,6 +21,8 @@ EXTRA_DIST += rdata/ch_3/a_1.cc
 EXTRA_DIST += rdata/ch_3/a_1.h
 EXTRA_DIST += rdata/generic/cname_5.cc
 EXTRA_DIST += rdata/generic/cname_5.h
+EXTRA_DIST += rdata/generic/detail/char_string.cc
+EXTRA_DIST += rdata/generic/detail/char_string.h
 EXTRA_DIST += rdata/generic/detail/nsec_bitmap.cc
 EXTRA_DIST += rdata/generic/detail/nsec_bitmap.h
 EXTRA_DIST += rdata/generic/detail/nsec3param_common.cc
@@ -123,6 +125,8 @@ libb10_dns___la_SOURCES += tsigrecord.h tsigrecord.cc
 libb10_dns___la_SOURCES += character_string.h character_string.cc
 libb10_dns___la_SOURCES += master_loader_callbacks.h
 libb10_dns___la_SOURCES += master_loader.h
+libb10_dns___la_SOURCES += rdata/generic/detail/char_string.h
+libb10_dns___la_SOURCES += rdata/generic/detail/char_string.cc
 libb10_dns___la_SOURCES += rdata/generic/detail/nsec_bitmap.h
 libb10_dns___la_SOURCES += rdata/generic/detail/nsec_bitmap.cc
 libb10_dns___la_SOURCES += rdata/generic/detail/nsec3param_common.cc

src/lib/dns/gen-rdatacode.py.in

@@ -32,7 +32,8 @@ import sys
 #
 # Example:
 #     new_rdata_factory_users = [('a', 'in'), ('a', 'ch'), ('soa', 'generic')]
-new_rdata_factory_users = []
+new_rdata_factory_users = [('aaaa', 'in'), ('txt', 'generic'),
+                           ('spf', 'generic')]
 
 re_typecode = re.compile('([\da-z]+)_(\d+)')
 classcode2txt = {}
@@ -126,6 +127,9 @@ class AbstractMessageRenderer;\n\n'''
     explicit ''' + type_utxt + '''(const std::string& type_str);
     ''' + type_utxt + '''(isc::util::InputBuffer& buffer, size_t rdata_len);
     ''' + type_utxt + '''(const ''' + type_utxt + '''& other);
+    ''' + type_utxt + '''(
+        MasterLexer& lexer, const Name* name,
+        MasterLoader::Options options, MasterLoaderCallbacks& callbacks);
     virtual std::string toText() const;
     virtual void toWire(isc::util::OutputBuffer& buffer) const;
     virtual void toWire(AbstractMessageRenderer& renderer) const;
@@ -213,17 +217,33 @@ def generate_rdatadef(file, basemtime):
     rdata_deffile.write(class_definitions)
     rdata_deffile.close()
 
-def generate_rdatahdr(file, declarations, basemtime):
+def generate_rdatahdr(file, heading, declarations, basemtime):
     if not need_generate(file, basemtime):
         print('skip generating ' + file);
         return
+    heading += '''
+#ifndef DNS_RDATACLASS_H
+#define DNS_RDATACLASS_H 1
+
+#include <dns/master_loader.h>
+
+namespace isc {
+namespace dns {
+class Name;
+class MasterLexer;
+class MasterLoaderCallbacks;
+}
+}
+'''
     declarations += '''
+#endif // DNS_RDATACLASS_H
+
 // Local Variables:
 // mode: c++
 // End:
 '''
     rdata_header = open(file, 'w')
-    rdata_header.write(heading_txt)
+    rdata_header.write(heading)
     rdata_header.write(declarations)
     rdata_header.close()
 
@@ -320,8 +340,8 @@ if __name__ == "__main__":
     try:
         import_definitions(classcode2txt, typecode2txt, typeandclass)
         generate_rdatadef('@builddir@/rdataclass.cc', rdatadef_mtime)
-        generate_rdatahdr('@builddir@/rdataclass.h', rdata_declarations,
-                          rdatahdr_mtime)
+        generate_rdatahdr('@builddir@/rdataclass.h', heading_txt,
+                          rdata_declarations, rdatahdr_mtime)
         generate_typeclasscode('rrtype', rdatahdr_mtime, typecode2txt, 'Type')
         generate_typeclasscode('rrclass', classdir_mtime,
                                classcode2txt, 'Class')

src/lib/dns/master_lexer.cc

@@ -36,7 +36,7 @@ using namespace master_lexer_internal;
 
 
 struct MasterLexer::MasterLexerImpl {
-    MasterLexerImpl() : source_(NULL), token_(Token::NOT_STARTED),
+    MasterLexerImpl() : source_(NULL), token_(MasterToken::NOT_STARTED),
                         paren_count_(0), last_was_eol_(false),
                         has_previous_(false),
                         previous_paren_count_(0),
@@ -82,7 +82,7 @@ struct MasterLexer::MasterLexerImpl {
 
     std::vector<InputSourcePtr> sources_;
     InputSource* source_;       // current source (NULL if sources_ is empty)
-    Token token_;               // currently recognized token (set by a state)
+    MasterToken token_;         // currently recognized token (set by a state)
     std::vector<char> data_;    // placeholder for string data
 
     // These are used in states, and defined here only as a placeholder.
@@ -165,9 +165,8 @@ MasterLexer::getSourceLine() const {
     return (impl_->sources_.back()->getCurrentLine());
 }
 
-const MasterLexer::Token&
+const MasterToken&
 MasterLexer::getNextToken(Options options) {
-    // If the source is not available
     if (impl_->source_ == NULL) {
         isc_throw(isc::InvalidOperation, "No source to read tokens from");
     }
@@ -178,7 +177,7 @@ MasterLexer::getNextToken(Options options) {
     impl_->has_previous_ = true;
     // Reset the token now. This is to check a token was actually produced.
     // This is debugging aid.
-    impl_->token_ = Token(Token::NO_TOKEN_PRODUCED);
+    impl_->token_ = MasterToken(MasterToken::NO_TOKEN_PRODUCED);
     // And get the token
 
     // This actually handles EOF internally too.
@@ -188,8 +187,62 @@ MasterLexer::getNextToken(Options options) {
     }
     // Make sure a token was produced. Since this Can Not Happen, we assert
     // here instead of throwing.
-    assert(impl_->token_.getType() != Token::ERROR ||
-           impl_->token_.getErrorCode() != Token::NO_TOKEN_PRODUCED);
+    assert(impl_->token_.getType() != MasterToken::ERROR ||
+           impl_->token_.getErrorCode() != MasterToken::NO_TOKEN_PRODUCED);
+    return (impl_->token_);
+}
+
+namespace {
+inline MasterLexer::Options
+optionsForTokenType(MasterToken::Type expect) {
+    switch (expect) {
+    case MasterToken::STRING:
+        return (MasterLexer::NONE);
+    case MasterToken::QSTRING:
+        return (MasterLexer::QSTRING);
+    case MasterToken::NUMBER:
+        return (MasterLexer::NUMBER);
+    default:
+        isc_throw(InvalidParameter,
+                  "expected type for getNextToken not supported: " << expect);
+    }
+}
+}
+
+const MasterToken&
+MasterLexer::getNextToken(MasterToken::Type expect, bool eol_ok) {
+    // Get the next token, specifying an appropriate option corresponding to
+    // the expected type.  The result should be set in impl_->token_.
+    getNextToken(optionsForTokenType(expect));
+
+    if (impl_->token_.getType() == MasterToken::ERROR) {
+        if (impl_->token_.getErrorCode() == MasterToken::NUMBER_OUT_OF_RANGE) {
+            ungetToken();
+        }
+        throw LexerError(__FILE__, __LINE__, impl_->token_);
+    }
+
+    const bool is_eol_like =
+        (impl_->token_.getType() == MasterToken::END_OF_LINE ||
+         impl_->token_.getType() == MasterToken::END_OF_FILE);
+    if (eol_ok && is_eol_like) {
+        return (impl_->token_);
+    }
+    if (impl_->token_.getType() == MasterToken::STRING &&
+        expect == MasterToken::QSTRING) {
+        return (impl_->token_);
+    }
+    if (impl_->token_.getType() != expect) {
+        ungetToken();
+        if (is_eol_like) {
+            throw LexerError(__FILE__, __LINE__,
+                             MasterToken(MasterToken::UNEXPECTED_END));
+        }
+        assert(expect == MasterToken::NUMBER);
+        throw LexerError(__FILE__, __LINE__,
+                         MasterToken(MasterToken::BAD_NUMBER));
+    }
+
     return (impl_->token_);
 }
 
@@ -212,16 +265,17 @@ const char* const error_text[] = {
     "unexpected end of input",  // UNEXPECTED_END
     "unbalanced quotes",        // UNBALANCED_QUOTES
     "no token produced",        // NO_TOKEN_PRODUCED
-    "number out of range"       // NUMBER_OUT_OF_RANGE
+    "number out of range",      // NUMBER_OUT_OF_RANGE
+    "not a valid number"        // BAD_NUMBER
 };
 const size_t error_text_max_count = sizeof(error_text) / sizeof(error_text[0]);
 } // end unnamed namespace
 
 std::string
-MasterLexer::Token::getErrorText() const {
+MasterToken::getErrorText() const {
     if (type_ != ERROR) {
         isc_throw(InvalidOperation,
-                  "Token::getErrorText() for non error type");
+                  "MasterToken::getErrorText() for non error type");
     }
 
     // The class integrity ensures the following:
@@ -234,14 +288,12 @@ namespace master_lexer_internal {
 // Note that these need to be defined here so that they can refer to
 // the details of MasterLexerImpl.
 
-typedef MasterLexer::Token Token; // convenience shortcut
-
 bool
 State::wasLastEOL(const MasterLexer& lexer) const {
     return (lexer.impl_->last_was_eol_);
 }
 
-const MasterLexer::Token&
+const MasterToken&
 State::getToken(const MasterLexer& lexer) const {
     return (lexer.impl_->token_);
 }
@@ -271,7 +323,7 @@ public:
         if (c != '\n') {
             getLexerImpl(lexer)->source_->ungetChar();
         }
-        getLexerImpl(lexer)->token_ = Token(Token::END_OF_LINE);
+        getLexerImpl(lexer)->token_ = MasterToken(MasterToken::END_OF_LINE);
         getLexerImpl(lexer)->last_was_eol_ = true;
     }
 };
@@ -342,24 +394,24 @@ State::start(MasterLexer& lexer, MasterLexer::Options options) {
         if (c == InputSource::END_OF_STREAM) {
             lexerimpl.last_was_eol_ = false;
             if (paren_count != 0) {
-                lexerimpl.token_ = Token(Token::UNBALANCED_PAREN);
+                lexerimpl.token_ = MasterToken(MasterToken::UNBALANCED_PAREN);
                 paren_count = 0; // reset to 0; this helps in lenient mode.
                 return (NULL);
             }
-            lexerimpl.token_ = Token(Token::END_OF_FILE);
+            lexerimpl.token_ = MasterToken(MasterToken::END_OF_FILE);
             return (NULL);
         } else if (c == ' ' || c == '\t') {
             // If requested and we are not in (), recognize the initial space.
             if (lexerimpl.last_was_eol_ && paren_count == 0 &&
                 (options & MasterLexer::INITIAL_WS) != 0) {
                 lexerimpl.last_was_eol_ = false;
-                lexerimpl.token_ = Token(Token::INITIAL_WS);
+                lexerimpl.token_ = MasterToken(MasterToken::INITIAL_WS);
                 return (NULL);
             }
         } else if (c == '\n') {
             lexerimpl.last_was_eol_ = true;
             if (paren_count == 0) { // we don't recognize EOL if we are in ()
-                lexerimpl.token_ = Token(Token::END_OF_LINE);
+                lexerimpl.token_ = MasterToken(MasterToken::END_OF_LINE);
                 return (NULL);
             }
         } else if (c == '\r') {
@@ -375,7 +427,7 @@ State::start(MasterLexer& lexer, MasterLexer::Options options) {
         } else if (c == ')') {
             lexerimpl.last_was_eol_ = false;
             if (paren_count == 0) {
-                lexerimpl.token_ = Token(Token::UNBALANCED_PAREN);
+                lexerimpl.token_ = MasterToken(MasterToken::UNBALANCED_PAREN);
                 return (NULL);
             }
             --paren_count;
@@ -406,8 +458,11 @@ String::handle(MasterLexer& lexer) const {
 
         if (getLexerImpl(lexer)->isTokenEnd(c, escaped)) {
             getLexerImpl(lexer)->source_->ungetChar();
+            // make sure it nul-terminated as a c-str (excluded from token
+            // data).
+            data.push_back('\0');
             getLexerImpl(lexer)->token_ =
-                MasterLexer::Token(&data.at(0), data.size());
+                MasterToken(&data.at(0), data.size() - 1);
             return;
         }
         escaped = (c == '\\' && !escaped);
@@ -417,7 +472,7 @@ String::handle(MasterLexer& lexer) const {
 
 void
 QString::handle(MasterLexer& lexer) const {
-    MasterLexer::Token& token = getLexerImpl(lexer)->token_;
+    MasterToken& token = getLexerImpl(lexer)->token_;
     std::vector<char>& data = getLexerImpl(lexer)->data_;
     data.clear();
 
@@ -425,7 +480,7 @@ QString::handle(MasterLexer& lexer) const {
     while (true) {
         const int c = getLexerImpl(lexer)->source_->getChar();
         if (c == InputSource::END_OF_STREAM) {
-            token = Token(Token::UNEXPECTED_END);
+            token = MasterToken(MasterToken::UNEXPECTED_END);
             return;
         } else if (c == '"') {
             if (escaped) {
@@ -434,12 +489,15 @@ QString::handle(MasterLexer& lexer) const {
                 escaped = false;
                 data.back() = '"';
             } else {
-                token = MasterLexer::Token(&data.at(0), data.size(), true);
+                // make sure it nul-terminated as a c-str (excluded from token
+                // data).  This also simplifies the case of an empty string.
+                data.push_back('\0');
+                token = MasterToken(&data.at(0), data.size() - 1, true);
                 return;
             }
         } else if (c == '\n' && !escaped) {
             getLexerImpl(lexer)->source_->ungetChar();
-            token = Token(Token::UNBALANCED_QUOTES);
+            token = MasterToken(MasterToken::UNBALANCED_QUOTES);
             return;
         } else {
             escaped = (c == '\\' && !escaped);
@@ -450,7 +508,7 @@ QString::handle(MasterLexer& lexer) const {
 
 void
 Number::handle(MasterLexer& lexer) const {
-    MasterLexer::Token& token = getLexerImpl(lexer)->token_;
+    MasterToken& token = getLexerImpl(lexer)->token_;
 
     // It may yet turn out to be a string, so we first
     // collect all the data
@@ -464,21 +522,21 @@ Number::handle(MasterLexer& lexer) const {
             getLexerImpl(lexer)->source_->getChar(), escaped);
         if (getLexerImpl(lexer)->isTokenEnd(c, escaped)) {
             getLexerImpl(lexer)->source_->ungetChar();
+            // We need to close the string whether it's digits-only (for
+            // lexical_cast) or not (see String::handle()).
+            data.push_back('\0');
             if (digits_only) {
-                // Close the string for lexical_cast
-                data.push_back('\0');
                 try {
                     const uint32_t number32 =
                         boost::lexical_cast<uint32_t, const char*>(&data[0]);
-                    token = MasterLexer::Token(number32);
+                    token = MasterToken(number32);
                 } catch (const boost::bad_lexical_cast&) {
                     // Since we already know we have only digits,
                     // range should be the only possible problem.
-                    token = Token(Token::NUMBER_OUT_OF_RANGE);
+                    token = MasterToken(MasterToken::NUMBER_OUT_OF_RANGE);
                 }
             } else {
-                token = MasterLexer::Token(&data.at(0),
-                                           data.size());
+                token = MasterToken(&data.at(0), data.size() - 1);
             }
             return;
         }

src/lib/dns/master_lexer.h

@@ -28,225 +28,6 @@ namespace master_lexer_internal {
 class State;
 }
 
-/// \brief Tokenizer for parsing DNS master files.
-///
-/// The \c MasterLexer class provides tokenize interfaces for parsing DNS
-/// master files.  It understands some special rules of master files as
-/// defined in RFC 1035, such as comments, character escaping, and multi-line
-/// data, and provides the user application with the actual data in a
-/// more convenient form such as a std::string object.
-///
-/// In order to support the $INCLUDE notation, this class is designed to be
-/// able to operate on multiple files or input streams in the nested way.
-/// The \c pushSource() and \c popSource() methods correspond to the push
-/// and pop operations.
-///
-/// While this class is public, it is less likely to be used by normal
-/// applications; it's mainly expected to be used within this library,
-/// specifically by the \c MasterLoader class and \c Rdata implementation
-/// classes.
-///
-/// \note The error handling policy of this class is slightly different from
-/// that of other classes of this library.  We generally throw an exception
-/// for an invalid input, whether it's more likely to be a program error or
-/// a "user error", which means an invalid input that comes from outside of
-/// the library.  But, this class returns an error code for some certain
-/// types of user errors instead of throwing an exception.  Such cases include
-/// a syntax error identified by the lexer or a misspelled file name that
-/// causes a system error at the time of open.  This is based on the assumption
-/// that the main user of this class is a parser of master files, where
-/// we want to give an option to ignore some non fatal errors and continue
-/// the parsing.  This will be useful if it just performs overall error
-/// checks on a master file.  When the (immediate) caller needs to do explicit
-/// error handling, exceptions are not that a useful tool for error reporting
-/// because we cannot separate the normal and error cases anyway, which would
-/// be one major advantage when we use exceptions.  And, exceptions are
-/// generally more expensive, either when it happens or just by being able
-/// to handle with \c try and \c catch (depending on the underlying
-/// implementation of the exception handling).  For these reasons, some of
-/// this class does not throw for an error that would be reported as an
-/// exception in other classes.
-class MasterLexer {
-    friend class master_lexer_internal::State;
-public:
-    /// \brief Exception thrown when we fail to read from the input
-    /// stream or file.
-    struct ReadError : public Unexpected {
-        ReadError(const char* file, size_t line, const char* what) :
-            Unexpected(file, line, what)
-        {}
-    };
-
-    class Token;       // we define it separately for better readability
-
-    /// \brief Options for getNextToken.
-    ///
-    /// A compound option, indicating multiple options are set, can be
-    /// specified using the logical OR operator (operator|()).
-    enum Options {
-        NONE = 0,               ///< No option
-        INITIAL_WS = 1, ///< recognize begin-of-line spaces after an
-                        ///< end-of-line
-        QSTRING = 2,    ///< recognize quoted string
-        NUMBER = 4   ///< recognize numeric text as integer
-    };
-
-    /// \brief The constructor.
-    ///
-    /// \throw std::bad_alloc Internal resource allocation fails (rare case).
-    MasterLexer();
-
-    /// \brief The destructor.
-    ///
-    /// It internally closes any remaining input sources.
-    ~MasterLexer();
-
-    /// \brief Open a file and make it the current input source of MasterLexer.
-    ///
-    /// The opened file can be explicitly closed by the \c popSource() method;
-    /// if \c popSource() is not called within the lifetime of the
-    /// \c MasterLexer, it will be closed in the destructor.
-    ///
-    /// In the case possible system errors in opening the file (most likely
-    /// because of specifying a non-existent or unreadable file), it returns
-    /// false, and if the optional \c error parameter is non NULL, it will be
-    /// set to a description of the error (any existing content of the string
-    /// will be discarded).  If opening the file succeeds, the given
-    /// \c error parameter will be intact.
-    ///
-    /// Note that this method has two styles of error reporting: one by
-    /// returning \c false (and setting \c error optionally) and the other
-    /// by throwing an exception.  See the note for the class description
-    /// about the distinction.
-    ///
-    /// \throw InvalidParameter filename is NULL
-    /// \param filename A non NULL string specifying a master file
-    /// \param error If non null, a placeholder to set error description in
-    /// case of failure.
-    ///
-    /// \return true if pushing the file succeeds; false otherwise.
-    bool pushSource(const char* filename, std::string* error = NULL);
-
-    /// \brief Make the given stream the current input source of MasterLexer.
-    ///
-    /// The caller still holds the ownership of the passed stream; it's the
-    /// caller's responsibility to keep it valid as long as it's used in
-    /// \c MasterLexer or to release any resource for the stream after that.
-    /// The caller can explicitly tell \c MasterLexer to stop using the
-    /// stream by calling the \c popSource() method.
-    ///
-    /// \param input An input stream object that produces textual
-    /// representation of DNS RRs.
-    void pushSource(std::istream& input);
-
-    /// \brief Stop using the most recently opened input source (file or
-    /// stream).
-    ///
-    /// If it's a file, the previously opened file will be closed internally.
-    /// If it's a stream, \c MasterLexer will simply stop using
-    /// the stream; the caller can assume it will be never used in
-    /// \c MasterLexer thereafter.
-    ///
-    /// This method must not be called when there is no source pushed for
-    /// \c MasterLexer.  This method is otherwise exception free.
-    ///
-    /// \throw isc::InvalidOperation Called with no pushed source.
-    void popSource();
-
-    /// \brief Return the name of the current input source name.
-    ///
-    /// If it's a file, it will be the C string given at the corresponding
-    /// \c pushSource() call, that is, its filename.  If it's a stream, it will
-    /// be formatted as \c "stream-%p" where \c %p is hex representation
-    /// of the address of the stream object.
-    ///
-    /// If there is no opened source at the time of the call, this method
-    /// returns an empty string.
-    ///
-    /// \throw std::bad_alloc Resource allocation failed for string
-    /// construction (rare case)
-    ///
-    /// \return A string representation of the current source (see the
-    /// description)
-    std::string getSourceName() const;
-
-    /// \brief Return the input source line number.
-    ///
-    /// If there is an opened source, the return value will be a non-0
-    /// integer indicating the line number of the current source where
-    /// the \c MasterLexer is currently working.  The expected usage of
-    /// this value is to print a helpful error message when parsing fails
-    /// by specifically identifying the position of the error.
-    ///
-    /// If there is no opened source at the time of the call, this method
-    /// returns 0.
-    ///
-    /// \throw None
-    ///
-    /// \return The current line number of the source (see the description)
-    size_t getSourceLine() const;
-
-    /// \brief Parse and return another token from the input.
-    ///
-    /// It reads a bit of the last opened source and produces another token
-    /// found in it.
-    ///
-    /// This method does not provide the strong exception guarantee. Generally,
-    /// if it throws, the object should not be used any more and should be
-    /// discarded. It was decided all the exceptions thrown from here are
-    /// serious enough that aborting the loading process is the only reasonable
-    /// recovery anyway, so the strong exception guarantee is not needed.
-    ///
-    /// \param options The options can be used to modify the tokenization.
-    ///     The method can be made reporting things which are usually ignored
-    ///     by this parameter. Multiple options can be passed at once by
-    ///     bitwise or (eg. option1 | option 2). See description of available
-    ///     options.
-    /// \return Next token found in the input. Note that the token refers to
-    ///     some internal data in the lexer. It is valid only until
-    ///     getNextToken or ungetToken is called. Also, the token becomes
-    ///     invalid when the lexer is destroyed.
-    /// \throw isc::InvalidOperation in case the source is not available. This
-    ///     may mean the pushSource() has not been called yet, or that the
-    ///     current source has been read past the end.
-    /// \throw ReadError in case there's problem reading from the underlying
-    ///     source (eg. I/O error in the file on the disk).
-    /// \throw std::bad_alloc in case allocation of some internal resources
-    ///     or the token fail.
-    const Token& getNextToken(Options options = NONE);
-
-    /// \brief Return the last token back to the lexer.
-    ///
-    /// The method undoes the lasts call to getNextToken(). If you call the
-    /// getNextToken() again with the same options, it'll return the same
-    /// token. If the options are different, it may return a different token,
-    /// but it acts as if the previous getNextToken() was never called.
-    ///
-    /// It is possible to return only one token back in time (you can't call
-    /// ungetToken() twice in a row without calling getNextToken() in between
-    /// successfully).
-    ///
-    /// It does not work after change of source (by pushSource or popSource).
-    ///
-    /// \throw isc::InvalidOperation If called second time in a row or if
-    ///     getNextToken() was not called since the last change of the source.
-    void ungetToken();
-
-private:
-    struct MasterLexerImpl;
-    MasterLexerImpl* impl_;
-};
-
-/// \brief Operator to combine \c MasterLexer options
-///
-/// This is a trivial shortcut so that compound options can be specified
-/// in an intuitive way.
-inline MasterLexer::Options
-operator|(MasterLexer::Options o1, MasterLexer::Options o2) {
-    return (static_cast<MasterLexer::Options>(
-                static_cast<unsigned>(o1) | static_cast<unsigned>(o2)));
-}
-
 /// \brief Tokens for \c MasterLexer
 ///
 /// This is a simple value-class encapsulating a type of a lexer token and
@@ -261,7 +42,7 @@ operator|(MasterLexer::Options o1, MasterLexer::Options o2) {
 /// (using the default version of copy constructor and assignment operator),
 /// but it's mainly for internal implementation convenience.  Applications will
 /// simply refer to Token object as a reference via the \c MasterLexer class.
-class MasterLexer::Token {
+class MasterToken {
 public:
     /// \brief Enumeration for token types
     ///
@@ -293,6 +74,7 @@ public:
         NO_TOKEN_PRODUCED, ///< No token was produced. This means programmer
                            /// error and should never get out of the lexer.
         NUMBER_OUT_OF_RANGE, ///< Number was out of range
+        BAD_NUMBER,    ///< Number is expected but not recognized
         MAX_ERROR_CODE ///< Max integer corresponding to valid error codes.
                        /// (excluding this one). Mainly for internal use.
     };
@@ -308,6 +90,13 @@ public:
     /// the region.  On the other hand, it is not ensured that the string
     /// is nul-terminated.  So the usual string manipulation API may not work
     /// as expected.
+    ///
+    /// The `MasterLexer` implementation ensures that there are at least
+    /// len + 1 bytes of valid memory region starting from beg, and that
+    /// beg[len] is \0.  This means the application can use the bytes as a
+    /// validly nul-terminated C string if there is no intermediate nul
+    /// character.  Note also that due to this property beg is always non
+    /// NULL; for an empty string len will be set to 0 and beg[0] is \0.
     struct StringRegion {
         const char* beg;        ///< The start address of the string
         size_t len;             ///< The length of the string in bytes
@@ -318,7 +107,7 @@ public:
     /// \throw InvalidParameter A value type token is specified.
     /// \param type The type of the token.  It must indicate a non-value
     /// type (not larger than \c NOVALUE_TYPE_MAX).
-    explicit Token(Type type) : type_(type) {
+    explicit MasterToken(Type type) : type_(type) {
         if (type > NOVALUE_TYPE_MAX) {
             isc_throw(InvalidParameter, "Token per-type constructor "
                       "called with invalid type: " << type);
@@ -340,7 +129,7 @@ public:
     /// \param str_beg The start address of the string
     /// \param str_len The size of the string in bytes
     /// \param quoted true if it's a quoted string; false otherwise.
-    Token(const char* str_beg, size_t str_len, bool quoted = false) :
+    MasterToken(const char* str_beg, size_t str_len, bool quoted = false) :
         type_(quoted ? QSTRING : STRING)
     {
         val_.str_region_.beg = str_beg;
@@ -351,7 +140,7 @@ public:
     ///
     /// \brief number An unsigned 32-bit integer corresponding to the token
     /// value.
-    explicit Token(uint32_t number) : type_(NUMBER) {
+    explicit MasterToken(uint32_t number) : type_(NUMBER) {
         val_.number_ = number;
     }
 
@@ -359,7 +148,7 @@ public:
     ///
     /// \throw InvalidParameter Invalid error code value is specified.
     /// \brief error_code A pre-defined constant of \c ErrorCode.
-    explicit Token(ErrorCode error_code) : type_(ERROR) {
+    explicit MasterToken(ErrorCode error_code) : type_(ERROR) {
         if (!(error_code < MAX_ERROR_CODE)) {
             isc_throw(InvalidParameter, "Invalid master lexer error code: "
                       << error_code);
@@ -476,6 +265,310 @@ private:
     } val_;
 };
 
+/// \brief Tokenizer for parsing DNS master files.
+///
+/// The \c MasterLexer class provides tokenize interfaces for parsing DNS
+/// master files.  It understands some special rules of master files as
+/// defined in RFC 1035, such as comments, character escaping, and multi-line
+/// data, and provides the user application with the actual data in a
+/// more convenient form such as a std::string object.
+///
+/// In order to support the $INCLUDE notation, this class is designed to be
+/// able to operate on multiple files or input streams in the nested way.
+/// The \c pushSource() and \c popSource() methods correspond to the push
+/// and pop operations.
+///
+/// While this class is public, it is less likely to be used by normal
+/// applications; it's mainly expected to be used within this library,
+/// specifically by the \c MasterLoader class and \c Rdata implementation
+/// classes.
+///
+/// \note The error handling policy of this class is slightly different from
+/// that of other classes of this library.  We generally throw an exception
+/// for an invalid input, whether it's more likely to be a program error or
+/// a "user error", which means an invalid input that comes from outside of
+/// the library.  But, this class returns an error code for some certain
+/// types of user errors instead of throwing an exception.  Such cases include
+/// a syntax error identified by the lexer or a misspelled file name that
+/// causes a system error at the time of open.  This is based on the assumption
+/// that the main user of this class is a parser of master files, where
+/// we want to give an option to ignore some non fatal errors and continue
+/// the parsing.  This will be useful if it just performs overall error
+/// checks on a master file.  When the (immediate) caller needs to do explicit
+/// error handling, exceptions are not that a useful tool for error reporting
+/// because we cannot separate the normal and error cases anyway, which would
+/// be one major advantage when we use exceptions.  And, exceptions are
+/// generally more expensive, either when it happens or just by being able
+/// to handle with \c try and \c catch (depending on the underlying
+/// implementation of the exception handling).  For these reasons, some of
+/// this class does not throw for an error that would be reported as an
+/// exception in other classes.
+class MasterLexer {
+    friend class master_lexer_internal::State;
+public:
+    /// \brief Exception thrown when we fail to read from the input
+    /// stream or file.
+    class ReadError : public Unexpected {
+    public:
+        ReadError(const char* file, size_t line, const char* what) :
+            Unexpected(file, line, what)
+        {}
+    };
+
+    /// \brief Exception thrown from a wrapper version of
+    /// \c MasterLexer::getNextToken() for non fatal errors.
+    ///
+    /// See the method description for more details.
+    ///
+    /// The \c token_ member variable (read-only) is set to a \c MasterToken
+    /// object of type ERROR indicating the reason for the error.
+    class LexerError : public Exception {
+    public:
+        LexerError(const char* file, size_t line, MasterToken error_token) :
+            Exception(file, line, error_token.getErrorText().c_str()),
+            token_(error_token)
+        {}
+        const MasterToken token_;
+    };
+
+    /// \brief Options for getNextToken.
+    ///
+    /// A compound option, indicating multiple options are set, can be
+    /// specified using the logical OR operator (operator|()).
+    enum Options {
+        NONE = 0,               ///< No option
+        INITIAL_WS = 1, ///< recognize begin-of-line spaces after an
+                        ///< end-of-line
+        QSTRING = 2,    ///< recognize quoted string
+        NUMBER = 4   ///< recognize numeric text as integer
+    };
+
+    /// \brief The constructor.
+    ///
+    /// \throw std::bad_alloc Internal resource allocation fails (rare case).
+    MasterLexer();
+
+    /// \brief The destructor.
+    ///
+    /// It internally closes any remaining input sources.
+    ~MasterLexer();
+
+    /// \brief Open a file and make it the current input source of MasterLexer.
+    ///
+    /// The opened file can be explicitly closed by the \c popSource() method;
+    /// if \c popSource() is not called within the lifetime of the
+    /// \c MasterLexer, it will be closed in the destructor.
+    ///
+    /// In the case possible system errors in opening the file (most likely
+    /// because of specifying a non-existent or unreadable file), it returns
+    /// false, and if the optional \c error parameter is non NULL, it will be
+    /// set to a description of the error (any existing content of the string
+    /// will be discarded).  If opening the file succeeds, the given
+    /// \c error parameter will be intact.
+    ///
+    /// Note that this method has two styles of error reporting: one by
+    /// returning \c false (and setting \c error optionally) and the other
+    /// by throwing an exception.  See the note for the class description
+    /// about the distinction.
+    ///
+    /// \throw InvalidParameter filename is NULL
+    /// \param filename A non NULL string specifying a master file
+    /// \param error If non null, a placeholder to set error description in
+    /// case of failure.
+    ///
+    /// \return true if pushing the file succeeds; false otherwise.
+    bool pushSource(const char* filename, std::string* error = NULL);
+
+    /// \brief Make the given stream the current input source of MasterLexer.
+    ///
+    /// The caller still holds the ownership of the passed stream; it's the
+    /// caller's responsibility to keep it valid as long as it's used in
+    /// \c MasterLexer or to release any resource for the stream after that.
+    /// The caller can explicitly tell \c MasterLexer to stop using the
+    /// stream by calling the \c popSource() method.
+    ///
+    /// \param input An input stream object that produces textual
+    /// representation of DNS RRs.
+    void pushSource(std::istream& input);
+
+    /// \brief Stop using the most recently opened input source (file or
+    /// stream).
+    ///
+    /// If it's a file, the previously opened file will be closed internally.
+    /// If it's a stream, \c MasterLexer will simply stop using
+    /// the stream; the caller can assume it will be never used in
+    /// \c MasterLexer thereafter.
+    ///
+    /// This method must not be called when there is no source pushed for
+    /// \c MasterLexer.  This method is otherwise exception free.
+    ///
+    /// \throw isc::InvalidOperation Called with no pushed source.
+    void popSource();
+
+    /// \brief Return the name of the current input source name.
+    ///
+    /// If it's a file, it will be the C string given at the corresponding
+    /// \c pushSource() call, that is, its filename.  If it's a stream, it will
+    /// be formatted as \c "stream-%p" where \c %p is hex representation
+    /// of the address of the stream object.
+    ///
+    /// If there is no opened source at the time of the call, this method
+    /// returns an empty string.
+    ///
+    /// \throw std::bad_alloc Resource allocation failed for string
+    /// construction (rare case)
+    ///
+    /// \return A string representation of the current source (see the
+    /// description)
+    std::string getSourceName() const;
+
+    /// \brief Return the input source line number.
+    ///
+    /// If there is an opened source, the return value will be a non-0
+    /// integer indicating the line number of the current source where
+    /// the \c MasterLexer is currently working.  The expected usage of
+    /// this value is to print a helpful error message when parsing fails
+    /// by specifically identifying the position of the error.
+    ///
+    /// If there is no opened source at the time of the call, this method
+    /// returns 0.
+    ///
+    /// \throw None
+    ///
+    /// \return The current line number of the source (see the description)
+    size_t getSourceLine() const;
+
+    /// \brief Parse and return another token from the input.
+    ///
+    /// It reads a bit of the last opened source and produces another token
+    /// found in it.
+    ///
+    /// This method does not provide the strong exception guarantee. Generally,
+    /// if it throws, the object should not be used any more and should be
+    /// discarded. It was decided all the exceptions thrown from here are
+    /// serious enough that aborting the loading process is the only reasonable
+    /// recovery anyway, so the strong exception guarantee is not needed.
+    ///
+    /// \param options The options can be used to modify the tokenization.
+    ///     The method can be made reporting things which are usually ignored
+    ///     by this parameter. Multiple options can be passed at once by
+    ///     bitwise or (eg. option1 | option 2). See description of available
+    ///     options.
+    /// \return Next token found in the input. Note that the token refers to
+    ///     some internal data in the lexer. It is valid only until
+    ///     getNextToken or ungetToken is called. Also, the token becomes
+    ///     invalid when the lexer is destroyed.
+    /// \throw isc::InvalidOperation in case the source is not available. This
+    ///     may mean the pushSource() has not been called yet, or that the
+    ///     current source has been read past the end.
+    /// \throw ReadError in case there's problem reading from the underlying
+    ///     source (eg. I/O error in the file on the disk).
+    /// \throw std::bad_alloc in case allocation of some internal resources
+    ///     or the token fail.
+    const MasterToken& getNextToken(Options options = NONE);
+
+    /// \brief Parse the input for the expected type of token.
+    ///
+    /// This method is a wrapper of the other version, customized for the case
+    /// where a particular type of token is expected as the next one.
+    /// More specifically, it's intended to be used to get tokens for RDATA
+    /// fields.  Since most RDATA types of fixed format, the token type is
+    /// often predictable and the method interface can be simplified.
+    ///
+    /// This method basically works as follows: it gets the type of the
+    /// expected token, calls the other version of \c getNextToken(Options),
+    /// and returns the token if it's of the expected type (due to the usage
+    /// assumption this should be normally the case).  There are some non
+    /// trivial details though:
+    ///
+    /// - If the expected type is MasterToken::QSTRING, both quoted and
+    ///   unquoted strings are recognized and returned.
+    /// - If the optional \c eol_ok parameter is \c true (very rare case),
+    ///   MasterToken::END_OF_LINE and MasterToken::END_OF_FILE are recognized
+    ///   and returned if they are found instead of the expected type of
+    ///   token.
+    /// - If the next token is not of the expected type (including the case
+    ///   a number is expected but it's out of range), ungetToken() is
+    ///   internally called so the caller can re-read that token.
+    /// - If other types or errors (such as unbalanced parentheses) are
+    ///   detected, the erroneous part isn't "ungotten"; the caller can
+    ///   continue parsing after that part.
+    ///
+    /// In some very rare cases where the RDATA has an optional trailing field,
+    /// the \c eol_ok parameter would be set to \c true.  This way the caller
+    /// can handle both cases (the field does or does not exist) by a single
+    /// call to this method.  In all other cases \c eol_ok should be set to
+    /// \c false, and that is the default and can be omitted.
+    ///
+    /// Unlike the other version of \c getNextToken(Options), this method
+    /// throws an exception of type \c LexerError for non fatal errors such as
+    /// broken syntax or encountering an unexpected type of token.  This way
+    /// the caller can write RDATA parser code without bothering to handle
+    /// errors for each field.  For example, pseudo parser code for MX RDATA
+    /// would look like this:
+    /// \code
+    ///    const uint32_t pref =
+    ///        lexer.getNextToken(MasterToken::NUMBER).getNumber();
+    ///    // check if pref is the uint16_t range; no other check is needed.
+    ///    const Name mx(lexer.getNextToken(MasterToken::STRING).getString());
+    /// \endcode
+    ///
+    /// In the case where \c LexerError exception is thrown, it's expected
+    /// to be handled comprehensively for the parser of the RDATA or at a
+    /// higher layer.  The \c token_ member variable of the corresponding
+    /// \c LexerError exception object stores a token of type
+    /// \c MasterToken::ERROR that indicates the reason for the error.
+    ///
+    /// Due to the specific intended usage of this method, only a subset
+    /// of \c MasterToken::Type values are acceptable for the \c expect
+    /// parameter: \c MasterToken::STRING, \c MasterToken::QSTRING, and
+    /// \c MasterToken::NUMBER.  Specifying other values will result in
+    /// an \c InvalidParameter exception.
+    ///
+    /// \throw InvalidParameter The expected token type is not allowed for
+    /// this method.
+    /// \throw LexerError The lexer finds non fatal error or it finds an
+    /// \throw other Anything the other version of getNextToken() can throw.
+    ///
+    /// \param expect Expected type of token.  Must be either STRING, QSTRING,
+    /// or NUMBER.
+    /// \param eol_ok \c true iff END_OF_LINE or END_OF_FILE is acceptable.
+    /// \return The expected type of token.
+    const MasterToken& getNextToken(MasterToken::Type expect,
+                                    bool eol_ok = false);
+
+    /// \brief Return the last token back to the lexer.
+    ///
+    /// The method undoes the lasts call to getNextToken(). If you call the
+    /// getNextToken() again with the same options, it'll return the same
+    /// token. If the options are different, it may return a different token,
+    /// but it acts as if the previous getNextToken() was never called.
+    ///
+    /// It is possible to return only one token back in time (you can't call
+    /// ungetToken() twice in a row without calling getNextToken() in between
+    /// successfully).
+    ///
+    /// It does not work after change of source (by pushSource or popSource).
+    ///
+    /// \throw isc::InvalidOperation If called second time in a row or if
+    ///     getNextToken() was not called since the last change of the source.
+    void ungetToken();
+
+private:
+    struct MasterLexerImpl;
+    MasterLexerImpl* impl_;
+};
+
+/// \brief Operator to combine \c MasterLexer options
+///
+/// This is a trivial shortcut so that compound options can be specified
+/// in an intuitive way.
+inline MasterLexer::Options
+operator|(MasterLexer::Options o1, MasterLexer::Options o2) {
+    return (static_cast<MasterLexer::Options>(
+                static_cast<unsigned>(o1) | static_cast<unsigned>(o2)));
+}
+
 } // namespace dns
 } // namespace isc
 #endif  // MASTER_LEXER_H

src/lib/dns/master_lexer_state.h

@@ -43,10 +43,10 @@ namespace master_lexer_internal {
 /// state, so it makes more sense to separate the interface for the transition
 /// from the initial state.
 ///
-/// When an object of a specific state class completes the session, it
-/// normally sets the identified token in the lexer, and returns NULL;
-/// if more transition is necessary, it returns a pointer to the next state
-/// object.
+/// If the whole lexer transition is completed within start(), it sets the
+/// identified token and returns NULL; otherwise it returns a pointer to
+/// an object of a specific state class that completes the session
+/// on the call of handle().
 ///
 /// As is usual in the state design pattern, the \c State class is made
 /// a friend class of \c MasterLexer and can refer to its internal details.
@@ -119,7 +119,7 @@ public:
     /// purposes.
     ///@{
     bool wasLastEOL(const MasterLexer& lexer) const;
-    const MasterLexer::Token& getToken(const MasterLexer& lexer) const;
+    const MasterToken& getToken(const MasterLexer& lexer) const;
     size_t getParenCount(const MasterLexer& lexer) const;
     ///@}
 

src/lib/dns/python/tests/rdata_python_test.py

@@ -38,7 +38,7 @@ class RdataTest(unittest.TestCase):
         self.assertRaises(InvalidRdataText, Rdata, RRType("A"), RRClass("IN"),
                           "Invalid Rdata Text")
         self.assertRaises(CharStringTooLong, Rdata, RRType("TXT"),
-                          RRClass("IN"), ' ' * 256)
+                          RRClass("IN"), 'x' * 256)
         self.assertRaises(InvalidRdataLength, Rdata, RRType("TXT"),
                           RRClass("IN"), bytes(65536))
         self.assertRaises(DNSMessageFORMERR, Rdata, RRType("TXT"),

src/lib/dns/rdata.cc

@@ -12,6 +12,20 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
+#include <exceptions/exceptions.h>
+
+#include <util/buffer.h>
+
+#include <dns/name.h>
+#include <dns/messagerenderer.h>
+#include <dns/master_lexer.h>
+#include <dns/rdata.h>
+#include <dns/rrparamregistry.h>
+#include <dns/rrtype.h>
+
+#include <boost/lexical_cast.hpp>
+#include <boost/shared_ptr.hpp>
+
 #include <algorithm>
 #include <cctype>
 #include <string>
@@ -24,16 +38,6 @@
 #include <stdint.h>
 #include <string.h>
 
-#include <boost/lexical_cast.hpp>
-#include <boost/shared_ptr.hpp>
-
-#include <util/buffer.h>
-#include <dns/name.h>
-#include <dns/messagerenderer.h>
-#include <dns/rdata.h>
-#include <dns/rrparamregistry.h>
-#include <dns/rrtype.h>
-
 using namespace std;
 using boost::lexical_cast;
 using namespace isc::util;
@@ -81,23 +85,92 @@ createRdata(const RRType& rrtype, const RRClass& rrclass, const Rdata& source)
                                                        source));
 }
 
+namespace {
+void
+fromtextError(bool& error_issued, const MasterLexer& lexer,
+              MasterLoaderCallbacks& callbacks,
+              const MasterToken* token, const char* reason)
+{
+    // Don't be too noisy if there are many issues for single RDATA
+    if (error_issued) {
+        return;
+    }
+    error_issued = true;
+
+    if (token == NULL) {
+        callbacks.error(lexer.getSourceName(), lexer.getSourceLine(),
+                        "createRdata from text failed: " + string(reason));
+        return;
+    }
+
+    switch (token->getType()) {
+    case MasterToken::STRING:
+    case MasterToken::QSTRING:
+        callbacks.error(lexer.getSourceName(), lexer.getSourceLine(),
+                        "createRdata from text failed near '" +
+                        token->getString() + "': " + string(reason));
+        break;
+    case MasterToken::ERROR:
+        callbacks.error(lexer.getSourceName(), lexer.getSourceLine(),
+                        "createRdata from text failed: " +
+                        token->getErrorText());
+        break;
+    default:
+        // This case shouldn't happen based on how we use MasterLexer in
+        // createRdata(), so we could assert() that here.  But since it
+        // depends on detailed behavior of other classes, we treat the case
+        // in a bit less harsh way.
+        isc_throw(Unexpected, "bug: createRdata() saw unexpected token type");
+    }
+}
+}
+
 RdataPtr
 createRdata(const RRType& rrtype, const RRClass& rrclass,
             MasterLexer& lexer, const Name* origin,
             MasterLoader::Options options,
             MasterLoaderCallbacks& callbacks)
 {
-    RdataPtr ret;
+    RdataPtr rdata;
 
+    bool error_issued = false;
     try {
-        ret = RRParamRegistry::getRegistry().createRdata(rrtype, rrclass,
-                                                         lexer, origin,
-                                                         options, callbacks);
-    } catch (...) {
-        // ret is NULL here.
+        rdata = RRParamRegistry::getRegistry().createRdata(
+            rrtype, rrclass, lexer, origin, options, callbacks);
+    } catch (const MasterLexer::LexerError& error) {
+        fromtextError(error_issued, lexer, callbacks, &error.token_, "");
+    } catch (const Exception& ex) {
+        // Catching all isc::Exception is too broad, but right now we don't
+        // have better granularity.  When we complete #2518 we can make this
+        // finer.
+        fromtextError(error_issued, lexer, callbacks, NULL, ex.what());
     }
+    // Other exceptions mean a serious implementation bug or fatal system
+    // error; it doesn't make sense to catch and try to recover from them
+    // here.  Just propagate.
+
+    // Consume to end of line / file.
+    // Call callback via fromtextError once if there was an error.
+    do {
+        const MasterToken& token = lexer.getNextToken();
+        switch (token.getType()) {
+        case MasterToken::END_OF_LINE:
+            return (rdata);
+        case MasterToken::END_OF_FILE:
+            callbacks.warning(lexer.getSourceName(), lexer.getSourceLine(),
+                              "file does not end with newline");
+            return (rdata);
+        default:
+            rdata.reset();      // we'll return NULL
+            fromtextError(error_issued, lexer, callbacks, &token,
+                          "extra input text");
+            // Continue until we see EOL or EOF
+        }
+    } while (true);
 
-    return (ret);
+    // We shouldn't reach here
+    assert(false);
+    return (RdataPtr()); // add explicit return to silence some compilers
 }
 
 int
@@ -211,9 +284,10 @@ Generic::Generic(MasterLexer& lexer, const Name*,
     std::string s;
 
     while (true) {
-        const MasterLexer::Token& token = lexer.getNextToken();
-        if ((token.getType() == MasterLexer::Token::END_OF_FILE) ||
-            (token.getType() == MasterLexer::Token::END_OF_LINE)) {
+        const MasterToken& token = lexer.getNextToken();
+        if ((token.getType() == MasterToken::END_OF_FILE) ||
+            (token.getType() == MasterToken::END_OF_LINE)) {
+            lexer.ungetToken(); // let the upper layer handle the end-of token
             break;
         }
 

src/lib/dns/rdata.h

@@ -485,8 +485,47 @@ RdataPtr createRdata(const RRType& rrtype, const RRClass& rrclass,
 RdataPtr createRdata(const RRType& rrtype, const RRClass& rrclass,
                      const Rdata& source);
 
-/// \brief Create RDATA of a given pair of RR type and class from the
+/// \brief Create RDATA of a given pair of RR type and class using the
 /// master lexer.
+///
+/// This is a more generic form of factory from textual RDATA, and is mainly
+/// intended to be used internally by the master file parser (\c MasterLoader)
+/// of this library.
+///
+/// The \c lexer is expected to be at the beginning of textual RDATA of the
+/// specified type and class.  This function (and its underlying Rdata
+/// implementations) extracts necessary tokens from the lexer and constructs
+/// the RDATA from them.
+///
+/// Due to the intended usage of this version, this function handles error
+/// cases quite differently from other versions.  It internally catches
+/// most of syntax and semantics errors of the input (reported as exceptions),
+/// calls the corresponding callback specified by the \c callbacks parameters,
+/// and returns a NULL smart pointer.  If the caller rather wants to get
+/// an exception in these cases, it can pass a callback that internally
+/// throws on error.  Some critical exceptions such as \c std::bad_alloc are
+/// still propagated to the upper layer as it doesn't make sense to try
+/// recovery from such a situation within this function.
+///
+/// Whether or not the creation succeeds, this function updates the lexer
+/// until it reaches either the end of line or file, starting from the end of
+/// the RDATA text (or the point of failure if the parsing fails in the
+/// middle of it).  The caller can therefore assume it's ready for reading
+/// the next data (which is normally a subsequent RR in the zone file) on
+/// return, whether or not this function succeeds.
+///
+/// \param rrtype An \c RRType object specifying the type/class pair.
+/// \param rrclass An \c RRClass object specifying the type/class pair.
+/// \param lexer A \c MasterLexer object parsing a master file for the
+/// RDATA to be created
+/// \param origin If non NULL, specifies the origin of any domain name fields
+/// of the RDATA that are non absolute.
+/// \param options Master loader options controlling how to deal with errors
+/// or non critical issues in the parsed RDATA.
+/// \param callbacks Callback to be called when an error or non critical issue
+/// is found.
+/// \return An \c RdataPtr object pointing to the created
+/// \c Rdata object.  Will be NULL if parsing fails.
 RdataPtr createRdata(const RRType& rrtype, const RRClass& rrclass,
                      MasterLexer& lexer, const Name* origin,
                      MasterLoader::Options options,

src/lib/dns/rdata/generic/detail/char_string.cc

@@ -0,0 +1,98 @@
+// Copyright (C) 2012  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <exceptions/exceptions.h>
+
+#include <dns/rdata.h>
+#include <dns/master_lexer.h>
+#include <dns/rdata/generic/detail/char_string.h>
+
+#include <boost/lexical_cast.hpp>
+
+#include <cassert>
+#include <cctype>
+#include <cstring>
+#include <vector>
+
+#include <stdint.h>
+
+namespace isc {
+namespace dns {
+namespace rdata {
+namespace generic {
+namespace detail {
+
+namespace {
+// Convert a DDD form to the corresponding integer
+int
+decimalToNumber(const char* s, const char* s_end) {
+    if (s_end - s < 3) {
+        isc_throw(InvalidRdataText, "Escaped digits too short");
+    }
+
+    const std::string num_str(s, s + 3);
+    try {
+        const int i = boost::lexical_cast<int>(num_str);
+        if (i > 255) {
+            isc_throw(InvalidRdataText, "Escaped digits too large: "
+                      << num_str);
+        }
+        return (i);
+    } catch (const boost::bad_lexical_cast&) {
+        isc_throw(InvalidRdataText,
+                  "Invalid form for escaped digits: " << num_str);
+    }
+}
+}
+
+void
+strToCharString(const MasterToken::StringRegion& str_region,
+                CharString& result)
+{
+    // make a space for the 1-byte length field; filled in at the end
+    result.push_back(0);
+
+    bool escape = false;
+    const char* s = str_region.beg;
+    const char* const s_end = str_region.beg + str_region.len;
+
+    for (size_t n = str_region.len; n != 0; --n, ++s) {
+        int c = (*s & 0xff);
+        if (escape && std::isdigit(c) != 0) {
+            c = decimalToNumber(s, s_end);
+            assert(n >= 3);
+            n -= 2;
+            s += 2;
+        } else if (!escape && c == '\\') {
+            escape = true;
+            continue;
+        }
+        escape = false;
+        result.push_back(c);
+    }
+    if (escape) {               // terminated by non-escaped '\'
+        isc_throw(InvalidRdataText, "character-string ends with '\\'");
+    }
+    if (result.size() > MAX_CHARSTRING_LEN + 1) { // '+ 1' due to the len field
+        isc_throw(CharStringTooLong, "character-string is too long: " <<
+                  (result.size() - 1) << "(+1) characters");
+    }
+    result[0] = result.size() - 1;
+}
+
+} // end of detail
+} // end of generic
+} // end of rdata
+} // end of dns
+} // end of isc

src/lib/dns/rdata/generic/detail/char_string.h

@@ -0,0 +1,63 @@
+// Copyright (C) 2012  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#ifndef DNS_RDATA_CHARSTRING_H
+#define DNS_RDATA_CHARSTRING_H 1
+
+#include <dns/master_lexer.h>
+
+#include <vector>
+#include <stdint.h>
+
+namespace isc {
+namespace dns {
+namespace rdata {
+namespace generic {
+namespace detail {
+
+/// \brief Type for DNS character string.
+///
+/// A character string can contain any unsigned 8-bit value, so this cannot
+/// be the bare char basis.
+typedef std::vector<uint8_t> CharString;
+
+/// \brief Convert a DNS character-string into corresponding binary data.
+///
+/// This helper function takes a string object that is expected to be a
+/// textual representation of a valid DNS character-string, and dumps
+/// the corresponding binary sequence in the given placeholder (passed
+/// via the \c result parameter).  It handles escape notations of
+/// character-strings with a backslash ('\'), and checks the length
+/// restriction.
+///
+/// \throw CharStringTooLong The resulting binary data are too large for a
+/// valid character-string.
+/// \throw InvalidRdataText Other syntax errors.
+///
+/// \brief str_region A string that represents a character-string.
+/// \brief result A placeholder vector where the resulting data are to be
+/// stored.  Expected to be empty, but it's not checked.
+void strToCharString(const MasterToken::StringRegion& str_region,
+                     CharString& result);
+
+} // namespace detail
+} // namespace generic
+} // namespace rdata
+} // namespace dns
+} // namespace isc
+#endif  // DNS_RDATA_CHARSTRING_H
+
+// Local Variables:
+// mode: c++
+// End:

src/lib/dns/rdata/generic/detail/txt_like.h

@@ -15,13 +15,20 @@
 #ifndef TXT_LIKE_H
 #define TXT_LIKE_H 1
 
+#include <dns/master_lexer.h>
+#include <dns/rdata/generic/detail/char_string.h>
+
 #include <stdint.h>
 
 #include <string>
+#include <sstream>
 #include <vector>
 
-using namespace std;
-using namespace isc::util;
+namespace isc {
+namespace dns {
+namespace rdata {
+namespace generic {
+namespace detail {
 
 /// \brief \c rdata::TXTLikeImpl class represents the TXT-like RDATA for TXT
 /// and SPF types.
@@ -41,7 +48,7 @@ public:
     ///
     /// \c InvalidRdataLength is thrown if rdata_len exceeds the maximum.
     /// \c DNSMessageFORMERR is thrown if the RR is misformed.
-    TXTLikeImpl(InputBuffer& buffer, size_t rdata_len) {
+    TXTLikeImpl(util::InputBuffer& buffer, size_t rdata_len) {
         if (rdata_len > MAX_RDLENGTH) {
             isc_throw(InvalidRdataLength, "RDLENGTH too large: " << rdata_len);
         }
@@ -59,7 +66,7 @@ public:
                           " RDATA: character string length is too large: " <<
                           static_cast<int>(len));
             }
-            vector<uint8_t> data(len + 1);
+            std::vector<uint8_t> data(len + 1);
             data[0] = len;
             buffer.readData(&data[0] + 1, len);
             string_list_.push_back(data);
@@ -70,46 +77,61 @@ public:
 
     /// \brief Constructor from string.
     ///
-    /// <b>Exceptions</b>
-    ///
-    /// \c CharStringTooLong is thrown if the parameter string length exceeds
-    /// maximum.
-    /// \c InvalidRdataText is thrown if the method cannot process the
-    /// parameter data.
+    /// \throw CharStringTooLong the parameter string length exceeds maximum.
+    /// \throw InvalidRdataText the method cannot process the parameter data
     explicit TXTLikeImpl(const std::string& txtstr) {
-        // TBD: this is a simple, incomplete implementation that only supports
-        // a single character-string.
+        std::istringstream ss(txtstr);
+        MasterLexer lexer;
+        lexer.pushSource(ss);
+
+        try {
+            buildFromTextHelper(lexer);
+            if (lexer.getNextToken().getType() != MasterToken::END_OF_FILE) {
+                isc_throw(InvalidRdataText, "Failed to construct " <<
+                          RRType(typeCode) << " RDATA from '" << txtstr <<
+                          "': extra new line");
+            }
+        } catch (const MasterLexer::LexerError& ex) {
+            isc_throw(InvalidRdataText, "Failed to construct " <<
+                      RRType(typeCode) << " RDATA from '" << txtstr << "': "
+                      << ex.what());
+        }
+    }
 
-        size_t length = txtstr.size();
-        size_t pos_begin = 0;
+    /// \brief Constructor using the master lexer.
+    ///
+    /// \throw CharStringTooLong the parameter string length exceeds maximum.
+    /// \throw InvalidRdataText the method cannot process the parameter data
+    ///
+    /// \param lexer A \c MasterLexer object parsing a master file for this
+    /// RDATA.
+    TXTLikeImpl(MasterLexer& lexer) {
+        buildFromTextHelper(lexer);
+    }
 
-        if (length > 1 && txtstr[0] == '"' && txtstr[length - 1] == '"') {
-            pos_begin = 1;
-            length -= 2;
+private:
+    void buildFromTextHelper(MasterLexer& lexer) {
+        while (true) {
+            const MasterToken& token = lexer.getNextToken(
+                MasterToken::QSTRING, true);
+            if (token.getType() != MasterToken::STRING &&
+                token.getType() != MasterToken::QSTRING) {
+                break;
+            }
+            string_list_.push_back(std::vector<uint8_t>());
+            strToCharString(token.getStringRegion(), string_list_.back());
         }
 
-        if (length > MAX_CHARSTRING_LEN) {
-            isc_throw(CharStringTooLong, RRType(typeCode) <<
-                      " RDATA construction from text:"
-                      " string length is too long: " << length);
-        }
+        // Let upper layer handle eol/eof.
+        lexer.ungetToken();
 
-        // TBD: right now, we don't support escaped characters
-        if (txtstr.find('\\') != string::npos) {
-            isc_throw(InvalidRdataText, RRType(typeCode) <<
-                      " RDATA from text:"
-                      " escaped character is currently not supported: " <<
-                      txtstr);
+        if (string_list_.empty()) {
+            isc_throw(InvalidRdataText, "Failed to construct" <<
+                      RRType(typeCode) << " RDATA: empty input");
         }
-
-        vector<uint8_t> data;
-        data.reserve(length + 1);
-        data.push_back(length);
-        data.insert(data.end(), txtstr.begin() + pos_begin,
-                    txtstr.begin() + pos_begin + length);
-        string_list_.push_back(data);
     }
 
+public:
     /// \brief The copy constructor.
     ///
     /// Trivial for now, we could've used the default one.
@@ -122,9 +144,9 @@ public:
     ///
     /// \param buffer An output buffer to store the wire data.
     void
-    toWire(OutputBuffer& buffer) const {
-        for (vector<vector<uint8_t> >::const_iterator it =
-                                                          string_list_.begin();
+    toWire(util::OutputBuffer& buffer) const {
+        for (std::vector<std::vector<uint8_t> >::const_iterator it =
+                 string_list_.begin();
              it != string_list_.end();
              ++it)
         {
@@ -139,8 +161,8 @@ public:
     /// to.
     void
     toWire(AbstractMessageRenderer& renderer) const {
-        for (vector<vector<uint8_t> >::const_iterator it =
-                                                          string_list_.begin();
+        for (std::vector<std::vector<uint8_t> >::const_iterator it =
+                 string_list_.begin();
              it != string_list_.end();
              ++it)
         {
@@ -151,14 +173,14 @@ public:
     /// \brief Convert the TXT-like data to a string.
     ///
     /// \return A \c string object that represents the TXT-like data.
-    string
+    std::string
     toText() const {
-        string s;
+        std::string s;
 
         // XXX: this implementation is not entirely correct.  for example, it
         // should escape double-quotes if they appear in the character string.
-        for (vector<vector<uint8_t> >::const_iterator it =
-                                                          string_list_.begin();
+        for (std::vector<std::vector<uint8_t> >::const_iterator it =
+                 string_list_.begin();
              it != string_list_.end();
              ++it)
         {
@@ -189,7 +211,7 @@ public:
         OutputBuffer this_buffer(0);
         toWire(this_buffer);
         uint8_t const* const this_data = (uint8_t const*)this_buffer.getData();
-        size_t this_len = this_buffer.getLength();
+        const size_t this_len = this_buffer.getLength();
 
         OutputBuffer other_buffer(0);
         other.toWire(other_buffer);
@@ -214,11 +236,14 @@ private:
     std::vector<std::vector<uint8_t> > string_list_;
 };
 
-// END_RDATA_NAMESPACE
-// END_ISC_NAMESPACE
+}      // namespace detail
+}      // namespace generic
+}      // namespace rdata
+}      // namespace dns
+}      // namespace isc
 
 #endif //  TXT_LIKE_H
 
-// Local Variables: 
+// Local Variables:
 // mode: c++
-// End: 
+// End:

src/lib/dns/rdata/generic/spf_99.cc

@@ -24,18 +24,17 @@
 #include <dns/rdata.h>
 #include <dns/rdataclass.h>
 
+/// This class implements the basic interfaces inherited from the abstract
+/// \c rdata::Rdata class. The semantics of the class is provided by
+/// a copy of instantiated TXTLikeImpl class common to both TXT and SPF.
+#include <dns/rdata/generic/detail/txt_like.h>
+
 using namespace std;
 using namespace isc::util;
 
 // BEGIN_ISC_NAMESPACE
 // BEGIN_RDATA_NAMESPACE
 
-/// This class implements the basic interfaces inherited from the abstract
-/// \c rdata::Rdata class. The semantics of the class is provided by
-/// a copy of instantiated TXTLikeImpl class common to both TXT and SPF.
-
-#include <dns/rdata/generic/detail/txt_like.h>
-
 /// \brief The assignment operator
 ///
 /// It internally allocates a resource, and if it fails a corresponding
@@ -67,6 +66,21 @@ SPF::SPF(InputBuffer& buffer, size_t rdata_len) :
     impl_(new SPFImpl(buffer, rdata_len))
 {}
 
+/// \brief Constructor using the master lexer.
+///
+/// This implementation only uses the \c lexer parameters; others are
+/// ignored.
+///
+/// \throw CharStringTooLong the parameter string length exceeds maximum.
+/// \throw InvalidRdataText the method cannot process the parameter data
+///
+/// \param lexer A \c MasterLexer object parsing a master file for this
+/// RDATA.
+SPF::SPF(MasterLexer& lexer, const Name*, MasterLoader::Options,
+         MasterLoaderCallbacks&) :
+    impl_(new SPFImpl(lexer))
+{}
+
 /// \brief Constructor from string.
 ///
 /// It internally allocates a resource, and if it fails a corresponding

src/lib/dns/rdata/generic/spf_99.h

@@ -28,7 +28,9 @@
 
 // BEGIN_RDATA_NAMESPACE
 
+namespace detail {
 template<class Type, uint16_t typeCode> class TXTLikeImpl;
+}
 
 /// \brief \c rdata::SPF class represents the SPF RDATA as defined %in
 /// RFC4408.
@@ -65,7 +67,7 @@ public:
     const std::vector<std::vector<uint8_t> >& getString() const;
 
 private:
-    typedef TXTLikeImpl<SPF, 99> SPFImpl;
+    typedef isc::dns::rdata::generic::detail::TXTLikeImpl<SPF, 99> SPFImpl;
     SPFImpl* impl_;
 };
 

src/lib/dns/rdata/generic/txt_16.cc

@@ -23,6 +23,7 @@
 #include <dns/messagerenderer.h>
 #include <dns/rdata.h>
 #include <dns/rdataclass.h>
+#include <dns/rdata/generic/detail/txt_like.h>
 
 using namespace std;
 using namespace isc::util;
@@ -30,8 +31,6 @@ using namespace isc::util;
 // BEGIN_ISC_NAMESPACE
 // BEGIN_RDATA_NAMESPACE
 
-#include <dns/rdata/generic/detail/txt_like.h>
-
 TXT&
 TXT::operator=(const TXT& source) {
     if (impl_ == source.impl_) {
@@ -53,6 +52,21 @@ TXT::TXT(InputBuffer& buffer, size_t rdata_len) :
     impl_(new TXTImpl(buffer, rdata_len))
 {}
 
+/// \brief Constructor using the master lexer.
+///
+/// This implementation only uses the \c lexer parameters; others are
+/// ignored.
+///
+/// \throw CharStringTooLong the parameter string length exceeds maximum.
+/// \throw InvalidRdataText the method cannot process the parameter data
+///
+/// \param lexer A \c MasterLexer object parsing a master file for this
+/// RDATA.
+TXT::TXT(MasterLexer& lexer, const Name*, MasterLoader::Options,
+         MasterLoaderCallbacks&) :
+    impl_(new TXTImpl(lexer))
+{}
+
 TXT::TXT(const std::string& txtstr) :
     impl_(new TXTImpl(txtstr))
 {}

src/lib/dns/rdata/generic/txt_16.h

@@ -28,7 +28,9 @@
 
 // BEGIN_RDATA_NAMESPACE
 
+namespace detail {
 template<class Type, uint16_t typeCode> class TXTLikeImpl;
+}
 
 class TXT : public Rdata {
 public:
@@ -39,7 +41,7 @@ public:
     ~TXT();
 
 private:
-    typedef TXTLikeImpl<TXT, 16> TXTImpl;
+    typedef isc::dns::rdata::generic::detail::TXTLikeImpl<TXT, 16> TXTImpl;
     TXTImpl* impl_;
 };
 

src/lib/dns/rdata/in_1/aaaa_28.cc

@@ -12,6 +12,15 @@
 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
+#include <exceptions/exceptions.h>
+#include <util/buffer.h>
+#include <dns/exceptions.h>
+#include <dns/messagerenderer.h>
+#include <dns/rdata.h>
+#include <dns/rdataclass.h>
+#include <dns/master_lexer.h>
+#include <dns/master_loader.h>
+
 #include <stdint.h>
 #include <string.h>
 
@@ -20,14 +29,6 @@
 #include <arpa/inet.h> // XXX: for inet_pton/ntop(), not exist in C++ standards
 #include <sys/socket.h> // for AF_INET/AF_INET6
 
-#include <exceptions/exceptions.h>
-
-#include <util/buffer.h>
-#include <dns/exceptions.h>
-#include <dns/messagerenderer.h>
-#include <dns/rdata.h>
-#include <dns/rdataclass.h>
-
 using namespace std;
 using namespace isc::util;
 
@@ -42,6 +43,16 @@ AAAA::AAAA(const std::string& addrstr) {
     }
 }
 
+AAAA::AAAA(MasterLexer& lexer, const Name*,
+           MasterLoader::Options, MasterLoaderCallbacks&)
+{
+    const MasterToken& token = lexer.getNextToken(MasterToken::STRING);
+    if (inet_pton(AF_INET6, token.getStringRegion().beg, &addr_) != 1) {
+        isc_throw(InvalidRdataText, "Failed to convert '"
+                  << token.getString() << "' to IN/AAAA RDATA");
+    }
+}
+
 AAAA::AAAA(InputBuffer& buffer, size_t rdata_len) {
     if (rdata_len != sizeof(addr_)) {
         isc_throw(DNSMessageFORMERR,

src/lib/dns/rdata/template.cc

@@ -34,6 +34,11 @@ using namespace isc::util;
 // If you added member functions specific to this derived class, you'll need
 // to implement them here, of course.
 
+MyType::MyType(MasterLexer& lexer, const Name* origin,
+               MasterLoader::Options options, MasterLoaderCallbacks& callbacks)
+{
+}
+
 MyType::MyType(const string& type_str) {
 }
 

src/lib/dns/rrparamregistry-placeholder.cc

@@ -51,9 +51,10 @@ AbstractRdataFactory::create(MasterLexer& lexer, const Name*,
     std::string s;
 
     while (true) {
-        const MasterLexer::Token& token = lexer.getNextToken();
-        if ((token.getType() == MasterLexer::Token::END_OF_FILE) ||
-            (token.getType() == MasterLexer::Token::END_OF_LINE)) {
+        const MasterToken& token = lexer.getNextToken();
+        if ((token.getType() == MasterToken::END_OF_FILE) ||
+            (token.getType() == MasterToken::END_OF_LINE)) {
+            lexer.ungetToken(); // let the upper layer handle the end-of token
             break;
         }
 

src/lib/dns/rrparamregistry.h

@@ -119,10 +119,22 @@ public:
     /// \return An \c RdataPtr object pointing to the created \c Rdata object.
     virtual RdataPtr create(const rdata::Rdata& source) const = 0;
 
-    /// \brief Create RDATA from MasterLexer
-    virtual RdataPtr create(MasterLexer& lexer, const Name*,
-                            MasterLoader::Options,
-                            MasterLoaderCallbacks&) const;
+    /// \brief Create RDATA using MasterLexer.
+    ///
+    /// This version of the method defines the entry point of factory
+    /// of a specific RR type and class for \c RRParamRegistry::createRdata()
+    /// that uses \c MasterLexer.  See its description for the expected
+    /// behavior and meaning of the parameters.
+    ///
+    /// \note Right now this is not defined as a pure virtual method and
+    /// provides the default implementation.  This is an intermediate
+    /// workaround until we implement the underlying constructor for all
+    /// supported \c Rdata classes; once it's completed the workaround
+    /// default implementation should be removed and this method should become
+    /// pure virtual.
+    virtual RdataPtr create(MasterLexer& lexer, const Name* origin,
+                            MasterLoader::Options options,
+                            MasterLoaderCallbacks& callbacks) const;
     //@}
 };
 
@@ -504,9 +516,20 @@ public:
     rdata::RdataPtr createRdata(const RRType& rrtype, const RRClass& rrclass,
                                 const rdata::Rdata& source);
 
-    /// \brief Create RDATA from MasterLexer
+    /// \brief Create RDATA using MasterLexer
+    ///
+    /// This method is expected to be used as the underlying implementation
+    /// of the same signature of \c rdata::createRdata().  One main
+    /// difference is that this method is only responsible for constructing
+    /// the Rdata; it doesn't update the lexer to reach the end of line or
+    /// file or doesn't care about whether there's an extra (garbage) token
+    /// after the textual RDATA representation.  Another difference is that
+    /// this method can throw on error and never returns a NULL pointer.
+    ///
+    /// For other details and parameters, see the description of
+    /// \c rdata::createRdata().
     rdata::RdataPtr createRdata(const RRType& rrtype, const RRClass& rrclass,
-                                MasterLexer& lexer, const Name* name,
+                                MasterLexer& lexer, const Name* origin,
                                 MasterLoader::Options options,
                                 MasterLoaderCallbacks& callbacks);
     //@}

src/lib/dns/tests/Makefile.am

@@ -36,6 +36,7 @@ run_unittests_SOURCES += opcode_unittest.cc
 run_unittests_SOURCES += rcode_unittest.cc
 run_unittests_SOURCES += rdata_unittest.h rdata_unittest.cc
 run_unittests_SOURCES += rdatafields_unittest.cc
+run_unittests_SOURCES += rdata_char_string_unittest.cc
 run_unittests_SOURCES += rdata_in_a_unittest.cc rdata_in_aaaa_unittest.cc
 run_unittests_SOURCES += rdata_ns_unittest.cc rdata_soa_unittest.cc
 run_unittests_SOURCES += rdata_txt_like_unittest.cc

src/lib/dns/tests/master_lexer_state_unittest.cc

@@ -24,7 +24,7 @@ using namespace isc::dns;
 using namespace master_lexer_internal;
 
 namespace {
-typedef MasterLexer::Token Token; // shortcut
+typedef MasterToken Token; // shortcut
 
 class MasterLexerStateTest : public ::testing::Test {
 protected:
@@ -260,7 +260,7 @@ TEST_F(MasterLexerStateTest, crlf) {
 // Commonly used check for string related test cases, checking if the given
 // token has expected values.
 void
-stringTokenCheck(const std::string& expected, const MasterLexer::Token& token,
+stringTokenCheck(const std::string& expected, const MasterToken& token,
                  bool quoted = false)
 {
     EXPECT_EQ(quoted ? Token::QSTRING : Token::STRING, token.getType());
@@ -269,6 +269,10 @@ stringTokenCheck(const std::string& expected, const MasterLexer::Token& token,
                              token.getStringRegion().beg +
                              token.getStringRegion().len);
     EXPECT_EQ(expected, actual);
+
+    // There should be "hidden" nul-terminator after the string data.
+    ASSERT_NE(static_cast<const char*>(NULL), token.getStringRegion().beg);
+    EXPECT_EQ(0, *(token.getStringRegion().beg + token.getStringRegion().len));
 }
 
 TEST_F(MasterLexerStateTest, string) {
@@ -365,6 +369,7 @@ TEST_F(MasterLexerStateTest, stringEscape) {
 TEST_F(MasterLexerStateTest, quotedString) {
     ss << "\"ignore-quotes\"\n";
     ss << "\"quoted string\" "; // space is part of the qstring
+    ss << "\"\" "; // empty quoted string
     // also check other separator characters. note that \r doesn't cause
     // UNBALANCED_QUOTES.  Not sure if it's intentional, but that's how the
     // BIND 9 version works, so we follow it (it should be too minor to matter
@@ -391,6 +396,11 @@ TEST_F(MasterLexerStateTest, quotedString) {
     s_qstring.handle(lexer);
     stringTokenCheck("quoted string", s_string.getToken(lexer), true);
 
+    // Empty string is okay as qstring
+    EXPECT_EQ(&s_qstring, State::start(lexer, options));
+    s_qstring.handle(lexer);
+    stringTokenCheck("", s_string.getToken(lexer), true);
+
     // Also checks other separator characters within a qstring
     EXPECT_EQ(&s_qstring, State::start(lexer, options));
     s_qstring.handle(lexer);

src/lib/dns/tests/master_lexer_token_unittest.cc

@@ -31,27 +31,27 @@ const size_t TEST_STRING_LEN = sizeof(TEST_STRING) - 1;
 class MasterLexerTokenTest : public ::testing::Test {
 protected:
     MasterLexerTokenTest() :
-        token_eof(MasterLexer::Token::END_OF_FILE),
+        token_eof(MasterToken::END_OF_FILE),
         token_str(TEST_STRING, TEST_STRING_LEN),
         token_num(42),
-        token_err(MasterLexer::Token::UNEXPECTED_END)
+        token_err(MasterToken::UNEXPECTED_END)
     {}
 
-    const MasterLexer::Token token_eof; // an example of non-value type token
-    const MasterLexer::Token token_str;
-    const MasterLexer::Token token_num;
-    const MasterLexer::Token token_err;
+    const MasterToken token_eof; // an example of non-value type token
+    const MasterToken token_str;
+    const MasterToken token_num;
+    const MasterToken token_err;
 };
 
 
 TEST_F(MasterLexerTokenTest, strings) {
     // basic construction and getter checks
-    EXPECT_EQ(MasterLexer::Token::STRING, token_str.getType());
+    EXPECT_EQ(MasterToken::STRING, token_str.getType());
     EXPECT_EQ(std::string("string token"), token_str.getString());
     std::string strval = "dummy"; // this should be replaced
     token_str.getString(strval);
     EXPECT_EQ(std::string("string token"), strval);
-    const MasterLexer::Token::StringRegion str_region =
+    const MasterToken::StringRegion str_region =
         token_str.getStringRegion();
     EXPECT_EQ(TEST_STRING, str_region.beg);
     EXPECT_EQ(TEST_STRING_LEN, str_region.len);
@@ -62,17 +62,17 @@ TEST_F(MasterLexerTokenTest, strings) {
     std::string expected_str("string token");
     expected_str.push_back('\0');
     EXPECT_EQ(expected_str,
-              MasterLexer::Token(TEST_STRING, TEST_STRING_LEN + 1).getString());
-    MasterLexer::Token(TEST_STRING, TEST_STRING_LEN + 1).getString(strval);
+              MasterToken(TEST_STRING, TEST_STRING_LEN + 1).getString());
+    MasterToken(TEST_STRING, TEST_STRING_LEN + 1).getString(strval);
     EXPECT_EQ(expected_str, strval);
 
     // Construct type of qstring
-    EXPECT_EQ(MasterLexer::Token::QSTRING,
-              MasterLexer::Token(TEST_STRING, sizeof(TEST_STRING), true).
+    EXPECT_EQ(MasterToken::QSTRING,
+              MasterToken(TEST_STRING, sizeof(TEST_STRING), true).
               getType());
     // if we explicitly set 'quoted' to false, it should be normal string
-    EXPECT_EQ(MasterLexer::Token::STRING,
-              MasterLexer::Token(TEST_STRING, sizeof(TEST_STRING), false).
+    EXPECT_EQ(MasterToken::STRING,
+              MasterToken(TEST_STRING, sizeof(TEST_STRING), false).
               getType());
 
     // getString/StringRegion() aren't allowed for non string(-variant) types
@@ -86,23 +86,23 @@ TEST_F(MasterLexerTokenTest, strings) {
 
 TEST_F(MasterLexerTokenTest, numbers) {
     EXPECT_EQ(42, token_num.getNumber());
-    EXPECT_EQ(MasterLexer::Token::NUMBER, token_num.getType());
+    EXPECT_EQ(MasterToken::NUMBER, token_num.getType());
 
     // It's copyable and assignable.
-    MasterLexer::Token token(token_num);
+    MasterToken token(token_num);
     EXPECT_EQ(42, token.getNumber());
-    EXPECT_EQ(MasterLexer::Token::NUMBER, token.getType());
+    EXPECT_EQ(MasterToken::NUMBER, token.getType());
 
     token = token_num;
     EXPECT_EQ(42, token.getNumber());
-    EXPECT_EQ(MasterLexer::Token::NUMBER, token.getType());
+    EXPECT_EQ(MasterToken::NUMBER, token.getType());
 
     // it's okay to replace it with a different type of token
     token = token_eof;
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, token.getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, token.getType());
 
     // Possible max value
-    token = MasterLexer::Token(0xffffffff);
+    token = MasterToken(0xffffffff);
     EXPECT_EQ(4294967295u, token.getNumber());
 
     // getNumber() isn't allowed for non number types
@@ -112,58 +112,52 @@ TEST_F(MasterLexerTokenTest, numbers) {
 
 TEST_F(MasterLexerTokenTest, novalues) {
     // Just checking we can construct them and getType() returns correct value.
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, token_eof.getType());
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE,
-              MasterLexer::Token(MasterLexer::Token::END_OF_LINE).getType());
-    EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
-              MasterLexer::Token(MasterLexer::Token::INITIAL_WS).getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, token_eof.getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE,
+              MasterToken(MasterToken::END_OF_LINE).getType());
+    EXPECT_EQ(MasterToken::INITIAL_WS,
+              MasterToken(MasterToken::INITIAL_WS).getType());
 
     // Special types of tokens cannot have value-based types
-    EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::STRING),
-                 isc::InvalidParameter);
-    EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::QSTRING),
-                 isc::InvalidParameter);
-    EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::NUMBER),
-                 isc::InvalidParameter);
-    EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::ERROR),
-                 isc::InvalidParameter);
+    EXPECT_THROW(MasterToken t(MasterToken::STRING), isc::InvalidParameter);
+    EXPECT_THROW(MasterToken t(MasterToken::QSTRING), isc::InvalidParameter);
+    EXPECT_THROW(MasterToken t(MasterToken::NUMBER), isc::InvalidParameter);
+    EXPECT_THROW(MasterToken t(MasterToken::ERROR), isc::InvalidParameter);
 }
 
 TEST_F(MasterLexerTokenTest, errors) {
-    EXPECT_EQ(MasterLexer::Token::ERROR, token_err.getType());
-    EXPECT_EQ(MasterLexer::Token::UNEXPECTED_END, token_err.getErrorCode());
+    EXPECT_EQ(MasterToken::ERROR, token_err.getType());
+    EXPECT_EQ(MasterToken::UNEXPECTED_END, token_err.getErrorCode());
     EXPECT_EQ("unexpected end of input", token_err.getErrorText());
-    EXPECT_EQ("lexer not started",
-              MasterLexer::Token(MasterLexer::Token::NOT_STARTED).
+    EXPECT_EQ("lexer not started", MasterToken(MasterToken::NOT_STARTED).
               getErrorText());
     EXPECT_EQ("unbalanced parentheses",
-              MasterLexer::Token(MasterLexer::Token::UNBALANCED_PAREN).
+              MasterToken(MasterToken::UNBALANCED_PAREN).
               getErrorText());
-    EXPECT_EQ("unbalanced quotes",
-              MasterLexer::Token(MasterLexer::Token::UNBALANCED_QUOTES).
+    EXPECT_EQ("unbalanced quotes", MasterToken(MasterToken::UNBALANCED_QUOTES).
               getErrorText());
-    EXPECT_EQ("no token produced",
-              MasterLexer::Token(MasterLexer::Token::NO_TOKEN_PRODUCED).
+    EXPECT_EQ("no token produced", MasterToken(MasterToken::NO_TOKEN_PRODUCED).
               getErrorText());
     EXPECT_EQ("number out of range",
-              MasterLexer::Token(MasterLexer::Token::NUMBER_OUT_OF_RANGE).
+              MasterToken(MasterToken::NUMBER_OUT_OF_RANGE).
               getErrorText());
+    EXPECT_EQ("not a valid number",
+              MasterToken(MasterToken::BAD_NUMBER).getErrorText());
 
     // getErrorCode/Text() isn't allowed for non number types
     EXPECT_THROW(token_num.getErrorCode(), isc::InvalidOperation);
     EXPECT_THROW(token_num.getErrorText(), isc::InvalidOperation);
 
-    // Only the pre-defined error code is accepted.  Hardcoding '6' (max code
+    // Only the pre-defined error code is accepted.  Hardcoding '7' (max code
     // + 1) is intentional; it'd be actually better if we notice it when we
     // update the enum list (which shouldn't happen too often).
-    EXPECT_THROW(MasterLexer::Token(MasterLexer::Token::ErrorCode(6)),
+    EXPECT_THROW(MasterToken(MasterToken::ErrorCode(7)),
                  isc::InvalidParameter);
 
     // Check the coexistence of "from number" and "from error-code"
     // constructors won't cause confusion.
-    EXPECT_EQ(MasterLexer::Token::NUMBER,
-              MasterLexer::Token(static_cast<uint32_t>(
-                                     MasterLexer::Token::NOT_STARTED)).
+    EXPECT_EQ(MasterToken::NUMBER,
+              MasterToken(static_cast<uint32_t>(MasterToken::NOT_STARTED)).
               getType());
 }
 }

src/lib/dns/tests/master_lexer_unittest.cc

@@ -141,19 +141,19 @@ TEST_F(MasterLexerTest, getNextToken) {
     lexer.pushSource(ss);
 
     // First, the newline should get out.
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // Then the whitespace, if we specify the option.
-    EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
+    EXPECT_EQ(MasterToken::INITIAL_WS,
               lexer.getNextToken(MasterLexer::INITIAL_WS).getType());
     // The newline
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // The (quoted) string
-    EXPECT_EQ(MasterLexer::Token::QSTRING,
+    EXPECT_EQ(MasterToken::QSTRING,
               lexer.getNextToken(MasterLexer::QSTRING).getType());
 
     // And the end of line and file
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
 }
 
 // Test we correctly find end of file.
@@ -162,12 +162,12 @@ TEST_F(MasterLexerTest, eof) {
     lexer.pushSource(ss);
 
     // The first one is found to be EOF
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
     // And it stays on EOF for any following attempts
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
     // And we can step back one token, but that is the EOF too.
     lexer.ungetToken();
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
 }
 
 // Check we properly return error when there's an opened parentheses and no
@@ -177,12 +177,12 @@ TEST_F(MasterLexerTest, getUnbalancedParen) {
     lexer.pushSource(ss);
 
     // The string gets out first
-    EXPECT_EQ(MasterLexer::Token::STRING, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::STRING, lexer.getNextToken().getType());
     // Then an unbalanced parenthesis
-    EXPECT_EQ(MasterLexer::Token::UNBALANCED_PAREN,
+    EXPECT_EQ(MasterToken::UNBALANCED_PAREN,
               lexer.getNextToken().getErrorCode());
     // And then EOF
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
 }
 
 // Check we properly return error when there's an opened quoted string and no
@@ -192,10 +192,10 @@ TEST_F(MasterLexerTest, getUnbalancedString) {
     lexer.pushSource(ss);
 
     // Then an unbalanced qstring (reported as an unexpected end)
-    EXPECT_EQ(MasterLexer::Token::UNEXPECTED_END,
+    EXPECT_EQ(MasterToken::UNEXPECTED_END,
               lexer.getNextToken(MasterLexer::QSTRING).getErrorCode());
     // And then EOF
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
 }
 
 // Test ungetting tokens works
@@ -204,28 +204,28 @@ TEST_F(MasterLexerTest, ungetToken) {
     lexer.pushSource(ss);
 
     // Try getting the newline
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // Return it and get again
     lexer.ungetToken();
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // Get the string and return it back
-    EXPECT_EQ(MasterLexer::Token::QSTRING,
+    EXPECT_EQ(MasterToken::QSTRING,
               lexer.getNextToken(MasterLexer::QSTRING).getType());
     lexer.ungetToken();
     // But if we change the options, it honors them
-    EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
+    EXPECT_EQ(MasterToken::INITIAL_WS,
               lexer.getNextToken(MasterLexer::QSTRING |
                                  MasterLexer::INITIAL_WS).getType());
     // Get to the "more" string
-    EXPECT_EQ(MasterLexer::Token::QSTRING,
+    EXPECT_EQ(MasterToken::QSTRING,
               lexer.getNextToken(MasterLexer::QSTRING).getType());
-    EXPECT_EQ(MasterLexer::Token::STRING,
+    EXPECT_EQ(MasterToken::STRING,
               lexer.getNextToken(MasterLexer::QSTRING).getType());
     // Return it back. It should get inside the parentheses.
     // Upon next attempt to get it again, the newline inside the parentheses
     // should be still ignored.
     lexer.ungetToken();
-    EXPECT_EQ(MasterLexer::Token::STRING,
+    EXPECT_EQ(MasterToken::STRING,
               lexer.getNextToken(MasterLexer::QSTRING).getType());
 }
 
@@ -235,16 +235,16 @@ TEST_F(MasterLexerTest, ungetRealOptions) {
     ss << "\n    \n";
     lexer.pushSource(ss);
     // Skip the first newline
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
 
     // If we call it the usual way, it skips up to the newline and returns
     // it
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
 
     // Now we return it. If we call it again, but with different options,
     // we get the initial whitespace.
     lexer.ungetToken();
-    EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
+    EXPECT_EQ(MasterToken::INITIAL_WS,
               lexer.getNextToken(MasterLexer::INITIAL_WS).getType());
 }
 
@@ -253,7 +253,7 @@ TEST_F(MasterLexerTest, ungetTwice) {
     ss << "\n";
     lexer.pushSource(ss);
 
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // Unget the token. It can be done once
     lexer.ungetToken();
     // But not twice
@@ -271,17 +271,164 @@ TEST_F(MasterLexerTest, ungetBeforeGet) {
 TEST_F(MasterLexerTest, ungetAfterSwitch) {
     ss << "\n\n";
     lexer.pushSource(ss);
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // Switch the source
     std::stringstream ss2;
     ss2 << "\n\n";
     lexer.pushSource(ss2);
     EXPECT_THROW(lexer.ungetToken(), isc::InvalidOperation);
     // We can get from the new source
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // And when we drop the current source, we can't unget again
     lexer.popSource();
     EXPECT_THROW(lexer.ungetToken(), isc::InvalidOperation);
 }
 
+// Common checks for the case when getNextToken() should result in LexerError
+void
+lexerErrorCheck(MasterLexer& lexer, MasterToken::Type expect,
+                MasterToken::ErrorCode expected_error)
+{
+    bool thrown = false;
+    try {
+        lexer.getNextToken(expect);
+    } catch (const MasterLexer::LexerError& error) {
+        EXPECT_EQ(expected_error, error.token_.getErrorCode());
+        thrown = true;
+    }
+    EXPECT_TRUE(thrown);
+}
+
+// Common checks regarding expected/unexpected end-of-line
+//
+// The 'lexer' should be at a position before two consecutive '\n's.
+// The first one will be recognized, and the second one will be considered an
+// unexpected token.  Then this helper consumes the second '\n', so the caller
+// can continue the test after these '\n's.
+void
+eolCheck(MasterLexer& lexer, MasterToken::Type expect) {
+    // If EOL is found and eol_ok is true, we get it.
+    EXPECT_EQ(MasterToken::END_OF_LINE,
+              lexer.getNextToken(expect, true).getType());
+    // We'll see the second '\n'; by default it will fail.
+    EXPECT_THROW(lexer.getNextToken(expect), MasterLexer::LexerError);
+    // Same if eol_ok is explicitly set to false.  This also checks the
+    // offending '\n' was "ungotten".
+    EXPECT_THROW(lexer.getNextToken(expect, false), MasterLexer::LexerError);
+
+    // And also check the error token set in the exception object.
+    lexerErrorCheck(lexer, expect, MasterToken::UNEXPECTED_END);
+
+    // Then skip the 2nd '\n'
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
+}
+
+// Common checks regarding expected/unexpected end-of-file
+//
+// The 'lexer' should be at a position just before an end-of-file.
+void
+eofCheck(MasterLexer& lexer, MasterToken::Type expect) {
+    EXPECT_EQ(MasterToken::END_OF_FILE,
+              lexer.getNextToken(expect, true).getType());
+    EXPECT_THROW(lexer.getNextToken(expect), MasterLexer::LexerError);
+    EXPECT_THROW(lexer.getNextToken(expect, false), MasterLexer::LexerError);
+}
+
+TEST_F(MasterLexerTest, getNextTokenString) {
+    ss << "normal-string\n";
+    ss << "\n";
+    ss << "another-string";
+    lexer.pushSource(ss);
+
+    // Normal successful case: Expecting a string and get one.
+    EXPECT_EQ("normal-string",
+              lexer.getNextToken(MasterToken::STRING).getString());
+    eolCheck(lexer, MasterToken::STRING);
+
+    // Same set of tests but for end-of-file
+    EXPECT_EQ("another-string",
+              lexer.getNextToken(MasterToken::STRING, true).getString());
+    eofCheck(lexer, MasterToken::STRING);
+}
+
+TEST_F(MasterLexerTest, getNextTokenQString) {
+    ss << "\"quoted-string\"\n";
+    ss << "\n";
+    ss << "normal-string";
+    lexer.pushSource(ss);
+
+    // Expecting a quoted string and get one.
+    EXPECT_EQ("quoted-string",
+              lexer.getNextToken(MasterToken::QSTRING).getString());
+    eolCheck(lexer, MasterToken::QSTRING);
+
+    // Expecting a quoted string but see a normal string.  It's okay.
+    EXPECT_EQ("normal-string",
+              lexer.getNextToken(MasterToken::QSTRING).getString());
+    eofCheck(lexer, MasterToken::QSTRING);
+}
+
+TEST_F(MasterLexerTest, getNextTokenNumber) {
+    ss << "3600\n";
+    ss << "\n";
+    ss << "4294967296 ";        // =2^32, out of range
+    ss << "not-a-number ";
+    ss << "123abc "; // starting with digits, but resulting in a string
+    ss << "86400";
+    lexer.pushSource(ss);
+
+    // Expecting a number string and get one.
+    EXPECT_EQ(3600,
+              lexer.getNextToken(MasterToken::NUMBER).getNumber());
+    eolCheck(lexer, MasterToken::NUMBER);
+
+    // Expecting a number, but it's too big for uint32.
+    lexerErrorCheck(lexer, MasterToken::NUMBER,
+                    MasterToken::NUMBER_OUT_OF_RANGE);
+    // The token should have been "ungotten".  Re-read and skip it.
+    EXPECT_EQ(MasterToken::STRING, lexer.getNextToken().getType());
+
+    // Expecting a number, but see a string.
+    lexerErrorCheck(lexer, MasterToken::NUMBER, MasterToken::BAD_NUMBER);
+    // The unexpected string should have been "ungotten".  Re-read and skip it.
+    EXPECT_EQ(MasterToken::STRING, lexer.getNextToken().getType());
+
+    // Expecting a number, but see a string.
+    lexerErrorCheck(lexer, MasterToken::NUMBER, MasterToken::BAD_NUMBER);
+    // The unexpected string should have been "ungotten".  Re-read and skip it.
+    EXPECT_EQ(MasterToken::STRING, lexer.getNextToken().getType());
+
+    // Unless we specify NUMBER, decimal number string should be recognized
+    // as a string.
+    EXPECT_EQ("86400",
+              lexer.getNextToken(MasterToken::STRING).getString());
+    eofCheck(lexer, MasterToken::NUMBER);
+}
+
+TEST_F(MasterLexerTest, getNextTokenErrors) {
+    // Check miscellaneous error cases
+
+    ss << ") ";                 // unbalanced parenthesis
+    ss << "string-after-error ";
+    lexer.pushSource(ss);
+
+    // Only string/qstring/number can be "expected".
+    EXPECT_THROW(lexer.getNextToken(MasterToken::END_OF_LINE),
+                 isc::InvalidParameter);
+    EXPECT_THROW(lexer.getNextToken(MasterToken::END_OF_FILE),
+                 isc::InvalidParameter);
+    EXPECT_THROW(lexer.getNextToken(MasterToken::INITIAL_WS),
+                 isc::InvalidParameter);
+    EXPECT_THROW(lexer.getNextToken(MasterToken::ERROR),
+                 isc::InvalidParameter);
+
+    // If it encounters a syntax error, it results in LexerError exception.
+    lexerErrorCheck(lexer, MasterToken::STRING, MasterToken::UNBALANCED_PAREN);
+
+    // Unlike the NUMBER_OUT_OF_RANGE case, the error part has been skipped
+    // within getNextToken().  We should be able to get the next token.
+    EXPECT_EQ("string-after-error",
+              lexer.getNextToken(MasterToken::STRING).getString());
+}
+
 }

src/lib/dns/tests/rdata_char_string_unittest.cc

@@ -0,0 +1,147 @@
+// Copyright (C) 2012  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <util/unittests/wiredata.h>
+
+#include <dns/rdata.h>
+#include <dns/rdata/generic/detail/char_string.h>
+
+#include <gtest/gtest.h>
+
+#include <string>
+#include <vector>
+
+using namespace isc::dns;
+using namespace isc::dns::rdata;
+using isc::dns::rdata::generic::detail::CharString;
+using isc::dns::rdata::generic::detail::strToCharString;
+using isc::util::unittests::matchWireData;
+
+namespace {
+const uint8_t test_charstr[] = {
+    sizeof("Test String") - 1,
+    'T', 'e', 's', 't', ' ', 'S', 't', 'r', 'i', 'n', 'g'
+};
+
+class CharStringTest : public ::testing::Test {
+protected:
+    CharStringTest() :
+        // char-string representation for test data using two types of escape
+        // ('r' = 114)
+        test_str("Test\\ St\\114ing")
+    {
+        str_region.beg = &test_str[0];
+        str_region.len = test_str.size();
+    }
+    CharString chstr;           // place holder
+    const std::string test_str;
+    MasterToken::StringRegion str_region;
+};
+
+MasterToken::StringRegion
+createStringRegion(const std::string& str) {
+    MasterToken::StringRegion region;
+    region.beg = &str[0]; // note std ensures this works even if str is empty
+    region.len = str.size();
+    return (region);
+}
+
+TEST_F(CharStringTest, normalConversion) {
+    uint8_t tmp[3];             // placeholder for expected sequence
+
+    strToCharString(str_region, chstr);
+    matchWireData(test_charstr, sizeof(test_charstr), &chstr[0], chstr.size());
+
+    // Empty string
+    chstr.clear();
+    strToCharString(createStringRegion(""), chstr);
+    tmp[0] = 0;
+    matchWireData(tmp, 1, &chstr[0], chstr.size());
+
+    // Possible largest char string
+    chstr.clear();
+    std::string long_str(255, 'x');
+    strToCharString(createStringRegion(long_str), chstr);
+    std::vector<uint8_t> expected;
+    expected.push_back(255);    // len of char string
+    expected.insert(expected.end(), long_str.begin(), long_str.end());
+    matchWireData(&expected[0], expected.size(), &chstr[0], chstr.size());
+
+    // Same data as the previous case, but the original string is longer than
+    // the max; this shouldn't be rejected
+    chstr.clear();
+    long_str.at(254) = '\\';    // replace the last 'x' with '\'
+    long_str.append("120");     // 'x' = 120
+    strToCharString(createStringRegion(long_str), chstr);
+    matchWireData(&expected[0], expected.size(), &chstr[0], chstr.size());
+
+    // Escaped '\'
+    chstr.clear();
+    tmp[0] = 1;
+    tmp[1] = '\\';
+    strToCharString(createStringRegion("\\\\"), chstr);
+    matchWireData(tmp, 2, &chstr[0], chstr.size());
+
+    // Boundary values for \DDD
+    chstr.clear();
+    tmp[0] = 1;
+    tmp[1] = 0;
+    strToCharString(createStringRegion("\\000"), chstr);
+    matchWireData(tmp, 2, &chstr[0], chstr.size());
+
+    chstr.clear();
+    strToCharString(createStringRegion("\\255"), chstr);
+    tmp[0] = 1;
+    tmp[1] = 255;
+    matchWireData(tmp, 2, &chstr[0], chstr.size());
+
+    // Another digit follows DDD; it shouldn't cause confusion
+    chstr.clear();
+    strToCharString(createStringRegion("\\2550"), chstr);
+    tmp[0] = 2;                 // string len is now 2
+    tmp[2] = '0';
+    matchWireData(tmp, 3, &chstr[0], chstr.size());
+}
+
+TEST_F(CharStringTest, badConversion) {
+    // string cannot exceed 255 bytes
+    EXPECT_THROW(strToCharString(createStringRegion(std::string(256, 'a')),
+                                 chstr),
+                 CharStringTooLong);
+
+    // input string ending with (non escaped) '\'
+    chstr.clear();
+    EXPECT_THROW(strToCharString(createStringRegion("foo\\"), chstr),
+                 InvalidRdataText);
+}
+
+TEST_F(CharStringTest, badDDD) {
+    // Check various type of bad form of \DDD
+
+    // Not a number
+    EXPECT_THROW(strToCharString(createStringRegion("\\1a2"), chstr),
+                 InvalidRdataText);
+    EXPECT_THROW(strToCharString(createStringRegion("\\12a"), chstr),
+                 InvalidRdataText);
+
+    // Not in the range of uint8_t
+    EXPECT_THROW(strToCharString(createStringRegion("\\256"), chstr),
+                 InvalidRdataText);
+
+    // Short buffer
+    EXPECT_THROW(strToCharString(createStringRegion("\\42"), chstr),
+                 InvalidRdataText);
+}
+
+} // unnamed namespace

src/lib/dns/tests/rdata_txt_like_unittest.cc

@@ -17,17 +17,25 @@
 #include <util/buffer.h>
 #include <dns/exceptions.h>
 #include <dns/rdataclass.h>
-#include <gtest/gtest.h>
 
 #include <dns/tests/unittest_util.h>
 #include <dns/tests/rdata_unittest.h>
 
+#include <gtest/gtest.h>
+
+#include <boost/bind.hpp>
+
+#include <string>
+#include <sstream>
+#include <vector>
+
 using isc::UnitTestUtil;
 using namespace std;
 using namespace isc::dns;
 using namespace isc::util;
 using namespace isc::dns::rdata;
 
+namespace {
 
 template<class T>
 class RRTYPE : public RRType {
@@ -38,36 +46,42 @@ public:
 template<> RRTYPE<generic::TXT>::RRTYPE() : RRType(RRType::TXT()) {}
 template<> RRTYPE<generic::SPF>::RRTYPE() : RRType(RRType::SPF()) {}
 
-namespace {
 const uint8_t wiredata_txt_like[] = {
-    sizeof("Test String") - 1,
-    'T', 'e', 's', 't', ' ', 'S', 't', 'r', 'i', 'n', 'g'
+    sizeof("Test-String") - 1,
+    'T', 'e', 's', 't', '-', 'S', 't', 'r', 'i', 'n', 'g'
 };
 
 const uint8_t wiredata_nulltxt[] = { 0 };
-vector<uint8_t> wiredata_longesttxt(256, 'a');
+
+// For lexer-based constructor
+void
+dummyCallback(const string&, size_t, const string&) {
+}
 
 template<class TXT_LIKE>
 class Rdata_TXT_LIKE_Test : public RdataTest {
 protected:
-    Rdata_TXT_LIKE_Test() {
+    Rdata_TXT_LIKE_Test() :
+        callback(boost::bind(&dummyCallback, _1, _2, _3)),
+        loader_cb(callback, callback),
+        wiredata_longesttxt(256, 'a'),
+        rdata_txt_like("Test-String"),
+        rdata_txt_like_empty("\"\""),
+        rdata_txt_like_quoted("\"Test-String\"")
+    {
         wiredata_longesttxt[0] = 255; // adjust length
     }
 
-    static const TXT_LIKE rdata_txt_like;
-    static const TXT_LIKE rdata_txt_like_empty;
-    static const TXT_LIKE rdata_txt_like_quoted;
-};
-
-template<class TXT_LIKE>
-const TXT_LIKE Rdata_TXT_LIKE_Test<TXT_LIKE>::rdata_txt_like("Test String");
-
-template<class TXT_LIKE>
-const TXT_LIKE Rdata_TXT_LIKE_Test<TXT_LIKE>::rdata_txt_like_empty("");
+private:
+    const MasterLoaderCallbacks::IssueCallback callback;
 
-template<class TXT_LIKE>
-const TXT_LIKE Rdata_TXT_LIKE_Test<TXT_LIKE>::rdata_txt_like_quoted
-                                                          ("\"Test String\"");
+protected:
+    MasterLoaderCallbacks loader_cb;
+    vector<uint8_t> wiredata_longesttxt;
+    const TXT_LIKE rdata_txt_like;
+    const TXT_LIKE rdata_txt_like_empty;
+    const TXT_LIKE rdata_txt_like_quoted;
+};
 
 // The list of types we want to test.
 typedef testing::Types<generic::TXT, generic::SPF> Implementations;
@@ -75,37 +89,155 @@ typedef testing::Types<generic::TXT, generic::SPF> Implementations;
 TYPED_TEST_CASE(Rdata_TXT_LIKE_Test, Implementations);
 
 TYPED_TEST(Rdata_TXT_LIKE_Test, createFromText) {
-    // normal case is covered in toWireBuffer.
+    // Below we check the behavior for the "from text" constructors, both
+    // from std::string and with MasterLexer.  The underlying implementation
+    // is the same, so both should work exactly same, but we confirm both
+    // cases.
+
+    const std::string multi_line = "(\n \"Test-String\" )";
+    const std::string escaped_txt = "Test\\045Strin\\g";
+
+    // test input for the lexer version
+    std::stringstream ss;
+    ss << "Test-String\n";
+    ss << "\"Test-String\"\n";   // explicitly surrounded by '"'s
+    ss << multi_line << "\n";   // multi-line text with ()
+    ss << escaped_txt << "\n";   // using the two types of escape with '\'
+    ss << "\"\"\n";              // empty string (note: still valid char-str)
+    ss << string(255, 'a') << "\n"; // Longest possible character-string.
+    ss << string(256, 'a') << "\n"; // char-string too long
+    ss << "\"Test-String\\\"\n";    // unbalanced quote
+    ss << "\"Test-String\\\"\"\n";
+    this->lexer.pushSource(ss);
+
+    // commonly used Rdata to compare below, created from wire
+    ConstRdataPtr const rdata =
+        this->rdataFactoryFromFile(RRTYPE<TypeParam>(),
+                                   RRClass("IN"), "rdata_txt_fromWire1");
+
+    // normal case is covered in toWireBuffer.  First check the std::string
+    // case, then with MasterLexer.  For the latter, we need to read and skip
+    // '\n'.  These apply to most of the other cases below.
+    EXPECT_EQ(0, this->rdata_txt_like.compare(*rdata));
+    EXPECT_EQ(0, TypeParam(this->lexer, NULL, MasterLoader::MANY_ERRORS,
+                           this->loader_cb).compare(*rdata));
+    EXPECT_EQ(MasterToken::END_OF_LINE, this->lexer.getNextToken().getType());
 
     // surrounding double-quotes shouldn't change the result.
-    EXPECT_EQ(0, this->rdata_txt_like.compare(this->rdata_txt_like_quoted));
+    EXPECT_EQ(0, this->rdata_txt_like_quoted.compare(*rdata));
+    EXPECT_EQ(0, TypeParam(this->lexer, NULL, MasterLoader::MANY_ERRORS,
+                           this->loader_cb).compare(*rdata));
+    EXPECT_EQ(MasterToken::END_OF_LINE, this->lexer.getNextToken().getType());
+
+    // multi-line input with ()
+    EXPECT_EQ(0, TypeParam(multi_line).compare(*rdata));
+    EXPECT_EQ(0, TypeParam(this->lexer, NULL, MasterLoader::MANY_ERRORS,
+                           this->loader_cb).compare(*rdata));
+    EXPECT_EQ(MasterToken::END_OF_LINE, this->lexer.getNextToken().getType());
+
+    // for the same data using escape
+    EXPECT_EQ(0, TypeParam(escaped_txt).compare(*rdata));
+    EXPECT_EQ(0, TypeParam(this->lexer, NULL, MasterLoader::MANY_ERRORS,
+                           this->loader_cb).compare(*rdata));
+    EXPECT_EQ(MasterToken::END_OF_LINE, this->lexer.getNextToken().getType());
 
     // Null character-string.
     this->obuffer.clear();
-    TypeParam(string("")).toWire(this->obuffer);
+    TypeParam(string("\"\"")).toWire(this->obuffer);
     EXPECT_PRED_FORMAT4(UnitTestUtil::matchWireData,
-                        this->obuffer.getData(),
-                        this->obuffer.getLength(),
+                        this->obuffer.getData(), this->obuffer.getLength(),
                         wiredata_nulltxt, sizeof(wiredata_nulltxt));
+    this->obuffer.clear();
+    TypeParam(this->lexer, NULL, MasterLoader::MANY_ERRORS, this->loader_cb).
+        toWire(this->obuffer);
+    EXPECT_PRED_FORMAT4(UnitTestUtil::matchWireData,
+                        this->obuffer.getData(), this->obuffer.getLength(),
+                        wiredata_nulltxt, sizeof(wiredata_nulltxt));
+    EXPECT_EQ(MasterToken::END_OF_LINE, this->lexer.getNextToken().getType());
 
     // Longest possible character-string.
     this->obuffer.clear();
     TypeParam(string(255, 'a')).toWire(this->obuffer);
     EXPECT_PRED_FORMAT4(UnitTestUtil::matchWireData,
-                        this->obuffer.getData(),
-                        this->obuffer.getLength(),
-                        &wiredata_longesttxt[0], wiredata_longesttxt.size());
+                        this->obuffer.getData(), this->obuffer.getLength(),
+                        &this->wiredata_longesttxt[0],
+                        this->wiredata_longesttxt.size());
+    this->obuffer.clear();
+    TypeParam(this->lexer, NULL, MasterLoader::MANY_ERRORS, this->loader_cb).
+        toWire(this->obuffer);
+    EXPECT_PRED_FORMAT4(UnitTestUtil::matchWireData,
+                        this->obuffer.getData(), this->obuffer.getLength(),
+                        &this->wiredata_longesttxt[0],
+                        this->wiredata_longesttxt.size());
+    EXPECT_EQ(MasterToken::END_OF_LINE, this->lexer.getNextToken().getType());
 
     // Too long text for a valid character-string.
     EXPECT_THROW(TypeParam(string(256, 'a')), CharStringTooLong);
+    EXPECT_THROW(TypeParam(this->lexer, NULL, MasterLoader::MANY_ERRORS,
+                           this->loader_cb), CharStringTooLong);
+    EXPECT_EQ(MasterToken::END_OF_LINE, this->lexer.getNextToken().getType());
 
     // The escape character makes the double quote a part of character-string,
     // so this is invalid input and should be rejected.
-    EXPECT_THROW(TypeParam("\"Test String\\\""), InvalidRdataText);
+    EXPECT_THROW(TypeParam("\"Test-String\\\""), InvalidRdataText);
+    EXPECT_THROW(TypeParam(this->lexer, NULL, MasterLoader::MANY_ERRORS,
+                           this->loader_cb), MasterLexer::LexerError);
+    EXPECT_EQ(MasterToken::END_OF_LINE, this->lexer.getNextToken().getType());
+}
+
+TYPED_TEST(Rdata_TXT_LIKE_Test, createMultiStringsFromText) {
+    // Tests for "from text" variants construction with various forms of
+    // multi character-strings.
+
+    std::vector<std::string > texts;
+    texts.push_back("\"Test-String\" \"Test-String\""); // most common form
+    texts.push_back("\"Test-String\"\"Test-String\"");  // no space between'em
+    texts.push_back("\"Test-String\" Test-String");  // no '"' for one
+    texts.push_back("\"Test-String\"Test-String"); // and no space either
+    texts.push_back("Test-String \"Test-String\""); // no '"' for the other
+    // This one currently doesn't work
+    //texts.push_back("Test-String\"Test-String\""); // and no space either
+
+    std::stringstream ss;
+    for (std::vector<std::string >::const_iterator it = texts.begin();
+         it != texts.end(); ++it) {
+        ss << *it << "\n";
+    }
+    this->lexer.pushSource(ss);
+
+    // The corresponding Rdata built from wire to compare in the checks below.
+    ConstRdataPtr const rdata =
+        this->rdataFactoryFromFile(RRTYPE<TypeParam>(),
+                                   RRClass("IN"), "rdata_txt_fromWire3.wire");
+
+    // Confirm we can construct the Rdata from the test text, both from
+    // std::string and with lexer, and that matches the from-wire data.
+    for (std::vector<std::string >::const_iterator it = texts.begin();
+         it != texts.end(); ++it) {
+        SCOPED_TRACE(*it);
+        EXPECT_EQ(0, TypeParam(*it).compare(*rdata));
+
+        EXPECT_EQ(0, TypeParam(this->lexer, NULL, MasterLoader::MANY_ERRORS,
+                               this->loader_cb).compare(*rdata));
+        EXPECT_EQ(MasterToken::END_OF_LINE,
+                  this->lexer.getNextToken().getType());
+    }
+}
+
+TYPED_TEST(Rdata_TXT_LIKE_Test, createFromTextExtra) {
+    // This is for the std::string version only: the input must end with EOF;
+    // an extra new-line will result in an exception.
+    EXPECT_THROW(TypeParam("\"Test-String\"\n"), InvalidRdataText);
+    // Same if there's a space before '\n'
+    EXPECT_THROW(TypeParam("\"Test-String\" \n"), InvalidRdataText);
+}
 
-    // Terminating double-quote is provided, so this is valid, but in this
-    // version of implementation we reject escaped characters.
-    EXPECT_THROW(TypeParam("\"Test String\\\"\""), InvalidRdataText);
+TYPED_TEST(Rdata_TXT_LIKE_Test, fromTextEmpty) {
+    // If the input text doesn't contain any character-string, it should be
+    // rejected
+    EXPECT_THROW(TypeParam(""), InvalidRdataText);
+    EXPECT_THROW(TypeParam(" "), InvalidRdataText); // even with a space
+    EXPECT_THROW(TypeParam("(\n)"), InvalidRdataText); // or multi-line with ()
 }
 
 void
@@ -129,13 +261,15 @@ makeLargest(vector<uint8_t>& data) {
 
 TYPED_TEST(Rdata_TXT_LIKE_Test, createFromWire) {
     EXPECT_EQ(0, this->rdata_txt_like.compare(
-                  *this->rdataFactoryFromFile(RRTYPE<TypeParam>(), RRClass("IN"),
-                                        "rdata_txt_fromWire1")));
+                  *this->rdataFactoryFromFile(RRTYPE<TypeParam>(),
+                                              RRClass("IN"),
+                                              "rdata_txt_fromWire1")));
 
     // Empty character string
     EXPECT_EQ(0, this->rdata_txt_like_empty.compare(
-                  *this->rdataFactoryFromFile(RRTYPE<TypeParam>(), RRClass("IN"),
-                                        "rdata_txt_fromWire2.wire")));
+                  *this->rdataFactoryFromFile(RRTYPE<TypeParam>(),
+                                              RRClass("IN"),
+                                              "rdata_txt_fromWire2.wire")));
 
     // Multiple character strings
     this->obuffer.clear();
@@ -188,7 +322,7 @@ TYPED_TEST(Rdata_TXT_LIKE_Test, createFromWire) {
 TYPED_TEST(Rdata_TXT_LIKE_Test, createFromLexer) {
     EXPECT_EQ(0, this->rdata_txt_like.compare(
         *test::createRdataUsingLexer(RRTYPE<TypeParam>(), RRClass::IN(),
-                                     "Test String")));
+                                     "Test-String")));
 }
 
 TYPED_TEST(Rdata_TXT_LIKE_Test, toWireBuffer) {
@@ -208,7 +342,7 @@ TYPED_TEST(Rdata_TXT_LIKE_Test, toWireRenderer) {
 }
 
 TYPED_TEST(Rdata_TXT_LIKE_Test, toText) {
-    EXPECT_EQ("\"Test String\"", this->rdata_txt_like.toText());
+    EXPECT_EQ("\"Test-String\"", this->rdata_txt_like.toText());
 }
 
 TYPED_TEST(Rdata_TXT_LIKE_Test, assignment) {
@@ -238,8 +372,8 @@ TYPED_TEST(Rdata_TXT_LIKE_Test, compare) {
 
     EXPECT_EQ(TypeParam(txt1).compare(TypeParam(txt1)), 0);
 
-    EXPECT_LT(TypeParam("").compare(TypeParam(txt1)), 0);
-    EXPECT_GT(TypeParam(txt1).compare(TypeParam("")), 0);
+    EXPECT_LT(TypeParam("\"\"").compare(TypeParam(txt1)), 0);
+    EXPECT_GT(TypeParam(txt1).compare(TypeParam("\"\"")), 0);
 
     EXPECT_LT(TypeParam(txt1).compare(TypeParam(txt2)), 0);
     EXPECT_GT(TypeParam(txt2).compare(TypeParam(txt1)), 0);

src/lib/dns/tests/rdata_unittest.cc

@@ -29,6 +29,7 @@
 #include <dns/tests/rdata_unittest.h>
 
 #include <boost/bind.hpp>
+#include <boost/lexical_cast.hpp>
 
 using isc::UnitTestUtil;
 using namespace std;
@@ -82,6 +83,138 @@ createRdataUsingLexer(const RRType& rrtype, const RRClass& rrclass,
 
 } // end of namespace isc::dns::rdata::test
 
+// A mock class to check parameters passed via loader callbacks.  Its callback
+// records the passed parameters, allowing the test to check them later via
+// the check() method.
+class CreateRdataCallback {
+public:
+    enum CallbackType { NONE, ERROR, WARN };
+    CreateRdataCallback() : type_(NONE), line_(0) {}
+    void callback(CallbackType type, const string& source, size_t line,
+                  const string& reason_txt) {
+        type_ = type;
+        source_ = source;
+        line_ = line;
+        reason_txt_ = reason_txt;
+    }
+
+    void clear() {
+        type_ = NONE;
+        source_.clear();
+        line_ = 0;
+        reason_txt_.clear();
+    }
+
+    // Return if callback is called since the previous call to clear().
+    bool isCalled() const { return (type_ != NONE); }
+
+    void check(const string& expected_srcname, size_t expected_line,
+               CallbackType expected_type, const string& expected_reason)
+        const
+    {
+        EXPECT_EQ(expected_srcname, source_);
+        EXPECT_EQ(expected_line, line_);
+        EXPECT_EQ(expected_type, type_);
+        EXPECT_EQ(expected_reason, reason_txt_);
+    }
+
+private:
+    CallbackType type_;
+    string source_;
+    size_t line_;
+    string reason_txt_;
+};
+
+// Test class/type-independent behavior of createRdata().
+TEST_F(RdataTest, createRdataWithLexer) {
+    const in::AAAA aaaa_rdata("2001:db8::1");
+
+    stringstream ss;
+    const string src_name = "stream-" + boost::lexical_cast<string>(&ss);
+    ss << aaaa_rdata.toText() << "\n"; // valid case
+    ss << aaaa_rdata.toText() << "; comment, should be ignored\n";
+    ss << aaaa_rdata.toText() << " extra-token\n"; // extra token
+    ss << aaaa_rdata.toText() << " extra token\n"; // 2 extra tokens
+    ss << ")\n"; // causing lexer error in parsing the RDATA text
+    ss << "192.0.2.1\n"; // semantics error: IPv4 address is given for AAAA
+    ss << aaaa_rdata.toText();  // valid, but end with EOF, not EOL
+    lexer.pushSource(ss);
+
+    CreateRdataCallback callback;
+    MasterLoaderCallbacks callbacks(
+        boost::bind(&CreateRdataCallback::callback, &callback,
+                    CreateRdataCallback::ERROR, _1, _2, _3),
+        boost::bind(&CreateRdataCallback::callback, &callback,
+                    CreateRdataCallback::WARN,  _1, _2, _3));
+
+    size_t line = 0;
+
+    // Valid case.
+    ++line;
+    ConstRdataPtr rdata = createRdata(RRType::AAAA(), RRClass::IN(), lexer,
+                                      NULL, MasterLoader::MANY_ERRORS,
+                                      callbacks);
+    EXPECT_EQ(0, aaaa_rdata.compare(*rdata));
+    EXPECT_FALSE(callback.isCalled());
+
+    // Similar to the previous case, but RDATA is followed by a comment.
+    // It should cause any confusion.
+    ++line;
+    callback.clear();
+    rdata = createRdata(RRType::AAAA(), RRClass::IN(), lexer, NULL,
+                        MasterLoader::MANY_ERRORS, callbacks);
+    EXPECT_EQ(0, aaaa_rdata.compare(*rdata));
+    EXPECT_FALSE(callback.isCalled());
+
+    // Broken RDATA text: extra token.  createRdata() returns NULL, error
+    // callback is called.
+    ++line;
+    callback.clear();
+    EXPECT_FALSE(createRdata(RRType::AAAA(), RRClass::IN(), lexer, NULL,
+                             MasterLoader::MANY_ERRORS, callbacks));
+    callback.check(src_name, line, CreateRdataCallback::ERROR,
+                   "createRdata from text failed near 'extra-token': "
+                   "extra input text");
+
+    // Similar to the previous case, but only the first extra token triggers
+    // callback.
+    ++line;
+    callback.clear();
+    EXPECT_FALSE(createRdata(RRType::AAAA(), RRClass::IN(), lexer, NULL,
+                             MasterLoader::MANY_ERRORS, callbacks));
+    callback.check(src_name, line, CreateRdataCallback::ERROR,
+                   "createRdata from text failed near 'extra': "
+                   "extra input text");
+
+    // Lexer error will happen, corresponding error callback will be triggered.
+    ++line;
+    callback.clear();
+    EXPECT_FALSE(createRdata(RRType::AAAA(), RRClass::IN(), lexer, NULL,
+                             MasterLoader::MANY_ERRORS, callbacks));
+    callback.check(src_name, line, CreateRdataCallback::ERROR,
+                   "createRdata from text failed: unbalanced parentheses");
+
+    // Semantics level error will happen, corresponding error callback will be
+    // triggered.
+    ++line;
+    callback.clear();
+    EXPECT_FALSE(createRdata(RRType::AAAA(), RRClass::IN(), lexer, NULL,
+                             MasterLoader::MANY_ERRORS, callbacks));
+    callback.check(src_name, line, CreateRdataCallback::ERROR,
+                   "createRdata from text failed: Failed to convert "
+                   "'192.0.2.1' to IN/AAAA RDATA");
+
+    // Input is valid and parse will succeed, but with a warning that the
+    // file is not ended with a newline.
+    ++line;
+    callback.clear();
+    rdata = createRdata(RRType::AAAA(), RRClass::IN(), lexer, NULL,
+                        MasterLoader::MANY_ERRORS, callbacks);
+    EXPECT_EQ(0, aaaa_rdata.compare(*rdata));
+    callback.check(src_name, line, CreateRdataCallback::WARN,
+                   "file does not end with newline");
+}
+
 }
 }
 }

src/lib/dns/tests/rdata_unittest.h

@@ -20,6 +20,7 @@
 #include <dns/rrclass.h>
 #include <dns/rrtype.h>
 #include <dns/rdata.h>
+#include <dns/master_lexer.h>
 
 #include <gtest/gtest.h>
 
@@ -40,6 +41,7 @@ protected:
     /// This is an RDATA object of some "unknown" RR type so that it can be
     /// used to test the compare() method against a well-known RR type.
     RdataPtr rdata_nomatch;
+    MasterLexer lexer;
 };
 
 namespace test {

src/lib/dns/tests/testdata/rdata_txt_fromWire1

@@ -1,9 +1,9 @@
 #
 # various kinds of TXT RDATA stored in an input buffer
 #
-# Valid RDATA for "Test String"
+# Valid RDATA for "Test-String"
 #
 # RDLENGHT=12 bytes
  00 0c
-#    T  e  s  t     S  t  r  i  n  g
- 0b 54 65 73 74 20 53 74 72 69 6e 67
+#    T  e  s  t  -  S  t  r  i  n  g
+ 0b 54 65 73 74 2d 53 74 72 69 6e 67

src/lib/util/python/gen_wiredata.py.in

@@ -770,7 +770,7 @@ class TXT(RR):
 
     nstring = 1
     stringlen = None
-    string = 'Test String'
+    string = 'Test-String'
 
     def dump(self, f):
         stringlen_list = []

tests/tools/perfdhcp/Makefile.am

@@ -1,4 +1,4 @@
-SUBDIRS = . tests templates
+SUBDIRS = . tests
 
 AM_CPPFLAGS = -I$(top_srcdir)/src/lib -I$(top_builddir)/src/lib
 AM_CPPFLAGS += -I$(top_srcdir)/src/lib/log -I$(top_builddir)/src/lib/log

tests/tools/perfdhcp/tests/Makefile.am

@@ -1,6 +1,7 @@
-SUBDIRS = .
+SUBDIRS = . testdata
 
 AM_CPPFLAGS = -I$(top_builddir)/src/lib -I$(top_srcdir)/src/lib
+AM_CPPFLAGS += -DTEST_DATA_DIR=\"$(abs_srcdir)/testdata\"
 AM_CPPFLAGS += $(BOOST_INCLUDES)
 AM_CXXFLAGS = $(B10_CXXFLAGS)
 

tests/tools/perfdhcp/tests/test_control_unittest.cc

@@ -185,6 +185,18 @@ public:
         return ("");
     }
 
+    /// \brief Get full path to a file in testdata directory.
+    ///
+    /// \param filename filename being appended to absolute
+    /// path to testdata directory
+    ///
+    /// \return full path to a file in testdata directory.
+    std::string getFullPath(const std::string& filename) const {
+        std::ostringstream stream;
+        stream << TEST_DATA_DIR << "/" << filename;
+        return (stream.str());
+    }
+
     /// \brief Match requested options in the buffer with given list.
     ///
     /// This method iterates through options provided in the buffer
@@ -896,7 +908,7 @@ TEST_F(TestControlTest, Packet6) {
     }
 }
 
-TEST_F(TestControlTest, DISABLED_Packet4Exchange) {
+TEST_F(TestControlTest, Packet4Exchange) {
     // Get the local loopback interface to open socket on
     // it and test packets exchanges. We don't want to fail
     // the test if interface is not available.
@@ -925,8 +937,8 @@ TEST_F(TestControlTest, DISABLED_Packet4Exchange) {
     // Use templates for this test.
     processCmdLine("perfdhcp -l " + loopback_iface
                    + " -r 100 -R 20 -n 20 -D 10% -L 10547"
-                   + " -T ../templates/discover-example.hex"
-                   + " -T ../templates/request4-example.hex"
+                   + " -T " + getFullPath("discover-example.hex")
+                   + " -T " + getFullPath("request4-example.hex")
                    + " 127.0.0.1");
     // The number iterations is restricted by the percentage of
     // dropped packets (-D 10%). We also have to bump up the number
@@ -939,7 +951,7 @@ TEST_F(TestControlTest, DISABLED_Packet4Exchange) {
     EXPECT_EQ(12, iterations_performed);
 }
 
-TEST_F(TestControlTest, DISABLED_Packet6Exchange) {
+TEST_F(TestControlTest, Packet6Exchange) {
     // Get the local loopback interface to open socket on
     // it and test packets exchanges. We don't want to fail
     // the test if interface is not available.
@@ -967,8 +979,8 @@ TEST_F(TestControlTest, DISABLED_Packet6Exchange) {
     use_templates = true;
     processCmdLine("perfdhcp -l " + loopback_iface
                    + " -6 -r 100 -n 10 -R 20 -D 3 -L 10547"
-                   + " -T ../templates/solicit-example.hex"
-                   + " -T ../templates/request6-example.hex ::1");
+                   + " -T " + getFullPath("solicit-example.hex")
+                   + " -T " + getFullPath("request6-example.hex ::1"));
     // For the first 3 packets we are simulating responses from server.
     // For other packets we don't so packet as 4,5,6 will be dropped and
     // then test should be interrupted and actual number of iterations will
@@ -981,9 +993,9 @@ TEST_F(TestControlTest, DISABLED_Packet6Exchange) {
 
 TEST_F(TestControlTest, PacketTemplates) {
     std::vector<uint8_t> template1(256);
-    std::string file1("../templates/test1.hex");
+    std::string file1(getFullPath("test1.hex"));
     std::vector<uint8_t> template2(233);
-    std::string file2("../templates/test2.hex");
+    std::string file2(getFullPath("test2.hex"));
     for (int i = 0; i < template1.size(); ++i) {
         template1[i] = static_cast<uint8_t>(random() % 256);
     }
@@ -1011,7 +1023,7 @@ TEST_F(TestControlTest, PacketTemplates) {
     EXPECT_TRUE(std::equal(template2.begin(), template2.end(), buf2.begin()));
 
     // Try to read template file with odd number of digits.
-    std::string file3("../templates/test3.hex");
+    std::string file3(getFullPath("test3.hex"));
     // Size of the file is 2 times larger than binary data size and it is always
     // even number. Substracting 1 makes file size odd.
     ASSERT_TRUE(createTemplateFile(file3, template1, template1.size() * 2 - 1));
@@ -1021,7 +1033,7 @@ TEST_F(TestControlTest, PacketTemplates) {
     EXPECT_THROW(tc.initPacketTemplates(), isc::OutOfRange);
 
     // Try to read empty file.
-    std::string file4("../templates/test4.hex");
+    std::string file4(getFullPath("test4.hex"));
     ASSERT_TRUE(createTemplateFile(file4, template2, 0));
     ASSERT_NO_THROW(
         processCmdLine("perfdhcp -l 127.0.0.1 -T " + file4 + " all")
@@ -1029,7 +1041,7 @@ TEST_F(TestControlTest, PacketTemplates) {
     EXPECT_THROW(tc.initPacketTemplates(), isc::OutOfRange);
 
     // Try reading file with non hexadecimal characters.
-    std::string file5("../templates/test5.hex");
+    std::string file5(getFullPath("test5.hex"));
     ASSERT_TRUE(createTemplateFile(file5, template1, template1.size() * 2, true));
     ASSERT_NO_THROW(
         processCmdLine("perfdhcp -l 127.0.0.1 -T " + file5 + " all")

tests/tools/perfdhcp/templates/Makefile.am

@@ -4,7 +4,5 @@ SUBDIRS = .
 # unit tests and have to be removed.
 CLEANFILES = test1.hex test2.hex test3.hex test4.hex test5.hex
 
-perfdhcpdir = $(pkgdatadir)
-
 EXTRA_DIST = discover-example.hex request4-example.hex
 EXTRA_DIST += solicit-example.hex request6-example.hex