[5014] Added scanner include

src/bin/dhcp6/dhcp6_lexer.ll

@@ -21,13 +21,29 @@
 # undef yywrap
 # define yywrap() 1
 
+namespace {
+
 // The location of the current token. The lexer will keep updating it. This
 // variable will be useful for logging errors.
-static isc::dhcp::location loc;
+isc::dhcp::location loc;
+
+/// @brief Location stack.
+std::vector<isc::dhcp::location> locs;
+
+/// @brief File name.
+std::string file;
 
-static bool start_token_flag = false;
+/// @brief File name stack.
+std::vector<std::string> files;
 
-static isc::dhcp::Parser6Context::ParserType start_token_value;
+/// @brief State stack.
+std::vector<struct yy_buffer_state*> states;
+
+bool start_token_flag = false;
+
+isc::dhcp::Parser6Context::ParserType start_token_value;
+
+};
 
 // To avoid the call to exit... oops!
 #define YY_FATAL_ERROR(msg) isc::dhcp::Parser6Context::fatal(msg)
@@ -61,6 +77,7 @@ static isc::dhcp::Parser6Context::ParserType start_token_value;
 %option yylineno
 
 %x COMMENT
+%x DIR_ENTER DIR_INCLUDE DIR_EXIT
 
 /* These are not token expressions yet, just convenience expressions that
    can be used during actual token definitions. Note some can match
@@ -121,6 +138,20 @@ JSONString                              \"{JSONStringCharacter}*\"
     isc_throw(isc::BadValue, "Comment not closed. (/* in line " << comment_start_line);
 }
 
+"<?" BEGIN(DIR_ENTER);
+<DIR_ENTER>"include" BEGIN(DIR_INCLUDE);
+<DIR_INCLUDE>\"([^\"\n])+\" {
+    // Include directive.
+
+    // Extract the filename.
+    std::string tmp(yytext+1);
+    tmp.resize(tmp.size() - 1);
+
+    Parser6Context::includeFile(tmp);
+}
+<DIR_EXIT>"?>" BEGIN(INITIAL);
+    
+
 {blank}+   {
     // Ok, we found a with space. Let's ignore it and update loc variable.
     loc.step();
@@ -269,7 +300,21 @@ null {
 }
 
 .          driver.error (loc, "Invalid character: " + std::string(yytext));
-<<EOF>>    return isc::dhcp::Dhcp6Parser::make_END(loc);
+<<EOF>>    {
+    if (states.empty()) {
+        return isc::dhcp::Dhcp6Parser::make_END(loc);
+    }
+    loc = locs.back();
+    locs.pop_back();
+    file = files.back();
+    files.pop_back();
+    parser6__delete_buffer(YY_CURRENT_BUFFER);
+    parser6__switch_to_buffer(states.back());
+    states.pop_back();
+
+    BEGIN(DIR_EXIT);
+}
+
 %%
 
 using namespace isc::dhcp;
@@ -280,7 +325,8 @@ Parser6Context::scanStringBegin(const std::string& str, ParserType parser_type)
     start_token_flag = true;
     start_token_value = parser_type;
 
-    loc.initialize(&file_);
+    file = "<string>";
+    loc.initialize(&file);
     yy_flex_debug = trace_scanning_;
     YY_BUFFER_STATE buffer;
     buffer = yy_scan_bytes(str.c_str(), str.size());
@@ -297,19 +343,22 @@ Parser6Context::scanStringEnd()
 }
 
 void
-Parser6Context::scanFileBegin(FILE * f, ParserType parser_type) {
+Parser6Context::scanFileBegin(FILE * f,
+                              const std::string& filename,
+                              ParserType parser_type) {
 
     start_token_flag = true;
     start_token_value = parser_type;
 
-    loc.initialize(&file_);
+    file = filename;
+    loc.initialize(&file);
     yy_flex_debug = trace_scanning_;
     YY_BUFFER_STATE buffer;
 
     // See dhcp6_lexer.cc header for available definitions
     buffer = parser6__create_buffer(f, 65536 /*buffer size*/);
     if (!buffer) {
-        fatal("cannot scan file " + file_);
+        fatal("cannot scan file " + filename);
     }
     parser6__switch_to_buffer(buffer);
 }
@@ -322,7 +371,27 @@ Parser6Context::scanFileEnd(FILE * f) {
 
 void
 Parser6Context::includeFile(const std::string& filename) {
-    fprintf(stderr, "includeFile(\"%s\")\n", filename.c_str());
+    if (states.size() > 10) {
+        fatal("Too many nested include.");
+    }
+
+    FILE* f = fopen(filename.c_str(), "r");
+    if (!f) {
+        fatal("Can't open include file " + filename);
+    }
+    states.push_back(YY_CURRENT_BUFFER);
+    YY_BUFFER_STATE buffer;
+    buffer = parser6__create_buffer(f, 65536 /*buffer size*/);
+    if (!buffer) {
+        fatal( "Can't scan include file " + filename);
+    }
+    parser6__switch_to_buffer(buffer);
+    files.push_back(file);
+    file = filename;
+    locs.push_back(loc);
+    loc.initialize(&file);
+
+    BEGIN(INITIAL);
 }
 
 namespace {

src/bin/dhcp6/parser_context.cc

@@ -27,7 +27,6 @@ Parser6Context::~Parser6Context()
 isc::data::ConstElementPtr
 Parser6Context::parseString(const std::string& str, ParserType parser_type)
 {
-    file_ = "<string>";
     scanStringBegin(str, parser_type);
     isc::dhcp::Dhcp6Parser parser(*this);
     // Uncomment this to get detailed parser logs.
@@ -52,8 +51,7 @@ Parser6Context::parseFile(const std::string& filename, ParserType parser_type) {
     if (!f) {
         isc_throw(BadValue, "Unable to open file " << filename);
     }
-    file_ = filename;
-    scanFileBegin(f, parser_type);
+    scanFileBegin(f, filename, parser_type);
 
     isc::dhcp::Dhcp6Parser parser(*this);
     // Uncomment this to get detailed parser logs.

src/bin/dhcp6/parser_context.h

@@ -57,11 +57,14 @@ public:
     void scanStringEnd();
 
     /// @brief Method called before scanning starts on a file.
-    void scanFileBegin(FILE * f, ParserType type);
+    void scanFileBegin(FILE * f, const std::string& filename, ParserType type);
 
     /// @brief Method called after the last tokens are scanned from a file.
     void scanFileEnd(FILE * f);
 
+    /// @brief Divert input to an include file.
+    static void includeFile(const std::string& filename);
+
     /// @brief Run the parser on the string specified.
     ///
     /// @param str string to be parsed
@@ -74,10 +77,6 @@ public:
     isc::data::ConstElementPtr parseFile(const std::string& filename,
                                          ParserType parser_type);
 
-    /// @brief The name of the file being parsed.
-    /// Used later to pass the file name to the location tracker.
-    std::string file_;
-
     /// @brief Error handler
     ///
     /// @param loc location within the parsed file when experienced a problem.
@@ -96,18 +95,6 @@ public:
     static void fatal(const std::string& what);
 
  private:
-    /// @brief Divert input to an include file.
-    void includeFile(const std::string& filename);
-
-    /// @brief File name stack.
-    std::vector<std::string> files_;
-
-    /// @brief Location stack.
-    std::vector<isc::dhcp::location> locs_;
-
-    /// @brief State stack.
-    std::vector<struct yy_buffer_state*> states_;
-
     /// @brief Flag determining scanner debugging.
     bool trace_scanning_;
 

src/bin/dhcp6/parser_context_decl.h

@@ -7,14 +7,14 @@
 #ifndef PARSER6_CONTEXT_DECL_H
 #define PARSER6_CONTEXT_DECL_H
 
-/// @file eval_context_decl.h Forward declaration of the EvalContext class
+/// @file parser_context_decl.h Forward declaration of the ParserContext class
 
 namespace isc {
 namespace dhcp {
 
 class Parser6Context;
 
-}; // end of isc::eval namespace
+}; // end of isc::dhcp namespace
 }; // end of isc namespace
 
 #endif

src/bin/dhcp6/tests/kea_controller_unittest.cc

@@ -59,6 +59,7 @@ public:
         LeaseMgrFactory::destroy();
         isc::log::setDefaultLoggingOutput();
         static_cast<void>(remove(TEST_FILE));
+        static_cast<void>(remove(TEST_INCLUDE));
     };
 
     void writeFile(const std::string& file_name, const std::string& content) {
@@ -86,9 +87,11 @@ public:
     }
 
     static const char* TEST_FILE;
+    static const char* TEST_INCLUDE;
 };
 
 const char* JSONFileBackendTest::TEST_FILE = "test-config.json";
+const char* JSONFileBackendTest::TEST_INCLUDE = "test-include.json";
 
 // This test checks if configuration can be read from a JSON file.
 TEST_F(JSONFileBackendTest, jsonFile) {
@@ -167,8 +170,9 @@ TEST_F(JSONFileBackendTest, jsonFile) {
     EXPECT_EQ(Lease::TYPE_NA, pools3.at(0)->getType());
 }
 
-// This test checks if configuration can be read from a JSON file.
-TEST_F(JSONFileBackendTest, comments) {
+// This test checks if configuration can be read from a JSON file
+// using hash (#) line comments
+TEST_F(JSONFileBackendTest, hashComments) {
 
     string config_hash_comments = "# This is a comment. It should be \n"
         "#ignored. Real config starts in line below\n"
@@ -187,9 +191,6 @@ TEST_F(JSONFileBackendTest, comments) {
         "\"valid-lifetime\": 4000 }"
         "}";
 
-    /// @todo: Implement C++-style (// ...) comments
-    /// @todo: Implement C-style (/* ... */) comments
-
     writeFile(TEST_FILE, config_hash_comments);
 
     // Now initialize the server
@@ -219,6 +220,157 @@ TEST_F(JSONFileBackendTest, comments) {
     EXPECT_EQ(Lease::TYPE_NA, pools1.at(0)->getType());
 }
 
+// This test checks if configuration can be read from a JSON file
+// using C++ line (//) comments.
+TEST_F(JSONFileBackendTest, cppLineComments) {
+
+    string config_cpp_line_comments = "// This is a comment. It should be \n"
+        "//ignored. Real config starts in line below\n"
+        "{ \"Dhcp6\": {"
+        "\"interfaces-config\": {"
+        "  \"interfaces\": [ \"*\" ]"
+        "},"
+        "\"preferred-lifetime\": 3000,"
+        "\"rebind-timer\": 2000, "
+        "\"renew-timer\": 1000, \n"
+        "// comments in the middle should be ignored, too\n"
+        "\"subnet6\": [ { "
+        "    \"pools\": [ { \"pool\": \"2001:db8:1::/80\" } ],"
+        "    \"subnet\": \"2001:db8:1::/64\" "
+        " } ],"
+        "\"valid-lifetime\": 4000 }"
+        "}";
+
+    writeFile(TEST_FILE, config_cpp_line_comments);
+
+    // Now initialize the server
+    boost::scoped_ptr<ControlledDhcpv6Srv> srv;
+    ASSERT_NO_THROW(
+        srv.reset(new ControlledDhcpv6Srv(0))
+    );
+
+    // And configure it using config without
+    EXPECT_NO_THROW(srv->init(TEST_FILE));
+
+    // Now check if the configuration has been applied correctly.
+    const Subnet6Collection* subnets =
+        CfgMgr::instance().getCurrentCfg()->getCfgSubnets6()->getAll();
+    ASSERT_TRUE(subnets);
+    ASSERT_EQ(1, subnets->size());
+
+    // Check subnet 1.
+    EXPECT_EQ("2001:db8:1::", subnets->at(0)->get().first.toText());
+    EXPECT_EQ(64, subnets->at(0)->get().second);
+
+    // Check pools in the first subnet.
+    const PoolCollection& pools1 = subnets->at(0)->getPools(Lease::TYPE_NA);
+    ASSERT_EQ(1, pools1.size());
+    EXPECT_EQ("2001:db8:1::", pools1.at(0)->getFirstAddress().toText());
+    EXPECT_EQ("2001:db8:1::ffff:ffff:ffff", pools1.at(0)->getLastAddress().toText());
+    EXPECT_EQ(Lease::TYPE_NA, pools1.at(0)->getType());
+}
+
+// This test checks if configuration can be read from a JSON file
+// using C block (/* */) comments
+TEST_F(JSONFileBackendTest, cBlockComments) {
+
+    string config_c_block_comments = "/* This is a comment. It should be \n"
+        "ignored. Real config starts in line below*/\n"
+        "{ \"Dhcp6\": {"
+        "\"interfaces-config\": {"
+        "  \"interfaces\": [ \"*\" ]"
+        "},"
+        "\"preferred-lifetime\": 3000,"
+        "\"rebind-timer\": 2000, "
+        "\"renew-timer\": 1000, \n"
+        "/* comments in the middle should be ignored, too*/\n"
+        "\"subnet6\": [ { "
+        "    \"pools\": [ { \"pool\": \"2001:db8:1::/80\" } ],"
+        "    \"subnet\": \"2001:db8:1::/64\" "
+        " } ],"
+        "\"valid-lifetime\": 4000 }"
+        "}";
+
+      writeFile(TEST_FILE, config_c_block_comments);
+
+    // Now initialize the server
+    boost::scoped_ptr<ControlledDhcpv6Srv> srv;
+    ASSERT_NO_THROW(
+        srv.reset(new ControlledDhcpv6Srv(0))
+    );
+
+    // And configure it using config without
+    EXPECT_NO_THROW(srv->init(TEST_FILE));
+
+    // Now check if the configuration has been applied correctly.
+    const Subnet6Collection* subnets =
+        CfgMgr::instance().getCurrentCfg()->getCfgSubnets6()->getAll();
+    ASSERT_TRUE(subnets);
+    ASSERT_EQ(1, subnets->size());
+
+    // Check subnet 1.
+    EXPECT_EQ("2001:db8:1::", subnets->at(0)->get().first.toText());
+    EXPECT_EQ(64, subnets->at(0)->get().second);
+
+    // Check pools in the first subnet.
+    const PoolCollection& pools1 = subnets->at(0)->getPools(Lease::TYPE_NA);
+    ASSERT_EQ(1, pools1.size());
+    EXPECT_EQ("2001:db8:1::", pools1.at(0)->getFirstAddress().toText());
+    EXPECT_EQ("2001:db8:1::ffff:ffff:ffff", pools1.at(0)->getLastAddress().toText());
+    EXPECT_EQ(Lease::TYPE_NA, pools1.at(0)->getType());
+}
+
+// This test checks if configuration can be read from a JSON file
+// using an include file.
+TEST_F(JSONFileBackendTest, include) {
+
+    string config = "{ \"Dhcp6\": {"
+        "\"interfaces-config\": {"
+        "  \"interfaces\": [ \"*\" ]"
+        "},"
+        "\"preferred-lifetime\": 3000,"
+        "\"rebind-timer\": 2000, "
+        "\"renew-timer\": 1000, \n"
+        "<?include \"" + string(TEST_INCLUDE) + "\"?>,"
+        "\"valid-lifetime\": 4000 }"
+        "}";
+    string include = "\n"
+        "\"subnet6\": [ { "
+        "    \"pools\": [ { \"pool\": \"2001:db8:1::/80\" } ],"
+        "    \"subnet\": \"2001:db8:1::/64\" "
+        " } ]\n";
+
+    writeFile(TEST_FILE, config);
+    writeFile(TEST_INCLUDE, include);
+
+    // Now initialize the server
+    boost::scoped_ptr<ControlledDhcpv6Srv> srv;
+    ASSERT_NO_THROW(
+        srv.reset(new ControlledDhcpv6Srv(0))
+    );
+
+    // And configure it using config without
+    EXPECT_NO_THROW(srv->init(TEST_FILE));
+
+    // Now check if the configuration has been applied correctly.
+    const Subnet6Collection* subnets =
+        CfgMgr::instance().getCurrentCfg()->getCfgSubnets6()->getAll();
+    ASSERT_TRUE(subnets);
+    ASSERT_EQ(1, subnets->size());
+
+    // Check subnet 1.
+    EXPECT_EQ("2001:db8:1::", subnets->at(0)->get().first.toText());
+    EXPECT_EQ(64, subnets->at(0)->get().second);
+
+    // Check pools in the first subnet.
+    const PoolCollection& pools1 = subnets->at(0)->getPools(Lease::TYPE_NA);
+    ASSERT_EQ(1, pools1.size());
+    EXPECT_EQ("2001:db8:1::", pools1.at(0)->getFirstAddress().toText());
+    EXPECT_EQ("2001:db8:1::ffff:ffff:ffff", pools1.at(0)->getLastAddress().toText());
+    EXPECT_EQ(Lease::TYPE_NA, pools1.at(0)->getType());
+}
+
+// This test checks if configuration can be read from a JSON file.
 // This test checks if configuration detects failure when trying:
 // - empty file
 // - empty filename