[2850] Define and implement ZoneTableSegmentMapped

src/lib/datasrc/memory/Makefile.am

@@ -22,6 +22,7 @@ libdatasrc_memory_la_SOURCES += zone_table.h zone_table.cc
 libdatasrc_memory_la_SOURCES += zone_finder.h zone_finder.cc
 libdatasrc_memory_la_SOURCES += zone_table_segment.h zone_table_segment.cc
 libdatasrc_memory_la_SOURCES += zone_table_segment_local.h zone_table_segment_local.cc
+libdatasrc_memory_la_SOURCES += zone_table_segment_mapped.h zone_table_segment_mapped.cc
 libdatasrc_memory_la_SOURCES += zone_data_updater.h zone_data_updater.cc
 libdatasrc_memory_la_SOURCES += zone_data_loader.h zone_data_loader.cc
 libdatasrc_memory_la_SOURCES += memory_client.h memory_client.cc

src/lib/datasrc/memory/zone_table_segment.cc

@@ -14,6 +14,7 @@
 
 #include <datasrc/memory/zone_table_segment.h>
 #include <datasrc/memory/zone_table_segment_local.h>
+#include <datasrc/memory/zone_table_segment_mapped.h>
 #include <datasrc/memory/zone_writer.h>
 
 #include <string>
@@ -31,6 +32,8 @@ ZoneTableSegment::create(const RRClass& rrclass, const std::string& type) {
     // Until that it becomes a real issue we won't be too smart.
     if (type == "local") {
         return (new ZoneTableSegmentLocal(rrclass));
+    } else if (type == "mapped") {
+        return (new ZoneTableSegmentMapped(rrclass));
     }
     isc_throw(UnknownSegmentType, "Zone table segment type not supported: "
               << type);
@@ -46,6 +49,11 @@ ZoneTableSegment::getZoneWriter(const LoadAction& load_action,
                                 const dns::Name& name,
                                 const dns::RRClass& rrclass)
 {
+    if (!isWritable()) {
+        isc_throw(isc::Unexpected,
+                  "getZoneWriter() called on a read-only segment");
+    }
+
     return (new ZoneWriter(this, load_action, name, rrclass));
 }
 

src/lib/datasrc/memory/zone_table_segment_mapped.cc

@@ -0,0 +1,222 @@
+// Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <datasrc/memory/zone_table_segment_mapped.h>
+
+#include <memory>
+
+using namespace isc::data;
+using namespace isc::dns;
+using namespace isc::util;
+
+namespace isc {
+namespace datasrc {
+namespace memory {
+
+ZoneTableSegmentMapped::ZoneTableSegmentMapped(const RRClass& rrclass) :
+    ZoneTableSegment(rrclass),
+    rrclass_(rrclass),
+    header_(NULL)
+{
+}
+
+ZoneTableSegmentMapped::~ZoneTableSegmentMapped() {
+}
+
+void
+ZoneTableSegmentMapped::reset(MemorySegmentOpenMode mode,
+                              isc::data::ConstElementPtr params)
+{
+    if (mem_sgmt_) {
+        if (isWritable()) {
+            // If there is a previously opened segment, and it was
+            // opened in read-write mode, update its checksum.
+            mem_sgmt_->shrinkToFit();
+            uint32_t* checksum = static_cast<uint32_t*>
+                (mem_sgmt_->getNamedAddress("zone_table_checksum"));
+            // First, clear the checksum so that getCheckSum() returns
+            // a consistent value.
+            *checksum = 0;
+            const uint32_t new_checksum = mem_sgmt_->getCheckSum();
+            // Now, update it into place.
+            *checksum = new_checksum;
+        }
+        // Close the segment here in case the code further below
+        // doesn't complete successfully.
+        header_ = NULL;
+        mem_sgmt_.reset();
+    }
+
+    if (!params || params->getType() != Element::map) {
+        isc_throw(isc::InvalidParameter,
+                  "Configuration does not contain a map");
+    }
+
+    if (!params->contains("mapped-file")) {
+        isc_throw(isc::InvalidParameter,
+                  "Configuration does not contain a \"mapped-file\" key");
+    }
+
+    ConstElementPtr mapped_file = params->get("mapped-file");
+    if ((!mapped_file) || (mapped_file->getType() != Element::string)) {
+        isc_throw(isc::InvalidParameter,
+                  "Value of \"mapped-file\" is not a string");
+    }
+
+    const std::string filename = mapped_file->stringValue();
+
+    // In case there is a checksum mismatch, we throw. We want the
+    // segment to be automatically destroyed then.
+    std::auto_ptr<MemorySegmentMapped> segment;
+
+    switch (mode) {
+    case CREATE: {
+        segment.reset(new MemorySegmentMapped
+                      (filename,
+                       MemorySegmentMapped::CREATE_ONLY));
+        // There must be no previously saved checksum.
+        if (segment->getNamedAddress("zone_table_checksum")) {
+            isc_throw(isc::Unexpected,
+                      "There is already a saved checksum in a mapped segment "
+                      "opened in create mode.");
+        }
+        // Allocate space for a checksum (which is saved during close).
+        void* checksum = segment->allocate(sizeof(uint32_t));
+        *static_cast<uint32_t*>(checksum) = 0;
+        segment->setNamedAddress("zone_table_checksum", checksum);
+
+        // There must be no previously saved ZoneTableHeader.
+        if (segment->getNamedAddress("zone_table_header")) {
+            isc_throw(isc::Unexpected,
+                      "There is already a saved ZoneTableHeader in a "
+                      "mapped segment opened in create mode.");
+        }
+        void* ptr = segment->allocate(sizeof(ZoneTableHeader));
+        ZoneTableHeader* new_header = new(ptr)
+             ZoneTableHeader(ZoneTable::create(*segment, rrclass_));
+        segment->setNamedAddress("zone_table_header", new_header);
+        header_ = new_header;
+
+        break;
+    }
+    case READ_WRITE: {
+        segment.reset(new MemorySegmentMapped
+                      (filename, MemorySegmentMapped::OPEN_OR_CREATE));
+        // If there is a previously saved checksum, verify that it is
+        // consistent. Otherwise, allocate space for a checksum (which
+        // is saved during close).
+        if (segment->getNamedAddress("zone_table_checksum")) {
+            // The segment was already shrunk when it was last
+            // closed. Check that its checksum is consistent.
+            uint32_t* checksum = static_cast<uint32_t*>
+                (segment->getNamedAddress("zone_table_checksum"));
+            uint32_t saved_checksum = *checksum;
+            // First, clear the checksum so that getCheckSum() returns
+            // a consistent value.
+            *checksum = 0;
+            const uint32_t new_checksum = segment->getCheckSum();
+            if (saved_checksum != new_checksum) {
+                 isc_throw(isc::Unexpected,
+                           "Saved checksum doesn't match mapped segment data");
+            }
+        } else {
+            void* checksum = segment->allocate(sizeof(uint32_t));
+            *static_cast<uint32_t*>(checksum) = 0;
+            segment->setNamedAddress("zone_table_checksum", checksum);
+        }
+
+        // If there is a previously saved ZoneTableHeader, use
+        // it. Otherwise, allocate a new header.
+        header_ = static_cast<ZoneTableHeader*>
+            (segment->getNamedAddress("zone_table_header"));
+        if (!header_) {
+            void* ptr = segment->allocate(sizeof(ZoneTableHeader));
+            ZoneTableHeader* new_header = new(ptr)
+                ZoneTableHeader(ZoneTable::create(*segment, rrclass_));
+            segment->setNamedAddress("zone_table_header", new_header);
+            header_ = new_header;
+        }
+
+        break;
+    }
+    case READ_ONLY: {
+        segment.reset(new MemorySegmentMapped(filename));
+        // There must be a previously saved checksum.
+        if (!segment->getNamedAddress("zone_table_checksum")) {
+            isc_throw(isc::Unexpected,
+                      "There is no previously saved checksum in a "
+                      "mapped segment opened in read-only mode.");
+        }
+        // The segment was already shrunk when it was last closed. Check
+        // that its checksum is consistent.
+        // FIXME: We can't really do this as we can't set the checksum
+        // to 0 for checksum calculation in a read-only segment.
+
+        // There must be a previously saved ZoneTableHeader.
+        header_ = static_cast<ZoneTableHeader*>
+            (segment->getNamedAddress("zone_table_header"));
+        if (!header_) {
+            isc_throw(isc::Unexpected,
+                      "There is no previously saved ZoneTableHeader in a "
+                      "mapped segment opened in read-only mode.");
+        }
+    }
+    }
+
+    current_mode_ = mode;
+    mem_sgmt_.reset(segment.release());
+}
+
+// After more methods' definitions are added here, it would be a good
+// idea to move getHeader() and getMemorySegment() definitions to the
+// header file.
+ZoneTableHeader&
+ZoneTableSegmentMapped::getHeader() {
+    if (!mem_sgmt_) {
+        isc_throw(isc::Unexpected,
+                  "getHeader() called without calling reset() first");
+    }
+    return (*header_);
+}
+
+const ZoneTableHeader&
+ZoneTableSegmentMapped::getHeader() const {
+    if (!mem_sgmt_) {
+        isc_throw(isc::Unexpected,
+                  "getHeader() called without calling reset() first");
+    }
+    return (*header_);
+}
+
+MemorySegment&
+ZoneTableSegmentMapped::getMemorySegment() {
+    if (!mem_sgmt_) {
+        isc_throw(isc::Unexpected,
+                  "getMemorySegment() called without calling reset() first");
+    }
+    return (*mem_sgmt_);
+}
+
+bool
+ZoneTableSegmentMapped::isWritable() const {
+    if (!mem_sgmt_) {
+        isc_throw(isc::Unexpected,
+                  "isWritable() called without calling reset() first");
+    }
+    return ((current_mode_ == CREATE) || (current_mode_ == READ_WRITE));
+}
+
+} // namespace memory
+} // namespace datasrc
+} // namespace isc

src/lib/datasrc/memory/zone_table_segment_mapped.h

@@ -0,0 +1,84 @@
+// Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#ifndef ZONE_TABLE_SEGMENT_MAPPED_H
+#define ZONE_TABLE_SEGMENT_MAPPED_H
+
+#include <datasrc/memory/zone_table_segment.h>
+#include <util/memory_segment_mapped.h>
+
+#include <boost/scoped_ptr.hpp>
+
+namespace isc {
+namespace datasrc {
+namespace memory {
+
+/// \brief Mapped-file based implementation of ZoneTableSegment class
+///
+/// This class specifies a concrete implementation for a memory-mapped
+/// ZoneTableSegment. Please see the ZoneTableSegment class
+/// documentation for usage.
+class ZoneTableSegmentMapped : public ZoneTableSegment {
+    // This is so that ZoneTableSegmentMapped can be instantiated from
+    // ZoneTableSegment::create().
+    friend class ZoneTableSegment;
+protected:
+    /// \brief Protected constructor
+    ///
+    /// Instances are expected to be created by the factory method
+    /// (\c ZoneTableSegment::create()), so this constructor is
+    /// protected.
+    ZoneTableSegmentMapped(const isc::dns::RRClass& rrclass);
+public:
+    /// \brief Destructor
+    virtual ~ZoneTableSegmentMapped();
+
+    /// \brief Return the ZoneTableHeader for the mapped zone table
+    /// segment implementation.
+    virtual ZoneTableHeader& getHeader();
+
+    /// \brief const version of \c getHeader().
+    virtual const ZoneTableHeader& getHeader() const;
+
+    /// \brief Return the MemorySegment for the memory-mapped zone table
+    /// segment implementation (a MemorySegmentMapped instance).
+    virtual isc::util::MemorySegment& getMemorySegment();
+
+    /// \brief Return true if the segment is writable. For read-only
+    /// segments, false is returned.
+    virtual bool isWritable() const;
+
+    enum MemorySegmentOpenMode {
+        CREATE,
+        READ_WRITE,
+        READ_ONLY
+    };
+
+    virtual void reset(MemorySegmentOpenMode mode,
+                       isc::data::ConstElementPtr params);
+
+private:
+    // Internally holds a MemorySegmentMapped. This is NULL on
+    // construction, and is set by the \c reset() method.
+    isc::dns::RRClass rrclass_;
+    MemorySegmentOpenMode current_mode_;
+    boost::scoped_ptr<isc::util::MemorySegmentMapped> mem_sgmt_;
+    ZoneTableHeader* header_;
+};
+
+} // namespace memory
+} // namespace datasrc
+} // namespace isc
+
+#endif // ZONE_TABLE_SEGMENT_MAPPED_H

src/lib/datasrc/tests/memory/Makefile.am

@@ -4,6 +4,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/src/lib -I$(top_builddir)/src/lib
 AM_CPPFLAGS += -I$(top_builddir)/src/lib/dns -I$(top_srcdir)/src/lib/dns
 AM_CPPFLAGS += $(BOOST_INCLUDES)
 AM_CPPFLAGS += -DTEST_DATA_DIR=\"$(abs_srcdir)/testdata\"
+AM_CPPFLAGS += -DTEST_DATA_BUILDDIR=\"$(abs_builddir)\"
 
 AM_CXXFLAGS = $(B10_CXXFLAGS)
 
@@ -38,6 +39,7 @@ run_unittests_SOURCES += zone_data_loader_unittest.cc
 run_unittests_SOURCES += zone_data_updater_unittest.cc
 run_unittests_SOURCES += zone_table_segment_test.h
 run_unittests_SOURCES += zone_table_segment_unittest.cc
+run_unittests_SOURCES += zone_table_segment_mapped_unittest.cc
 run_unittests_SOURCES += zone_writer_unittest.cc
 
 run_unittests_CPPFLAGS = $(AM_CPPFLAGS) $(GTEST_INCLUDES)

src/lib/datasrc/tests/memory/zone_table_segment_mapped_unittest.cc

@@ -0,0 +1,172 @@
+// Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <datasrc/memory/zone_writer.h>
+#include <datasrc/memory/zone_table_segment_mapped.h>
+
+#include <gtest/gtest.h>
+#include <boost/scoped_ptr.hpp>
+#include <boost/interprocess/file_mapping.hpp>
+
+using namespace isc::dns;
+using namespace isc::datasrc::memory;
+using namespace isc::data;
+using namespace isc::util;
+using namespace std;
+using boost::scoped_ptr;
+
+namespace {
+
+const std::string mapped_file = TEST_DATA_BUILDDIR "/test.mapped";
+
+class ZoneTableSegmentMappedTest : public ::testing::Test {
+protected:
+    ZoneTableSegmentMappedTest() :
+        ztable_segment_(dynamic_cast<ZoneTableSegmentMapped*>(
+            ZoneTableSegment::create(RRClass::IN(), "mapped"))),
+        config_params_(
+            Element::fromJSON("{\"mapped-file\": \"" + mapped_file + "\"}"))
+    {}
+
+    ~ZoneTableSegmentMappedTest() {
+        boost::interprocess::file_mapping::remove(mapped_file.c_str());
+    }
+
+    void TearDown() {
+        ZoneTableSegment::destroy(ztable_segment_);
+        ztable_segment_ = NULL;
+    }
+
+    ZoneTableSegmentMapped* ztable_segment_;
+    const ConstElementPtr config_params_;
+};
+
+
+TEST_F(ZoneTableSegmentMappedTest, create) {
+    // Verify that a mapped segment is created.
+    EXPECT_NE(static_cast<void*>(NULL), ztable_segment_);
+}
+
+TEST_F(ZoneTableSegmentMappedTest, getHeaderUninitialized) {
+    // This should throw as we haven't called reset() yet.
+    EXPECT_THROW(ztable_segment_->getHeader(), isc::Unexpected);
+}
+
+TEST_F(ZoneTableSegmentMappedTest, getMemorySegmentUninitialized) {
+    // This should throw as we haven't called reset() yet.
+    EXPECT_THROW(ztable_segment_->getMemorySegment(), isc::Unexpected);
+}
+
+TEST_F(ZoneTableSegmentMappedTest, isWritableUninitialized) {
+    // This should throw as we haven't called reset() yet.
+    EXPECT_THROW(ztable_segment_->isWritable(), isc::Unexpected);
+}
+
+ZoneData*
+loadAction(MemorySegment&) {
+    // The function won't be called, so this is OK
+    return (NULL);
+}
+
+// Test we can get a writer.
+TEST_F(ZoneTableSegmentMappedTest, getZoneWriterUninitialized) {
+    // This should throw as we haven't called reset() yet.
+    EXPECT_THROW({
+        ztable_segment_->getZoneWriter(loadAction, Name("example.org"),
+                                       RRClass::IN());
+    }, isc::Unexpected);
+}
+
+TEST_F(ZoneTableSegmentMappedTest, resetBadConfig) {
+    // Not a map
+    EXPECT_THROW({
+        ztable_segment_->reset(ZoneTableSegmentMapped::CREATE,
+                               Element::fromJSON("42"));
+    }, isc::InvalidParameter);
+
+    // Empty map
+    EXPECT_THROW({
+        ztable_segment_->reset(ZoneTableSegmentMapped::CREATE,
+                               Element::fromJSON("{}"));
+    }, isc::InvalidParameter);
+
+    // No "mapped-file" key
+    EXPECT_THROW({
+        ztable_segment_->reset(ZoneTableSegmentMapped::CREATE,
+                               Element::fromJSON("{\"foo\": \"bar\"}"));
+    }, isc::InvalidParameter);
+
+    // Value of "mapped-file" key is not a string
+    EXPECT_THROW({
+        ztable_segment_->reset(ZoneTableSegmentMapped::CREATE,
+                               Element::fromJSON("{\"mapped-file\": 42}"));
+    }, isc::InvalidParameter);
+
+    // The following should still throw, unaffected by the failed opens.
+    EXPECT_THROW(ztable_segment_->getHeader(), isc::Unexpected);
+    EXPECT_THROW(ztable_segment_->getMemorySegment(), isc::Unexpected);
+    EXPECT_THROW(ztable_segment_->isWritable(), isc::Unexpected);
+}
+
+TEST_F(ZoneTableSegmentMappedTest, reset) {
+    // By default, the mapped file doesn't exist, so we cannot open it
+    // in READ_ONLY mode (which does not create the file).
+    EXPECT_THROW({
+        ztable_segment_->reset(ZoneTableSegmentMapped::READ_ONLY,
+                               config_params_);
+    }, MemorySegmentOpenError);
+
+    // The following should still throw, unaffected by the failed open.
+    EXPECT_THROW(ztable_segment_->getHeader(), isc::Unexpected);
+    EXPECT_THROW(ztable_segment_->getMemorySegment(), isc::Unexpected);
+    EXPECT_THROW(ztable_segment_->isWritable(), isc::Unexpected);
+
+    // READ_WRITE mode must create the mapped file if it doesn't exist
+    // (and must not result in an exception).
+    ztable_segment_->reset(ZoneTableSegmentMapped::READ_WRITE,
+                           config_params_);
+    // This must not throw now.
+    EXPECT_TRUE(ztable_segment_->isWritable());
+
+    // The following method calls should no longer throw:
+    EXPECT_NO_THROW(ztable_segment_->getHeader());
+    EXPECT_NO_THROW(ztable_segment_->getMemorySegment());
+    EXPECT_NO_THROW(ztable_segment_->isWritable());
+
+    // Let's try to re-open the mapped file in READ_ONLY mode. It should
+    // not fail now.
+    ztable_segment_->reset(ZoneTableSegmentMapped::READ_ONLY,
+                           config_params_);
+    EXPECT_FALSE(ztable_segment_->isWritable());
+
+    // Re-creating the mapped file should erase old data and should not
+    // trigger any exceptions inside reset() due to old data (such as
+    // named addresses).
+    ztable_segment_->reset(ZoneTableSegmentMapped::CREATE,
+                           config_params_);
+    EXPECT_TRUE(ztable_segment_->isWritable());
+
+    // When we reset() and it fails, then the segment should be
+    // unusable.
+    EXPECT_THROW({
+        ztable_segment_->reset(ZoneTableSegmentMapped::CREATE,
+                               Element::fromJSON("{}"));
+    }, isc::InvalidParameter);
+    // The following should throw now.
+    EXPECT_THROW(ztable_segment_->getHeader(), isc::Unexpected);
+    EXPECT_THROW(ztable_segment_->getMemorySegment(), isc::Unexpected);
+    EXPECT_THROW(ztable_segment_->isWritable(), isc::Unexpected);
+}
+
+} // anonymous namespace

src/lib/datasrc/tests/memory/zone_table_segment_unittest.cc

@@ -14,7 +14,6 @@
 
 #include <datasrc/memory/zone_writer.h>
 #include <datasrc/memory/zone_table_segment_local.h>
-#include <util/memory_segment_local.h>
 
 #include <gtest/gtest.h>
 #include <boost/scoped_ptr.hpp>