[2831] introduced file-mapped memory segment with some basic functionalities.

committing mostly as a backup.

src/lib/util/Makefile.am

@@ -18,6 +18,7 @@ libb10_util_la_SOURCES += interprocess_sync_file.h interprocess_sync_file.cc
 libb10_util_la_SOURCES += interprocess_sync_null.h interprocess_sync_null.cc
 libb10_util_la_SOURCES += memory_segment.h
 libb10_util_la_SOURCES += memory_segment_local.h memory_segment_local.cc
+libb10_util_la_SOURCES += memory_segment_mapped.h memory_segment_mapped.cc
 libb10_util_la_SOURCES += range_utilities.h
 libb10_util_la_SOURCES += hash/sha1.h hash/sha1.cc
 libb10_util_la_SOURCES += encode/base16_from_binary.h

src/lib/util/memory_segment.h

@@ -15,11 +15,25 @@
 #ifndef MEMORY_SEGMENT_H
 #define MEMORY_SEGMENT_H
 
+#include <exceptions/exceptions.h>
+
 #include <stdlib.h>
 
 namespace isc {
 namespace util {
 
+class MemorySegmentOpenError : public Exception {
+public:
+    MemorySegmentOpenError(const char* file, size_t line, const char* what) :
+        isc::Exception(file, line, what) {}
+};
+
+class MemorySegmentGrown : public Exception {
+public:
+    MemorySegmentGrown(const char* file, size_t line, const char* what) :
+        isc::Exception(file, line, what) {}
+};
+
 /// \brief Memory Segment Class
 ///
 /// This class specifies an interface for allocating memory

src/lib/util/memory_segment_mapped.cc

@@ -0,0 +1,128 @@
+// Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <util/memory_segment_mapped.h>
+
+#include <boost/scoped_ptr.hpp>
+#include <boost/interprocess/exceptions.hpp>
+#include <boost/interprocess/managed_mapped_file.hpp>
+
+#include <cassert>
+#include <string>
+#include <new>
+
+using boost::interprocess::managed_mapped_file;
+using boost::interprocess::open_or_create;
+using boost::interprocess::open_only;
+
+namespace isc {
+namespace util {
+
+struct MemorySegmentMapped::Impl {
+    Impl(const std::string& filename, size_t initial_size) :
+        filename_(filename),
+        base_sgmt_(new managed_mapped_file(open_or_create, filename.c_str(),
+                                           initial_size))
+    {}
+
+    Impl(const std::string& filename) :
+        filename_(filename),
+        base_sgmt_(new managed_mapped_file(open_only, filename.c_str()))
+    {}
+
+    // mapped file; remember it in case we need to grow it.
+    const std::string filename_;
+
+    // actual Boost implementation of mapped segment.
+    boost::scoped_ptr<managed_mapped_file> base_sgmt_;
+};
+
+MemorySegmentMapped::MemorySegmentMapped(const std::string& filename) :
+    impl_(0)
+{
+    try {
+        impl_ = new Impl(filename);
+    } catch (const boost::interprocess::interprocess_exception& ex) {
+        isc_throw(MemorySegmentOpenError,
+                  "failed to open mapped memory segment for " << filename
+                  << ": " << ex.what());
+    }
+}
+
+MemorySegmentMapped::MemorySegmentMapped(const std::string& filename,
+                                         bool create, size_t initial_size) :
+    impl_(0)
+{
+    try {
+        if (create) {
+            impl_ = new Impl(filename, initial_size);
+        } else {
+            impl_ = new Impl(filename);
+        }
+    } catch (const boost::interprocess::interprocess_exception& ex) {
+        isc_throw(MemorySegmentOpenError,
+                  "failed to open mapped memory segment for " << filename
+                  << ": " << ex.what());
+    }
+}
+
+MemorySegmentMapped::~MemorySegmentMapped() {
+    delete impl_;
+}
+
+void*
+MemorySegmentMapped::allocate(size_t size) {
+    void* ptr = impl_->base_sgmt_->allocate(size, std::nothrow);
+    if (ptr) {
+        return (ptr);
+    }
+
+    // Grow the mapped segment doubling the size until we have sufficient
+    // free memory in the revised segment for the requested size.
+    while (impl_->base_sgmt_->get_free_memory() < size) {
+        // We first need to unmap it before calling grow().
+        const size_t prev_size = impl_->base_sgmt_->get_size();
+        impl_->base_sgmt_.reset();
+
+        const size_t new_size = prev_size * 2;
+        assert(new_size != 0); // assume grow fails before size overflow
+
+        // TBD error handling
+        managed_mapped_file::grow(impl_->filename_.c_str(),
+                                  new_size - prev_size);
+        impl_->base_sgmt_.reset(
+            new managed_mapped_file(open_only, impl_->filename_.c_str()));
+    }
+    isc_throw(MemorySegmentGrown, "mapped memory segment grown, size: "
+              << impl_->base_sgmt_->get_size() << ", free size: "
+              << impl_->base_sgmt_->get_free_memory());
+}
+
+void
+MemorySegmentMapped::deallocate(void* ptr, size_t /*size*/) {
+    impl_->base_sgmt_->deallocate(ptr);
+}
+
+bool
+MemorySegmentMapped::allMemoryDeallocated() const {
+    return (impl_->base_sgmt_->all_memory_deallocated());
+}
+
+size_t
+MemorySegmentMapped::getSize() const {
+    return (impl_->base_sgmt_->get_size());
+}
+
+} // namespace util
+} // namespace isc

src/lib/util/memory_segment_mapped.h

@@ -0,0 +1,66 @@
+// Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#ifndef MEMORY_SEGMENT_MAPPED_H
+#define MEMORY_SEGMENT_MAPPED_H
+
+#include <util/memory_segment.h>
+
+#include <boost/noncopyable.hpp>
+
+#include <string>
+
+namespace isc {
+namespace util {
+
+/// \brief TBD
+class MemorySegmentMapped : boost::noncopyable, public MemorySegment {
+public:
+    /// arbitrary choice.
+    static const size_t INITIAL_SIZE = 32768;
+
+    /// \brief Constructor in the read-only mode
+    ///
+    /// Creates a local memory segment object
+    MemorySegmentMapped(const std::string& filename);
+
+    MemorySegmentMapped(const std::string& filename, bool create,
+                        size_t initial_size = INITIAL_SIZE);
+
+    /// \brief Destructor
+    virtual ~MemorySegmentMapped();
+
+    /// \brief Allocate/acquire a segment of memory.
+    virtual void* allocate(size_t size);
+
+    /// \brief Deallocate/release a segment of memory.
+    virtual void deallocate(void* ptr, size_t size);
+
+    virtual bool allMemoryDeallocated() const;
+
+    size_t getSize() const;
+
+private:
+    struct Impl;
+    Impl* impl_;
+};
+
+} // namespace util
+} // namespace isc
+
+#endif // MEMORY_SEGMENT_MAPPED_H
+
+// Local Variables:
+// mode: c++
+// End:

src/lib/util/tests/Makefile.am

@@ -34,6 +34,7 @@ run_unittests_SOURCES += lru_list_unittest.cc
 run_unittests_SOURCES += interprocess_sync_file_unittest.cc
 run_unittests_SOURCES += interprocess_sync_null_unittest.cc
 run_unittests_SOURCES += memory_segment_local_unittest.cc
+run_unittests_SOURCES += memory_segment_mapped_unittest.cc
 run_unittests_SOURCES += qid_gen_unittest.cc
 run_unittests_SOURCES += random_number_generator_unittest.cc
 run_unittests_SOURCES += sha1_unittest.cc

src/lib/util/tests/memory_segment_mapped_unittest.cc

@@ -0,0 +1,219 @@
+// Copyright (C) 2013  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <util/memory_segment_mapped.h>
+
+#include <gtest/gtest.h>
+
+#include <boost/interprocess/file_mapping.hpp>
+#include <boost/scoped_ptr.hpp>
+
+using namespace isc::util;
+using boost::scoped_ptr;
+
+namespace {
+
+const char* const mapped_file = TEST_DATA_BUILDDIR "/test.mapped";
+const size_t DEFAULT_INITIAL_SIZE = 32 * 1024; // intentionally hardcoded
+
+class MemorySegmentMappedTest : public ::testing::Test {
+protected:
+    MemorySegmentMappedTest() {
+        // Make sure the mapped file doesn't exist; actually it shouldn't
+        // exist in normal cases so remove() should normally fail (returning
+        // false), but we don't care.
+        boost::interprocess::file_mapping::remove(mapped_file);
+
+        // Create a new segment with a new file.  It also confirms the
+        // behavior of this mode of constructor.
+        segment_.reset(new MemorySegmentMapped(mapped_file, true));
+    }
+
+    ~MemorySegmentMappedTest() {
+        segment_.reset();
+        boost::interprocess::file_mapping::remove(mapped_file);
+    }
+
+    scoped_ptr<MemorySegmentMapped> segment_;
+};
+
+TEST_F(MemorySegmentMappedTest, createAndModify) {
+    // We are going to do the same set of basic tests twice; one after creating
+    // the mapped file, the other by re-opening the existing file in the
+    // read-write mode.
+    for (int i = 0; i < 2; ++i) {
+        // It should have the default size (intentionally hardcoded)
+        EXPECT_EQ(DEFAULT_INITIAL_SIZE, segment_->getSize());
+
+        // By default, nothing is allocated.
+        EXPECT_TRUE(segment_->allMemoryDeallocated());
+
+        void* ptr = segment_->allocate(1024);
+        EXPECT_NE(static_cast<void*>(0), ptr);
+
+        // Now, we have an allocation:
+        EXPECT_FALSE(segment_->allMemoryDeallocated());
+
+        // deallocate it; it shouldn't cause disruption.
+        segment_->deallocate(ptr, 1024);
+
+        EXPECT_TRUE(segment_->allMemoryDeallocated());
+
+        // re-open it.
+        segment_.reset(new MemorySegmentMapped(mapped_file, false));
+    }
+}
+
+TEST_F(MemorySegmentMappedTest, createWithSize) {
+    boost::interprocess::file_mapping::remove(mapped_file);
+
+    // Re-create the mapped file with a non-default initial size, and confirm
+    // the size is actually the specified one.
+    segment_.reset(new MemorySegmentMapped(mapped_file, true, 64 * 1024));
+    EXPECT_NE(DEFAULT_INITIAL_SIZE, 64 * 1024);
+    EXPECT_EQ(64 * 1024, segment_->getSize());
+}
+
+TEST_F(MemorySegmentMappedTest, openFail) {
+    // The given file is directory
+    EXPECT_THROW(MemorySegmentMapped("/", true), MemorySegmentOpenError);
+
+    // file doesn't exist and directory isn't writable (we assume the root
+    // directory is not writable for the user running the test).
+    EXPECT_THROW(MemorySegmentMapped("/test.mapped", true),
+                 MemorySegmentOpenError);
+
+    // file doesn't exist and it's read-only (so open-only)
+    EXPECT_THROW(MemorySegmentMapped(TEST_DATA_BUILDDIR "/nosuchfile.mapped"),
+                 MemorySegmentOpenError);
+    // Likewise.  read-write mode but creation is suppressed.
+    EXPECT_THROW(MemorySegmentMapped(TEST_DATA_BUILDDIR "/nosuchfile.mapped",
+                                     false),
+                 MemorySegmentOpenError);
+}
+
+TEST_F(MemorySegmentMappedTest, allocate) {
+    // Various case of allocation.  The simplest cases are covered above.
+
+    // (Clearly) exceeding the available size, which should cause growing
+    // the segment
+    EXPECT_THROW(segment_->allocate(DEFAULT_INITIAL_SIZE + 1),
+                 MemorySegmentGrown);
+    // The size should have been doubled.
+    EXPECT_EQ(DEFAULT_INITIAL_SIZE * 2,segment_->getSize());
+    // In this case it should now succeed.
+    void* ptr = segment_->allocate(DEFAULT_INITIAL_SIZE + 1);
+    EXPECT_NE(static_cast<void*>(0), ptr);
+
+    EXPECT_FALSE(segment_->allMemoryDeallocated());
+
+    // Same set of checks, but for a larger size.
+    EXPECT_THROW(segment_->allocate(DEFAULT_INITIAL_SIZE * 10),
+                 MemorySegmentGrown);
+    ptr = segment_->allocate(DEFAULT_INITIAL_SIZE * 10);
+    EXPECT_NE(static_cast<void*>(0), ptr);
+}
+
+TEST_F(MemorySegmentMappedTest, DISABLED_basics) {
+    MemorySegmentMapped segment(mapped_file);
+
+    // By default, nothing is allocated.
+    EXPECT_TRUE(segment.allMemoryDeallocated());
+
+#if 0
+    void* ptr = segment->allocate(1024);
+
+    // Now, we have an allocation:
+    EXPECT_FALSE(segment->allMemoryDeallocated());
+
+    void* ptr2 = segment->allocate(42);
+
+    // Still:
+    EXPECT_FALSE(segment->allMemoryDeallocated());
+
+    // These should not fail, because the buffers have been allocated.
+    EXPECT_NO_FATAL_FAILURE(memset(ptr, 0, 1024));
+    EXPECT_NO_FATAL_FAILURE(memset(ptr, 0, 42));
+
+    segment->deallocate(ptr, 1024);
+
+    // Still:
+    EXPECT_FALSE(segment->allMemoryDeallocated());
+
+    segment->deallocate(ptr2, 42);
+
+    // Now, we have an deallocated everything:
+    EXPECT_TRUE(segment->allMemoryDeallocated());
+#endif
+}
+
+/*
+TEST(MemorySegmentLocal, TestTooMuchMemory) {
+    auto_ptr<MemorySegment> segment(new MemorySegmentLocal());
+
+    EXPECT_THROW(segment->allocate(ULONG_MAX), bad_alloc);
+}
+*/
+
+/*
+TEST(MemorySegmentLocal, TestBadDeallocate) {
+    auto_ptr<MemorySegment> segment(new MemorySegmentLocal());
+
+    // By default, nothing is allocated.
+    EXPECT_TRUE(segment->allMemoryDeallocated());
+
+    void* ptr = segment->allocate(1024);
+
+    // Now, we have an allocation:
+    EXPECT_FALSE(segment->allMemoryDeallocated());
+
+    // This should not throw
+    EXPECT_NO_THROW(segment->deallocate(ptr, 1024));
+
+    // Now, we have an deallocated everything:
+    EXPECT_TRUE(segment->allMemoryDeallocated());
+
+    ptr = segment->allocate(1024);
+
+    // Now, we have another allocation:
+    EXPECT_FALSE(segment->allMemoryDeallocated());
+
+    // This should throw as the size passed to deallocate() is larger
+    // than what was allocated.
+    EXPECT_THROW(segment->deallocate(ptr, 2048), isc::OutOfRange);
+
+    // This should not throw
+    EXPECT_NO_THROW(segment->deallocate(ptr, 1024));
+
+    // Now, we have an deallocated everything:
+    EXPECT_TRUE(segment->allMemoryDeallocated());
+}
+*/
+
+/*
+TEST(MemorySegmentLocal, TestNullDeallocate) {
+    auto_ptr<MemorySegment> segment(new MemorySegmentLocal());
+
+    // By default, nothing is allocated.
+    EXPECT_TRUE(segment->allMemoryDeallocated());
+
+    // NULL deallocation is a no-op.
+    EXPECT_NO_THROW(segment->deallocate(NULL, 1024));
+
+    // This should still return true.
+    EXPECT_TRUE(segment->allMemoryDeallocated());
+}
+*/
+
+}