[3671] Isolated the container for leases into the separate file.

Also added an utility class to bulk load leases from the file.

src/lib/dhcpsrv/Makefile.am

@@ -80,8 +80,10 @@ libkea_dhcpsrv_la_SOURCES += dhcpsrv_log.cc dhcpsrv_log.h
 libkea_dhcpsrv_la_SOURCES += host.cc host.h
 libkea_dhcpsrv_la_SOURCES += host_container.h
 libkea_dhcpsrv_la_SOURCES += host_mgr.cc host_mgr.h
+libkea_dhcpsrv_la_SOURCES += inmemory_lease_storage.h
 libkea_dhcpsrv_la_SOURCES += key_from_key.h
 libkea_dhcpsrv_la_SOURCES += lease.cc lease.h
+libkea_dhcpsrv_la_SOURCES += lease_file_loader.h
 libkea_dhcpsrv_la_SOURCES += lease_mgr.cc lease_mgr.h
 libkea_dhcpsrv_la_SOURCES += lease_mgr_factory.cc lease_mgr_factory.h
 libkea_dhcpsrv_la_SOURCES += logging.cc logging.h

src/lib/dhcpsrv/dhcpsrv_messages.mes

@@ -284,23 +284,13 @@ subnet ID and hardware address.
 A debug message issued when the server is about to obtain schema version
 information from the memory file database.
 
-% DHCPSRV_MEMFILE_LEASES_RELOAD4 reloading leases from %1
-An info message issued when the server is about to start reading DHCPv4 leases
+% DHCPSRV_MEMFILE_LEASE_FILE_LOAD loading leases from file %1
+An info message issued when the server is about to start reading DHCP leases
 from the lease file. All leases currently held in the memory will be
 replaced by those read from the file.
 
-% DHCPSRV_MEMFILE_LEASES_RELOAD6 reloading leases from %1
-An info message issued when the server is about to start reading DHCPv6 leases
-from the lease file. All leases currently held in the memory will be
-replaced by those read from the file.
-
-% DHCPSRV_MEMFILE_LEASE_LOAD4 loading lease %1
-A debug message issued when DHCPv4 lease is being loaded from the file to
-memory.
-
-% DHCPSRV_MEMFILE_LEASE_LOAD6 loading lease %1
-A debug message issued when DHCPv6 lease is being loaded from the file to
-memory.
+% DHCPSRV_MEMFILE_LEASE_LOAD loading lease %1
+A debug message issued when DHCP lease is being loaded from the file to memory.
 
 % DHCPSRV_MEMFILE_NO_STORAGE running in non-persistent mode, leases will be lost after restart
 A warning message issued when writes of leases to disk have been disabled

src/lib/dhcpsrv/inmemory_lease_storage.h

@@ -0,0 +1,151 @@
+// Copyright (C) 2015 Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#ifndef INMEMORY_LEASE_STORAGE_H
+#define INMEMORY_LEASE_STORAGE_H
+
+#include <asiolink/io_address.h>
+#include <dhcpsrv/lease.h>
+#include <dhcpsrv/subnet_id.h>
+
+#include <boost/multi_index/indexed_by.hpp>
+#include <boost/multi_index/member.hpp>
+#include <boost/multi_index/mem_fun.hpp>
+#include <boost/multi_index/ordered_index.hpp>
+#include <boost/multi_index_container.hpp>
+#include <boost/multi_index/composite_key.hpp>
+
+#include <vector>
+
+namespace isc {
+namespace dhcp {
+
+/// @brief A multi index container holding DHCPv6 leases.
+///
+/// The leases in the container may be accessed using different indexes:
+/// - using an IPv6 address,
+/// - using a composite index: DUID, IAID and lease type.
+typedef boost::multi_index_container<
+    // It holds pointers to Lease6 objects.
+    Lease6Ptr,
+    boost::multi_index::indexed_by<
+        // Specification of the first index starts here.
+        // This index sorts leases by IPv6 addresses represented as
+        // IOAddress objects.
+        boost::multi_index::ordered_unique<
+            boost::multi_index::member<Lease, isc::asiolink::IOAddress, &Lease::addr_>
+        >,
+
+        // Specification of the second index starts here.
+        boost::multi_index::ordered_non_unique<
+            // This is a composite index that will be used to search for
+            // the lease using three attributes: DUID, IAID and lease type.
+            boost::multi_index::composite_key<
+                Lease6,
+                // The DUID can be retrieved from the Lease6 object using
+                // a getDuidVector const function.
+                boost::multi_index::const_mem_fun<Lease6, const std::vector<uint8_t>&,
+                                                  &Lease6::getDuidVector>,
+                // The two other ingredients of this index are IAID and
+                // lease type.
+                boost::multi_index::member<Lease6, uint32_t, &Lease6::iaid_>,
+                boost::multi_index::member<Lease6, Lease::Type, &Lease6::type_>
+            >
+        >
+     >
+> Lease6Storage; // Specify the type name of this container.
+
+/// @brief A multi index container holding DHCPv4 leases.
+///
+/// The leases in the container may be accessed using different indexes:
+/// - IPv6 address,
+/// - composite index: HW address and subnet id,
+/// - composite index: client id and subnet id,
+/// - composite index: HW address, client id and subnet id
+typedef boost::multi_index_container<
+    // It holds pointers to Lease4 objects.
+    Lease4Ptr,
+    // Specification of search indexes starts here.
+    boost::multi_index::indexed_by<
+        // Specification of the first index starts here.
+        // This index sorts leases by IPv4 addresses represented as
+        // IOAddress objects.
+        boost::multi_index::ordered_unique<
+            // The IPv4 address are held in addr_ members that belong to
+            // Lease class.
+            boost::multi_index::member<Lease, isc::asiolink::IOAddress, &Lease::addr_>
+        >,
+
+        // Specification of the second index starts here.
+        boost::multi_index::ordered_unique<
+            // This is a composite index that combines two attributes of the
+            // Lease4 object: hardware address and subnet id.
+            boost::multi_index::composite_key<
+                Lease4,
+                // The hardware address is held in the hwaddr_ member of the
+                // Lease4 object, which is a HWAddr object. Boost does not
+                // provide a key extractor for getting a member of a member,
+                // so we need a simple method for that.
+                boost::multi_index::const_mem_fun<Lease, const std::vector<uint8_t>&,
+                                                  &Lease::getHWAddrVector>,
+                // The subnet id is held in the subnet_id_ member of Lease4
+                // class. Note that the subnet_id_ is defined in the base
+                // class (Lease) so we have to point to this class rather
+                // than derived class: Lease4.
+                boost::multi_index::member<Lease, SubnetID, &Lease::subnet_id_>
+            >
+        >,
+
+        // Specification of the third index starts here.
+        boost::multi_index::ordered_non_unique<
+            // This is a composite index that uses two values to search for a
+            // lease: client id and subnet id.
+            boost::multi_index::composite_key<
+                Lease4,
+                // The client id can be retrieved from the Lease4 object by
+                // calling getClientIdVector const function.
+                boost::multi_index::const_mem_fun<Lease4, const std::vector<uint8_t>&,
+                                                  &Lease4::getClientIdVector>,
+                // The subnet id is accessed through the subnet_id_ member.
+                boost::multi_index::member<Lease, uint32_t, &Lease::subnet_id_>
+            >
+        >,
+
+        // Specification of the fourth index starts here.
+        boost::multi_index::ordered_non_unique<
+            // This is a composite index that uses three values to search for a
+            // lease: client id, HW address and subnet id.
+            boost::multi_index::composite_key<
+                Lease4,
+                // The client id can be retrieved from the Lease4 object by
+                // calling getClientIdVector const function.
+                boost::multi_index::const_mem_fun<Lease4, const std::vector<uint8_t>&,
+                                                  &Lease4::getClientIdVector>,
+                // The hardware address is held in the hwaddr_ object. We can
+                // access the raw data using lease->hwaddr_->hwaddr_, but Boost
+                // doesn't seem to provide a way to use member of a member for this,
+                // so we need a simple key extractor method (getHWAddrVector).
+                boost::multi_index::const_mem_fun<Lease, const std::vector<uint8_t>&,
+                                                  &Lease::getHWAddrVector>,
+                // The subnet id is accessed through the subnet_id_ member.
+                boost::multi_index::member<Lease, SubnetID, &Lease::subnet_id_>
+            >
+        >
+    >
+> Lease4Storage; // Specify the type name for this container.
+
+} // end of isc::dhcp namespace
+} // end of isc namespace
+
+#endif // INMEMORY_LEASE_STORAGE_H

src/lib/dhcpsrv/lease_file_loader.h

@@ -0,0 +1,147 @@
+// Copyright (C) 2015 Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#ifndef LEASE_FILE_LOADER_H
+#define LEASE_FILE_LOADER_H
+
+#include <dhcpsrv/dhcpsrv_log.h>
+#include <dhcpsrv/inmemory_lease_storage.h>
+#include <util/csv_file.h>
+
+#include <boost/shared_ptr.hpp>
+
+namespace isc {
+namespace dhcp {
+
+/// @brief Utility class to manage bulk of leases in the lease files.
+///
+/// This class exposes methods which allow for bulk loading leases from
+/// the lease file and dumping the leases held in memory into the
+/// lease file. There are two major use cases for this class:
+/// - load leases by the DHCP server when the server starts up or
+///   reloads configuration,
+/// - an application performing a lease file cleanup rewrites the whole
+///   lease file to remove the redundant lease entries.
+///
+/// In the former case, this class is used by the @c MemFile_LeaseMgr.
+/// In the latter case, this class is used by the standalone application
+/// which reads the whole lease file into memory (storage) and then
+/// dumps the leases held in the storage to another file.
+///
+/// The methods in this class are templated so as they can be used both
+/// with the @c Lease4Storage and @c Lease6Storage to process the DHCPv4
+/// and DHCPv6 leases respectively.
+///
+/// @todo Add a method which dumps all leases from the storage to a
+/// specified lease file.
+class LeaseFileLoader {
+public:
+
+    /// @brief Load leases from the lease file into the specified storage.
+    ///
+    /// This method iterates over the entries in the lease file in the
+    /// CSV format, creates @c Lease4 or @c Lease6 objects and inserts
+    /// them into the storage to which reference is specified as an
+    /// argument. If there are multiple entries for the particular lease
+    /// in the lease file the entries further in the lease file override
+    /// the previous entries.
+    ///
+    /// If the method finds the entry with the valid lifetime of 0 it
+    /// means that the particular lease was released and the method
+    /// removes an existing lease from the container.
+    ///
+    /// @param lease_file A reference to the @c CSVLeaseFile4 or
+    /// @c CSVLeaseFile6 object representing the lease file. The
+    /// lease file must be opened and the internal file pointer should
+    /// be set to the beginning of the file.
+    /// @param storage A reference to the container to which leases
+    /// should be inserted.
+    /// @param max_errors Maximum number of corrupted leases in the
+    /// lease file. The method will skip corrupted leases but after
+    /// exceeding the specified number of errors it will throw an
+    /// exception.
+    /// @tparam LeaseObjectType A @c Lease4 or @c Lease6.
+    /// @tparam LeaseFileType A @c CSVLeaseFile4 or @c CSVLeaseFile6.
+    /// @tparam StorageType A @c Lease4Storage or @c Lease6Storage.
+    ///
+    /// @throw isc::util::CSVFileError when the maximum number of errors
+    /// has been exceeded.
+    template<typename LeaseObjectType, typename LeaseFileType,
+             typename StorageType>
+    static void load(LeaseFileType& lease_file, StorageType& storage,
+                     const uint32_t max_errors = 0xFFFFFFFF) {
+
+        LOG_INFO(dhcpsrv_logger, DHCPSRV_MEMFILE_LEASE_FILE_LOAD)
+            .arg(lease_file.getFilename());
+
+        boost::shared_ptr<LeaseObjectType> lease;
+        // Track the number of corrupted leases.
+        uint32_t errcnt = 0;
+        while (true) {
+            // Unable to parse the lease.
+            if (!lease_file.next(lease)) {
+                // A value of 0xFFFFFFFF indicates that we don't return
+                // until the whole file is parsed, even if errors occur.
+                // Otherwise, check if we have exceeded the maximum number
+                // of errors and throw an exception if we have.
+                if ((max_errors < 0xFFFFFFFF) && (++errcnt > max_errors)) {
+                    isc_throw(util::CSVFileError, "exceeded maximum number of"
+                              " failures " << max_errors << " to read a lease"
+                              " from the lease file "
+                              << lease_file.getFilename());
+                }
+                // Skip the corrupted lease.
+                continue;
+            }
+
+            // Lease was found and we successfully parsed it.
+            if (lease) {
+                LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL_DATA,
+                          DHCPSRV_MEMFILE_LEASE_LOAD)
+                    .arg(lease->toText());
+
+                // Check if this lease exists.
+                typename StorageType::iterator lease_it =
+                    storage.find(lease->addr_);
+                // The lease doesn't exist yet. Insert the lease if
+                // it has a positive valid lifetime.
+                if (lease_it == storage.end()) {
+                    if (lease->valid_lft_ > 0) {
+                        storage.insert(lease);
+                    }
+                } else {
+                    // The lease exists. If the new entry has a valid
+                    // lifetime of 0 it is an indication to remove the
+                    // existing entry. Otherwise, we update the lease.
+                    if (lease->valid_lft_ == 0) {
+                        storage.erase(lease_it);
+
+                    } else {
+                        **lease_it = *lease;
+                    }
+                }
+
+            } else {
+                // Being here means that we hit the end of file.
+                break;
+
+            }
+        }
+    }
+};
+
+}
+}
+
+#endif // LEASE_FILE_LOADER_H

src/lib/dhcpsrv/memfile_lease_mgr.cc

@@ -14,11 +14,19 @@
 
 #include <dhcpsrv/cfgmgr.h>
 #include <dhcpsrv/dhcpsrv_log.h>
+#include <dhcpsrv/lease_file_loader.h>
 #include <dhcpsrv/memfile_lease_mgr.h>
 #include <exceptions/exceptions.h>
 
 #include <iostream>
 
+namespace {
+
+/// @brief Maximum number of errors to read the leases from the lease file.
+const uint32_t MAX_LEASE_ERRORS = 100;
+
+} // end of anonymous namespace
+
 using namespace isc::dhcp;
 
 Memfile_LeaseMgr::Memfile_LeaseMgr(const ParameterMap& parameters)
@@ -30,14 +38,18 @@ Memfile_LeaseMgr::Memfile_LeaseMgr(const ParameterMap& parameters)
         if (!file4.empty()) {
             lease_file4_.reset(new CSVLeaseFile4(file4));
             lease_file4_->open();
-            load4();
+            storage4_.clear();
+            LeaseFileLoader::load<Lease4>(*lease_file4_, storage4_,
+                                          MAX_LEASE_ERRORS);
         }
     } else {
         std::string file6 = initLeaseFilePath(V6);
         if (!file6.empty()) {
             lease_file6_.reset(new CSVLeaseFile6(file6));
             lease_file6_->open();
-            load6();
+            storage6_.clear();
+            LeaseFileLoader::load<Lease6>(*lease_file6_, storage6_,
+                                          MAX_LEASE_ERRORS);
         }
     }
 
@@ -165,7 +177,7 @@ Lease4Collection
 Memfile_LeaseMgr::getLease4(const ClientId& client_id) const {
     LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL,
               DHCPSRV_MEMFILE_GET_CLIENTID).arg(client_id.toText());
-    typedef Memfile_LeaseMgr::Lease4Storage::nth_index<0>::type SearchIndex;
+    typedef Lease4Storage::nth_index<0>::type SearchIndex;
     Lease4Collection collection;
     const SearchIndex& idx = storage4_.get<0>();
     for(SearchIndex::const_iterator lease = idx.begin();
@@ -468,127 +480,3 @@ Memfile_LeaseMgr::initLeaseFilePath(Universe u) {
     return (lease_file);
 }
 
-void
-Memfile_LeaseMgr::load4() {
-    // If lease file hasn't been opened, we are working in non-persistent mode.
-    // That's fine, just leave.
-    if (!persistLeases(V4)) {
-        return;
-    }
-
-    LOG_INFO(dhcpsrv_logger, DHCPSRV_MEMFILE_LEASES_RELOAD4)
-        .arg(lease_file4_->getFilename());
-
-    // Remove existing leases (if any). We will recreate them based on the
-    // data on disk.
-    storage4_.clear();
-
-    Lease4Ptr lease;
-    do {
-        /// @todo Currently we stop parsing on first failure. It is possible
-        /// that only one (or a few) leases are bad, so in theory we could
-        /// continue parsing but that would require some error counters to
-        /// prevent endless loops. That is enhancement for later time.
-        if (!lease_file4_->next(lease)) {
-            isc_throw(DbOperationError, "Failed to parse the DHCPv6 lease in"
-                      " the lease file: " << lease_file4_->getReadMsg());
-        }
-        // If we got the lease, we update the internal container holding
-        // leases. Otherwise, we reached the end of file and we leave.
-        if (lease) {
-            LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL_DATA,
-                      DHCPSRV_MEMFILE_LEASE_LOAD4)
-                .arg(lease->toText());
-            loadLease4(lease);
-        }
-    } while (lease);
-}
-
-void
-Memfile_LeaseMgr::loadLease4(Lease4Ptr& lease) {
-    // Check if the lease already exists.
-    Lease4Storage::iterator lease_it = storage4_.find(lease->addr_);
-    // Lease doesn't exist.
-    if (lease_it == storage4_.end()) {
-        // Add the lease only if valid lifetime is greater than 0.
-        // We use valid lifetime of 0 to indicate that lease should
-        // be removed.
-        if (lease->valid_lft_ > 0) {
-           storage4_.insert(lease);
-       }
-    } else {
-        // We use valid lifetime of 0 to indicate that the lease is
-        // to be removed. In such case, erase the lease.
-        if (lease->valid_lft_ == 0) {
-            storage4_.erase(lease_it);
-
-        } else {
-            // Update existing lease.
-            **lease_it = *lease;
-        }
-    }
-}
-
-void
-Memfile_LeaseMgr::load6() {
-    // If lease file hasn't been opened, we are working in non-persistent mode.
-    // That's fine, just leave.
-    if (!persistLeases(V6)) {
-        return;
-    }
-
-    LOG_INFO(dhcpsrv_logger, DHCPSRV_MEMFILE_LEASES_RELOAD6)
-        .arg(lease_file6_->getFilename());
-
-    // Remove existing leases (if any). We will recreate them based on the
-    // data on disk.
-    storage6_.clear();
-
-    Lease6Ptr lease;
-    do {
-        /// @todo Currently we stop parsing on first failure. It is possible
-        /// that only one (or a few) leases are bad, so in theory we could
-        /// continue parsing but that would require some error counters to
-        /// prevent endless loops. That is enhancement for later time.
-        if (!lease_file6_->next(lease)) {
-            isc_throw(DbOperationError, "Failed to parse the DHCPv6 lease in"
-                      " the lease file: " << lease_file6_->getReadMsg());
-        }
-        // If we got the lease, we update the internal container holding
-        // leases. Otherwise, we reached the end of file and we leave.
-        if (lease) {
-            LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL_DATA,
-                      DHCPSRV_MEMFILE_LEASE_LOAD6)
-                .arg(lease->toText());
-
-            loadLease6(lease);
-        }
-    } while (lease);
-}
-
-void
-Memfile_LeaseMgr::loadLease6(Lease6Ptr& lease) {
-    // Check if the lease already exists.
-    Lease6Storage::iterator lease_it = storage6_.find(lease->addr_);
-    // Lease doesn't exist.
-    if (lease_it == storage6_.end()) {
-        // Add the lease only if valid lifetime is greater than 0.
-        // We use valid lifetime of 0 to indicate that lease should
-        // be removed.
-        if (lease->valid_lft_ > 0) {
-            storage6_.insert(lease);
-       }
-    } else {
-        // We use valid lifetime of 0 to indicate that the lease is
-        // to be removed. In such case, erase the lease.
-        if (lease->valid_lft_ == 0) {
-            storage6_.erase(lease_it);
-
-        } else {
-            // Update existing lease.
-            **lease_it = *lease;
-        }
-    }
-
-}
-

src/lib/dhcpsrv/memfile_lease_mgr.h

@@ -18,14 +18,9 @@
 #include <dhcp/hwaddr.h>
 #include <dhcpsrv/csv_lease_file4.h>
 #include <dhcpsrv/csv_lease_file6.h>
+#include <dhcpsrv/inmemory_lease_storage.h>
 #include <dhcpsrv/lease_mgr.h>
 
-#include <boost/multi_index/indexed_by.hpp>
-#include <boost/multi_index/member.hpp>
-#include <boost/multi_index/ordered_index.hpp>
-#include <boost/multi_index_container.hpp>
-#include <boost/multi_index/composite_key.hpp>
-
 namespace isc {
 namespace dhcp {
 
@@ -335,50 +330,6 @@ public:
 
 protected:
 
-    /// @brief Load all DHCPv4 leases from the file.
-    ///
-    /// This method loads all DHCPv4 leases from a file to memory. It removes
-    /// existing leases before reading a file.
-    ///
-    /// @throw isc::DbOperationError If failed to read a lease from the lease
-    /// file.
-    void load4();
-
-    /// @brief Loads a single DHCPv4 lease from the file.
-    ///
-    /// This method reads a single lease record from the lease file. If the
-    /// corresponding record doesn't exist in the in-memory container, the
-    /// lease is added to the container (except for a lease which valid lifetime
-    /// is 0). If the corresponding lease exists, the lease being read updates
-    /// the existing lease. If the lease being read from the lease file has
-    /// valid lifetime of 0 and the corresponding lease exists in the in-memory
-    /// database, the existing lease is removed.
-    ///
-    /// @param lease Pointer to the lease read from the lease file.
-    void loadLease4(Lease4Ptr& lease);
-
-    /// @brief Load all DHCPv6 leases from the file.
-    ///
-    /// This method loads all DHCPv6 leases from a file to memory. It removes
-    /// existing leases before reading a file.
-    ///
-    /// @throw isc::DbOperationError If failed to read a lease from the lease
-    /// file.
-    void load6();
-
-    /// @brief Loads a single DHCPv6 lease from the file.
-    ///
-    /// This method reads a single lease record from the lease file. If the
-    /// corresponding record doesn't exist in the in-memory container, the
-    /// lease is added to the container (except for a lease which valid lifetime
-    /// is 0). If the corresponding lease exists, the lease being read updates
-    /// the existing lease. If the lease being read from the lease file has
-    /// valid lifetime of 0 and the corresponding lease exists in the in-memory
-    /// database, the existing lease is removed.
-    ///
-    /// @param lease Pointer to the lease read from the lease file.
-    void loadLease6(Lease6Ptr& lease);
-
     /// @brief Initialize the location of the lease file.
     ///
     /// This method uses the parameters passed as a map to the constructor to
@@ -396,112 +347,6 @@ protected:
     /// argument to this function.
     std::string initLeaseFilePath(Universe u);
 
-    // This is a multi-index container, which holds elements that can
-    // be accessed using different search indexes.
-    typedef boost::multi_index_container<
-        // It holds pointers to Lease6 objects.
-        Lease6Ptr,
-        boost::multi_index::indexed_by<
-            // Specification of the first index starts here.
-            // This index sorts leases by IPv6 addresses represented as
-            // IOAddress objects.
-            boost::multi_index::ordered_unique<
-                boost::multi_index::member<Lease, isc::asiolink::IOAddress, &Lease::addr_>
-            >,
-
-            // Specification of the second index starts here.
-            boost::multi_index::ordered_non_unique<
-                // This is a composite index that will be used to search for
-                // the lease using three attributes: DUID, IAID and lease type.
-                boost::multi_index::composite_key<
-                    Lease6,
-                    // The DUID can be retrieved from the Lease6 object using
-                    // a getDuidVector const function.
-                    boost::multi_index::const_mem_fun<Lease6, const std::vector<uint8_t>&,
-                                                      &Lease6::getDuidVector>,
-                    // The two other ingredients of this index are IAID and
-                    // lease type.
-                    boost::multi_index::member<Lease6, uint32_t, &Lease6::iaid_>,
-                    boost::multi_index::member<Lease6, Lease::Type, &Lease6::type_>
-                >
-            >
-        >
-     > Lease6Storage; // Specify the type name of this container.
-
-    // This is a multi-index container, which holds elements that can
-    // be accessed using different search indexes.
-    typedef boost::multi_index_container<
-        // It holds pointers to Lease4 objects.
-        Lease4Ptr,
-        // Specification of search indexes starts here.
-        boost::multi_index::indexed_by<
-            // Specification of the first index starts here.
-            // This index sorts leases by IPv4 addresses represented as
-            // IOAddress objects.
-            boost::multi_index::ordered_unique<
-                // The IPv4 address are held in addr_ members that belong to
-                // Lease class.
-                boost::multi_index::member<Lease, isc::asiolink::IOAddress, &Lease::addr_>
-            >,
-
-            // Specification of the second index starts here.
-            boost::multi_index::ordered_unique<
-                // This is a composite index that combines two attributes of the
-                // Lease4 object: hardware address and subnet id.
-                boost::multi_index::composite_key<
-                    Lease4,
-                    // The hardware address is held in the hwaddr_ member of the
-                    // Lease4 object, which is a HWAddr object. Boost does not
-                    // provide a key extractor for getting a member of a member,
-                    // so we need a simple method for that.
-                    boost::multi_index::const_mem_fun<Lease, const std::vector<uint8_t>&,
-                                               &Lease::getHWAddrVector>,
-                    // The subnet id is held in the subnet_id_ member of Lease4
-                    // class. Note that the subnet_id_ is defined in the base
-                    // class (Lease) so we have to point to this class rather
-                    // than derived class: Lease4.
-                    boost::multi_index::member<Lease, SubnetID, &Lease::subnet_id_>
-                >
-            >,
-
-            // Specification of the third index starts here.
-            boost::multi_index::ordered_non_unique<
-                // This is a composite index that uses two values to search for a
-                // lease: client id and subnet id.
-                boost::multi_index::composite_key<
-                    Lease4,
-                    // The client id can be retrieved from the Lease4 object by
-                    // calling getClientIdVector const function.
-                    boost::multi_index::const_mem_fun<Lease4, const std::vector<uint8_t>&,
-                                                      &Lease4::getClientIdVector>,
-                    // The subnet id is accessed through the subnet_id_ member.
-                    boost::multi_index::member<Lease, uint32_t, &Lease::subnet_id_>
-                >
-            >,
-
-            // Specification of the fourth index starts here.
-            boost::multi_index::ordered_non_unique<
-                // This is a composite index that uses two values to search for a
-                // lease: client id and subnet id.
-                boost::multi_index::composite_key<
-                    Lease4,
-                    // The client id can be retrieved from the Lease4 object by
-                    // calling getClientIdVector const function.
-                    boost::multi_index::const_mem_fun<Lease4, const std::vector<uint8_t>&,
-                                                      &Lease4::getClientIdVector>,
-                    // The hardware address is held in the hwaddr_ object. We can
-                    // access the raw data using lease->hwaddr_->hwaddr_, but Boost
-                    // doesn't seem to provide a way to use member of a member for this,
-                    // so we need a simple key extractor method (getRawHWAddr).
-                    boost::multi_index::const_mem_fun<Lease, const std::vector<uint8_t>&,
-                                            &Lease::getHWAddrVector>,
-                    // The subnet id is accessed through the subnet_id_ member.
-                    boost::multi_index::member<Lease, SubnetID, &Lease::subnet_id_>
-                >
-            >
-        >
-    > Lease4Storage; // Specify the type name for this container.
-
     /// @brief stores IPv4 leases
     Lease4Storage storage4_;
 

src/lib/dhcpsrv/tests/Makefile.am

@@ -74,6 +74,7 @@ libdhcpsrv_unittests_SOURCES += host_unittest.cc
 libdhcpsrv_unittests_SOURCES += host_reservation_parser_unittest.cc
 libdhcpsrv_unittests_SOURCES += host_reservations_list_parser_unittest.cc
 libdhcpsrv_unittests_SOURCES += lease_file_io.cc lease_file_io.h
+libdhcpsrv_unittests_SOURCES += lease_file_loader_unittest.cc
 libdhcpsrv_unittests_SOURCES += lease_unittest.cc
 libdhcpsrv_unittests_SOURCES += lease_mgr_factory_unittest.cc
 libdhcpsrv_unittests_SOURCES += lease_mgr_unittest.cc

src/lib/dhcpsrv/tests/lease_file_loader_unittest.cc

@@ -0,0 +1,313 @@
+// Copyright (C) 2015 Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <config.h>
+#include <asiolink/io_address.h>
+#include <dhcpsrv/csv_lease_file4.h>
+#include <dhcpsrv/csv_lease_file6.h>
+#include <dhcpsrv/inmemory_lease_storage.h>
+#include <dhcpsrv/lease_file_loader.h>
+#include <dhcpsrv/tests/lease_file_io.h>
+#include <boost/scoped_ptr.hpp>
+#include <gtest/gtest.h>
+#include <sstream>
+#include <string>
+
+using namespace isc;
+using namespace isc::asiolink;
+using namespace isc::dhcp;
+using namespace isc::dhcp::test;
+
+namespace {
+
+/// @brief Test fixture class for @c LeaseFileLoader class.
+class LeaseFileLoaderTest : public ::testing::Test {
+public:
+
+    /// @brief Constructor.
+    ///
+    /// Initializes filename used for unit tests and creates an
+    /// IO object to be used to write to this file.
+    LeaseFileLoaderTest();
+
+    /// @brief Prepends the absolute path to the file specified
+    /// as an argument.
+    ///
+    /// @param filename Name of the file.
+    /// @return Absolute path to the test file.
+    static std::string absolutePath(const std::string& filename);
+
+    /// @brief Retrieves the lease from the storage using an IP address.
+    ///
+    /// This method returns the pointer to the @c Lease4 or @c Lease6
+    /// object representing the lease in the container. This is used to
+    /// check if the lease was parsed in the lease file and added to the
+    /// container by the @c LeaseFileLoader::load method.
+    ///
+    /// @param address A string representation of the leased address.
+    /// @param storage A reference to the container in which the lease
+    /// is to be searched.
+    /// @tparam LeasePtrType Type of the returned object: @c Lease4Ptr
+    /// @c Lease6Ptr.
+    /// @tparam LeaseStorage Type of the container: @c Lease4Container
+    /// @c Lease6Container.
+    ///
+    /// @return A pointer to the lease or NULL if no lease found.
+    template<typename LeasePtrType, typename LeaseStorage>
+    static LeasePtrType getLease(const std::string& address, const LeaseStorage& storage) {
+        typedef typename LeaseStorage::template nth_index<0>::type SearchIndex;
+        // Both Lease4Storage and Lease6Storage use index 0 to retrieve the
+        // lease using an IP address.
+        const SearchIndex& idx = storage.template get<0>();
+        typename SearchIndex::iterator lease = idx.find(IOAddress(address));
+        // Lease found. Return it.
+        if (lease != idx.end()) {
+            return (*lease);
+        }
+        // No lease found.
+        return (LeasePtrType());
+    }
+
+    /// @brief Name of the test lease file.
+    std::string filename_;
+
+    /// @brief Object providing access to lease file IO.
+    LeaseFileIO io_;
+};
+
+LeaseFileLoaderTest::LeaseFileLoaderTest()
+    : filename_("leases4.csv"), io_(absolutePath(filename_)) {
+}
+
+std::string
+LeaseFileLoaderTest::absolutePath(const std::string& filename) {
+    std::ostringstream s;
+    s << DHCP_DATA_DIR << "/" << filename;
+    return (s.str());
+}
+
+// This test verifies that the DHCPv4 leases can be loaded from the lease
+// file and that only the most recent entry for each lease is loaded and
+// the previous entries are discarded.
+TEST_F(LeaseFileLoaderTest, load4) {
+    // Create lease file with leases for 192.0.2.1, 192.0.3.15. The lease
+    // entry for the 192.0.2.3 is invalid (lacks HW address) and should
+    // be discarded.
+    io_.writeFile("address,hwaddr,client_id,valid_lifetime,expire,subnet_id,"
+                  "fqdn_fwd,fqdn_rev,hostname\n"
+                  "192.0.2.1,06:07:08:09:0a:bc,,200,200,8,1,1,"
+                  "host.example.com\n"
+                  "192.0.3.15,dd:de:ba:0d:1b:2e:3e:4f,0a:00:01:04,100,100,7,"
+                  "0,0,\n"
+                  "192.0.2.3,,a:11:01:04,200,200,8,1,1,host.example.com\n"
+                  "192.0.3.15,dd:de:ba:0d:1b:2e:3e:4f,0a:00:01:04,100,135,7,"
+                  "0,0,\n"
+                  "192.0.2.1,06:07:08:09:0a:bc,,200,500,8,1,1,"
+                  "host.example.com\n");
+
+    boost::scoped_ptr<CSVLeaseFile4> lf(new CSVLeaseFile4(filename_));
+    ASSERT_NO_THROW(lf->open());
+
+    // Load leases from the file.
+    Lease4Storage storage;
+    ASSERT_NO_THROW(LeaseFileLoader::load<Lease4>(*lf, storage, 10));
+
+    // There are two unique leases.
+    ASSERT_EQ(2, storage.size());
+
+    // The lease for 192.0.2.1 should exist and the cltt should be
+    // set to the expire-valid_lifetime for the second entry for
+    // this lease, i.e. 500 - 200 = 300.
+    Lease4Ptr lease = getLease<Lease4Ptr>("192.0.2.1", storage);
+    ASSERT_TRUE(lease);
+    EXPECT_EQ(300, lease->cltt_);
+
+    // The invalid entry should not be loaded.
+    lease = getLease<Lease4Ptr>("192.0.2.3", storage);
+    ASSERT_FALSE(lease);
+
+    // The other lease should be present and the cltt time should
+    // be set according to the values in the second entry for this
+    // lease.
+    lease = getLease<Lease4Ptr>("192.0.3.15", storage);
+    ASSERT_TRUE(lease);
+    EXPECT_EQ(35, lease->cltt_);
+}
+
+// This test verifies that the lease with a valid lifetime of 0
+// is removed from the storage. The valid lifetime of 0 set set
+// for the released leases.
+TEST_F(LeaseFileLoaderTest, load4LeaseRemove) {
+    // Create lease file in which one of the entries for 192.0.2.1
+    // has a valid_lifetime of 0 and results in the deletion of the
+    // lease.
+    io_.writeFile("address,hwaddr,client_id,valid_lifetime,expire,subnet_id,"
+                  "fqdn_fwd,fqdn_rev,hostname\n"
+                  "192.0.2.1,06:07:08:09:0a:bc,,200,200,8,1,1,"
+                  "host.example.com\n"
+                  "192.0.3.15,dd:de:ba:0d:1b:2e:3e:4f,0a:00:01:04,100,100,7,"
+                  "0,0,\n"
+                  "192.0.3.15,dd:de:ba:0d:1b:2e:3e:4f,0a:00:01:04,100,135,7,"
+                  "0,0,\n"
+                  "192.0.2.1,06:07:08:09:0a:bc,,0,500,8,1,1,"
+                  "host.example.com\n");
+
+    boost::scoped_ptr<CSVLeaseFile4> lf(new CSVLeaseFile4(filename_));
+    ASSERT_NO_THROW(lf->open());
+
+    Lease4Storage storage;
+    ASSERT_NO_THROW(LeaseFileLoader::load<Lease4>(*lf, storage, 10));
+
+    // There should only be one lease. The one with the valid_lifetime
+    // of 0 should be removed.
+    ASSERT_EQ(1, storage.size());
+
+    Lease4Ptr lease = getLease<Lease4Ptr>("192.0.3.15", storage);
+    ASSERT_TRUE(lease);
+    EXPECT_EQ(35, lease->cltt_);
+}
+
+// This test verifies that the DHCPv6 leases can be loaded from the lease
+// file and that only the most recent entry for each lease is loaded and
+// the previous entries are discarded.
+TEST_F(LeaseFileLoaderTest, load6) {
+    // Create a lease file with three valid leases: 2001:db8:1::1,
+    // 3000:1:: and 2001:db8:2::10.
+    io_.writeFile("address,duid,valid_lifetime,expire,subnet_id,"
+                  "pref_lifetime,lease_type,iaid,prefix_len,fqdn_fwd,"
+                  "fqdn_rev,hostname,hwaddr\n"
+                  "2001:db8:1::1,00:01:02:03:04:05:06:0a:0b:0c:0d:0e:0f,"
+                  "200,200,8,100,0,7,0,1,1,host.example.com,\n"
+                  "2001:db8:1::1,,200,200,8,100,0,7,0,1,1,host.example.com,\n"
+                  "2001:db8:2::10,01:01:01:01:0a:01:02:03:04:05,300,300,6,150,"
+                  "0,8,0,0,0,,\n"
+                  "3000:1::,00:01:02:03:04:05:06:0a:0b:0c:0d:0e:0f,100,200,8,0,2,"
+                  "16,64,0,0,,\n"
+                  "2001:db8:2::10,01:01:01:01:0a:01:02:03:04:05,300,800,6,150,"
+                  "0,8,0,0,0,,\n"
+                  "2001:db8:1::1,00:01:02:03:04:05:06:0a:0b:0c:0d:0e:0f,"
+                  "200,400,8,100,0,7,0,1,1,host.example.com,\n");
+
+    boost::scoped_ptr<CSVLeaseFile6> lf(new CSVLeaseFile6(filename_));
+    ASSERT_NO_THROW(lf->open());
+
+    // Load leases from the lease file.
+    Lease6Storage storage;
+    ASSERT_NO_THROW(LeaseFileLoader::load<Lease6>(*lf, storage, 10));
+
+    // There should be 3 unique leases.
+    ASSERT_EQ(3, storage.size());
+
+    // The 2001:db8:1::1 should be present and its cltt should be
+    // calculated according to the expiration time and the valid
+    // lifetime from the last entry for this lease: 400 - 200 = 200.
+    Lease6Ptr lease = getLease<Lease6Ptr>("2001:db8:1::1", storage);
+    ASSERT_TRUE(lease);
+    EXPECT_EQ(200, lease->cltt_);
+
+    // The 3000:1:: lease should be present.
+    lease = getLease<Lease6Ptr>("3000:1::", storage);
+    ASSERT_TRUE(lease);
+    EXPECT_EQ(100, lease->cltt_);
+
+    // The 2001:db8:2::10 should be present and the cltt should be
+    // calculated according to the last entry in the lease file.
+    lease = getLease<Lease6Ptr>("2001:db8:2::10", storage);
+    ASSERT_TRUE(lease);
+    EXPECT_EQ(500, lease->cltt_);
+}
+
+// This test verifies that the lease with a valid lifetime of 0
+// is removed from the storage. The valid lifetime of 0 set set
+// for the released leases.
+TEST_F(LeaseFileLoaderTest, load6LeaseRemove) {
+    // Create lease file in which one of the entries for the 2001:db8:1::1
+    // has valid lifetime set to 0, in which case the lease should be
+    // deleted.
+    io_.writeFile("address,duid,valid_lifetime,expire,subnet_id,"
+                  "pref_lifetime,lease_type,iaid,prefix_len,fqdn_fwd,"
+                  "fqdn_rev,hostname,hwaddr\n"
+                  "2001:db8:1::1,00:01:02:03:04:05:06:0a:0b:0c:0d:0e:0f,"
+                  "200,200,8,100,0,7,0,1,1,host.example.com,\n"
+                  "2001:db8:2::10,01:01:01:01:0a:01:02:03:04:05,300,300,6,150,"
+                  "0,8,0,0,0,,\n"
+                  "2001:db8:2::10,01:01:01:01:0a:01:02:03:04:05,300,800,6,150,"
+                  "0,8,0,0,0,,\n"
+                  "2001:db8:1::1,00:01:02:03:04:05:06:0a:0b:0c:0d:0e:0f,"
+                  "0,400,8,100,0,7,0,1,1,host.example.com,\n");
+
+    boost::scoped_ptr<CSVLeaseFile6> lf(new CSVLeaseFile6(filename_));
+    ASSERT_NO_THROW(lf->open());
+
+    // Loaded leases.
+    Lease6Storage storage;
+    ASSERT_NO_THROW(LeaseFileLoader::load<Lease6>(*lf, storage, 10));
+
+    // There should be only one lease for 2001:db8:2::10. The other one
+    // should have been deleted (or rather not loaded).
+    ASSERT_EQ(1, storage.size());
+
+    Lease6Ptr lease = getLease<Lease6Ptr>("2001:db8:2::10", storage);
+    ASSERT_TRUE(lease);
+    EXPECT_EQ(500, lease->cltt_);
+}
+
+// This test verifies that the exception is thrown when the specific
+// number of errors occurs during reading of the lease file.
+TEST_F(LeaseFileLoaderTest, loadMaxErrors) {
+    // Create a lease file for which there is a number of invalid
+    // entries.
+    io_.writeFile("address,hwaddr,client_id,valid_lifetime,expire,subnet_id,"
+                  "fqdn_fwd,fqdn_rev,hostname\n"
+                  "192.0.2.1,06:07:08:09:0a:bc,,200,200,8,1,1,"
+                  "host.example.com\n"
+                  "192.0.2.3,,a:11:01:04,200,200,8,1,1,host.example.com\n"
+                  "192.0.2.3,,a:11:01:04,200,200,8,1,1,host.example.com\n"
+                  "192.0.2.10,01:02:03:04:05:06,,200,300,8,1,1,,\n"
+                  "192.0.2.3,,a:11:01:04,200,200,8,1,1,host.example.com\n"
+                  "192.0.2.3,,a:11:01:04,200,200,8,1,1,host.example.com\n"
+                  "192.0.2.1,06:07:08:09:0a:bc,,200,500,8,1,1,"
+                  "host.example.com\n");
+
+    boost::scoped_ptr<CSVLeaseFile4> lf(new CSVLeaseFile4(filename_));
+    ASSERT_NO_THROW(lf->open());
+
+    // Load leases and set the maximum number of errors to 3. This
+    // should result in an exception because there are 4 invalid entries.
+    Lease4Storage storage;
+    ASSERT_THROW(LeaseFileLoader::load<Lease4>(*lf, storage, 3),
+                 util::CSVFileError);
+
+    lf->close();
+    ASSERT_NO_THROW(lf->open());
+
+    // Repeat the test, but this time allow for 4 invalid entries. It
+    // should load just fine.
+    storage.clear();
+    ASSERT_NO_THROW(LeaseFileLoader::load<Lease4>(*lf, storage, 4));
+
+    ASSERT_EQ(2, storage.size());
+
+    Lease4Ptr lease = getLease<Lease4Ptr>("192.0.2.1", storage);
+    ASSERT_TRUE(lease);
+    EXPECT_EQ(300, lease->cltt_);
+
+    lease = getLease<Lease4Ptr>("192.0.2.10", storage);
+    ASSERT_TRUE(lease);
+    EXPECT_EQ(100, lease->cltt_);
+}
+
+
+
+} // end of anonymous namespace