kea/src/lib/dhcpsrv/memfile_lease_mgr.cc

// Copyright (C) 2012-2016 Internet Systems Consortium, Inc. ("ISC")
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

#include <config.h>
#include <dhcpsrv/cfgmgr.h>
#include <dhcpsrv/dhcpsrv_log.h>
#include <dhcpsrv/lease_file_loader.h>
#include <dhcpsrv/memfile_lease_mgr.h>
#include <dhcpsrv/timer_mgr.h>
#include <dhcpsrv/database_connection.h>
#include <exceptions/exceptions.h>
#include <util/pid_file.h>
#include <util/process_spawn.h>
#include <util/signal_set.h>
#include <cstdio>
#include <cstring>
#include <errno.h>
#include <iostream>
#include <limits>
#include <sstream>

namespace {

/// @brief Maximum number of errors to read the leases from the lease file.
const uint32_t MAX_LEASE_ERRORS = 100;

/// @brief A name of the environmental variable specifying the kea-lfc
/// program location.
///
/// This variable can be set by tests to point to the location of the
/// kea-lfc program within a build directory. If this variable is not
/// set, the backend will use the location of the kea-lfc in the
/// Kea installation directory.
const char* KEA_LFC_EXECUTABLE_ENV_NAME = "KEA_LFC_EXECUTABLE";

} // end of anonymous namespace

using namespace isc::util;

namespace isc {
namespace dhcp {

/// @brief Represents a configuration for Lease File Cleanup.
///
/// This class is solely used by the @c Memfile_LeaseMgr as a configuration
/// information storage for %Lease File Cleanup. Internally, it creates
/// the interval timer and assigns a callback function (pointer to which is
/// passed in the constructor), which will be called at the specified
/// intervals to perform the cleanup. It is also responsible for creating
/// and maintaing the object which is used to spawn the new process which
/// executes the @c kea-lfc program.
///
/// This functionality is enclosed in a separate class so as the implementation
/// details are not exposed in the @c Memfile_LeaseMgr header file and
/// to maintain a single place with the LFC configuration, instead of multiple
/// members and functions scattered in the @c Memfile_LeaseMgr class.
class LFCSetup {
public:

    /// @brief Constructor.
    ///
    /// Assigns a pointer to the function triggered to perform the cleanup.
    /// This pointer should point to the appropriate method of the
    /// @c Memfile_LeaseMgr class.
    ///
    /// @param callback A pointer to the callback function.
    LFCSetup(asiolink::IntervalTimer::Callback callback);

    /// @brief Destructor.
    ///
    /// Unregisters LFC timer.
    ~LFCSetup();

    /// @brief Sets the new configuration for the %Lease File Cleanup.
    ///
    /// @param lfc_interval An interval in seconds at which the cleanup should
    /// be performed.
    /// @param lease_file4 A pointer to the DHCPv4 lease file to be cleaned up
    /// or NULL. If this is NULL, the @c lease_file6 must be non-null.
    /// @param lease_file6 A pointer to the DHCPv6 lease file to be cleaned up
    /// or NULL. If this is NULL, the @c lease_file4 must be non-null.
    /// @param run_once_now A flag that causes LFC to be invoked immediately,
    /// regardless of the value of lfc_interval.  This is primarily used to
    /// cause lease file schema upgrades upon startup.
    void setup(const uint32_t lfc_interval,
               const boost::shared_ptr<CSVLeaseFile4>& lease_file4,
               const boost::shared_ptr<CSVLeaseFile6>& lease_file6,
               bool run_once_now = false);

    /// @brief Spawns a new process.
    void execute();

    /// @brief Checks if the lease file cleanup is in progress.
    ///
    /// @return true if the lease file cleanup is being executed.
    bool isRunning() const;

    /// @brief Returns exit code of the last completed cleanup.
    int getExitStatus() const;

private:

    /// @brief A pointer to the @c ProcessSpawn object used to execute
    /// the LFC.
    boost::scoped_ptr<util::ProcessSpawn> process_;

    /// @brief A pointer to the callback function executed by the timer.
    asiolink::IntervalTimer::Callback callback_;

    /// @brief A PID of the last executed LFC process.
    pid_t pid_;

    /// @brief Pointer to the timer manager.
    ///
    /// We have to hold this pointer here to make sure that the timer
    /// manager is not destroyed before the lease manager.
    TimerMgrPtr timer_mgr_;
};

LFCSetup::LFCSetup(asiolink::IntervalTimer::Callback callback)
    : process_(), callback_(callback), pid_(0),
      timer_mgr_(TimerMgr::instance()) {
}

LFCSetup::~LFCSetup() {
    try {
        // If we're here it means that either the process is terminating
        // or we're reconfiguring the server. In both cases the thread has
        // probably been stopped already, but we make sure by calling
        // stopThread explicitly here.
        timer_mgr_->stopThread();

        // Remove the timer. This will throw an exception if the timer does not
        // exist.  There are several possible reasons for this:
        // a) It hasn't been registered (although if the LFC Setup instance
        //    exists it means that the timer must have been registered or that
        //    such registration has been attempted).
        // b) The registration may fail if the duplicate timer exists or if the
        //    TimerMgr's worker thread is running but if this happens it is a
        //    programming error.
        // c) The program is shutting down and the timer has been removed by
        //    another component.
        timer_mgr_->unregisterTimer("memfile-lfc");

    } catch (const std::exception& ex) {
        // We don't want exceptions being thrown from the destructor so we just
        // log a message here.  The message is logged at debug severity as
        // we don't want an error message output during shutdown.
        LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE,
                  DHCPSRV_MEMFILE_LFC_UNREGISTER_TIMER_FAILED).arg(ex.what());
    }
}

void
LFCSetup::setup(const uint32_t lfc_interval,
                const boost::shared_ptr<CSVLeaseFile4>& lease_file4,
                const boost::shared_ptr<CSVLeaseFile6>& lease_file6,
                bool run_once_now) {

    // If to nothing to do, punt
    if (lfc_interval == 0 && !run_once_now) {
        return;
    }

    // Start preparing the command line for kea-lfc.
    std::string executable;
    char* c_executable = getenv(KEA_LFC_EXECUTABLE_ENV_NAME);
    if (c_executable == NULL) {
        executable = KEA_LFC_EXECUTABLE;
    } else {
        executable = c_executable;
    }

    // Gather the base file name.
    std::string lease_file = lease_file4 ? lease_file4->getFilename() :
                                           lease_file6->getFilename();

    // Create the other names by appending suffixes to the base name.
    util::ProcessArgs args;
    // Universe: v4 or v6.
    args.push_back(lease_file4 ? "-4" : "-6");

    // Previous file.
    args.push_back("-x");
    args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
                                                  Memfile_LeaseMgr::FILE_PREVIOUS));
    // Input file.
    args.push_back("-i");
    args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
                                                  Memfile_LeaseMgr::FILE_INPUT));
    // Output file.
    args.push_back("-o");
    args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
                                                  Memfile_LeaseMgr::FILE_OUTPUT));
    // Finish file.
    args.push_back("-f");
    args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
                                                  Memfile_LeaseMgr::FILE_FINISH));
    // PID file.
    args.push_back("-p");
    args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
                                                  Memfile_LeaseMgr::FILE_PID));

    // The configuration file is currently unused.
    args.push_back("-c");
    args.push_back("ignored-path");

    // Create the process (do not start it yet).
    process_.reset(new util::ProcessSpawn(executable, args));

    // If we've been told to run it once now, invoke the callback directly.
    if (run_once_now) {
        callback_();
    }

    // If it's suposed to run periodically, setup that now.
    if (lfc_interval > 0) {
        // Set the timer to call callback function periodically.
        LOG_INFO(dhcpsrv_logger, DHCPSRV_MEMFILE_LFC_SETUP).arg(lfc_interval);

        // Multiple the lfc_interval value by 1000 as this value specifies
        // a timeout in seconds, whereas the setup() method expects the
        // timeout in milliseconds.
        timer_mgr_->registerTimer("memfile-lfc", callback_, lfc_interval * 1000,
                                  asiolink::IntervalTimer::REPEATING);
        timer_mgr_->setup("memfile-lfc");
    }
}

void
LFCSetup::execute() {
    try {
        LOG_INFO(dhcpsrv_logger, DHCPSRV_MEMFILE_LFC_EXECUTE)
            .arg(process_->getCommandLine());
        pid_ = process_->spawn();

    } catch (const ProcessSpawnError&) {
        LOG_ERROR(dhcpsrv_logger, DHCPSRV_MEMFILE_LFC_SPAWN_FAIL);
    }
}

bool
LFCSetup::isRunning() const {
    return (process_ && process_->isRunning(pid_));
}

int
LFCSetup::getExitStatus() const {
    if (!process_) {
        isc_throw(InvalidOperation, "unable to obtain LFC process exit code: "
                  " the process is NULL");
    }
    return (process_->getExitStatus(pid_));
}


/// @brief Memfile derivation of the IPv4 statistical lease data query
///
/// This class is used to recalculate IPv4 lease statistics for Memfile
/// lease storage.  It does so by iterating over the given storage,
/// accumulating counts of leases in each of the monitored lease states
/// for each subnet and storing these counts in an internal collection.
/// The populated result set will contain one entry per monitored state
/// per subnet.
///
class MemfileAddressStatsQuery4 : public AddressStatsQuery4 {
public:
    /// @brief Constructor
    ///
    /// @param storage4 A pointer to the v4 lease storage to be counted
    MemfileAddressStatsQuery4(Lease4Storage& storage4);

    /// @brief Destructor
    virtual ~MemfileAddressStatsQuery4() {};

    /// @brief Creates the IPv4 lease statistical data result set
    ///
    /// The result is populated by iterating over the IPv4 leases in storage,
    /// in ascending order by subnet ID, accumulating the lease state counts.
    /// At the completion of all entries for a given subnet, the counts are 
    /// used to create AddressStatsRow4 instances which are appended to an 
    /// internal vector.  The process results in a vector containing one entry
    /// per state per subnet.
    ///
    /// Currently the states counted are:
    ///
    /// - Lease::STATE_DEFAULT (i.e. assigned) 
    /// - Lease::STATE_DECLINED 
    virtual void start();

    /// @brief Fetches the next row in the result set
    ///
    /// Once the internal result set has been populated by invoking the
    /// the start() method, this method is used to iterate over the
    /// result set rows.  Once the last row has been fetched, subsequent
    /// calls will return false. 
    /// @param row Storage for the fetched row
    ///
    /// @return True if the fetch succeeded, false if there are no more
    /// rows to fetch.
    virtual bool getNextRow(AddressStatsRow4& row);

    /// @brief Returns the number of rows in the result set
    /// @todo, should this be a virtual member of the base class?
    int getRowCount();

private:
    /// @brief The Memfile storage containing the IPv4 leases to analyze
    Lease4Storage& storage4_;

    /// @brief A vector containing the "result set" 
    std::vector<AddressStatsRow4> rows_;

    /// @brief An iterator for accessing the next row within the result set
    std::vector<AddressStatsRow4>::iterator next_pos_;
};

MemfileAddressStatsQuery4::MemfileAddressStatsQuery4(Lease4Storage& storage4) 
    : storage4_(storage4), rows_(0), next_pos_(rows_.end()) {};

void 
MemfileAddressStatsQuery4::start() {
    // Get the subnet_id index
    const Lease4StorageSubnetIdIndex& idx = storage4_.get<SubnetIdIndexTag>();

    // Iterate over the leases in order by subnet, accumulating per
    // subnet counts for each state of interest.  As we finish each
    // subnet, add the appropriate rows to our result set.
    SubnetID cur_id = 0;
    int64_t assigned = 0;
    int64_t declined = 0;
    for(Lease4StorageSubnetIdIndex::const_iterator lease = idx.begin();
        lease != idx.end(); ++lease) {

        // If we've hit the next subnet, add rows for the current subnet
        // and wipe the accumulators
        if ((*lease)->subnet_id_ > cur_id) {
            if (cur_id > 0) {
                rows_.push_back(AddressStatsRow4(cur_id,Lease::STATE_DEFAULT, 
                                                 assigned));
                assigned = 0;
                rows_.push_back(AddressStatsRow4(cur_id, Lease::STATE_DECLINED, 
                                                 declined));
                declined = 0;
            }

            // Update current subnet id
            cur_id = (*lease)->subnet_id_;
        }

        // Bump the appropriate accumulator
        switch ((*lease)->state_) {
        case Lease::STATE_DEFAULT:
            ++assigned;
            break;
        case Lease::STATE_DECLINED:
            ++declined;
            break;
        default:
            // Not one we're tracking.
            break;
        }
    }

    // Make the rows for last subnet, unless there were no rows
    if (idx.begin() != idx.end()) {
        rows_.push_back(AddressStatsRow4(cur_id, Lease::STATE_DEFAULT,
                                         assigned));
        rows_.push_back(AddressStatsRow4(cur_id, Lease::STATE_DECLINED, 
                                         declined));
    }

    // Set the next row position to the beginning of the rows.
    next_pos_ = rows_.begin();
}

bool 
MemfileAddressStatsQuery4::getNextRow(AddressStatsRow4& row) { 
    if (next_pos_ == rows_.end()) {
        return (false);
    }

    row = *next_pos_;
    ++next_pos_;
    return (true);
}

int 
MemfileAddressStatsQuery4::getRowCount() {
    return (rows_.size());
}

// Explicit definition of class static constants.  Values are given in the
// declaration so they're not needed here.
const int Memfile_LeaseMgr::MAJOR_VERSION;
const int Memfile_LeaseMgr::MINOR_VERSION;

Memfile_LeaseMgr::Memfile_LeaseMgr(const DatabaseConnection::ParameterMap& parameters)
    : LeaseMgr(), lfc_setup_(), conn_(parameters)
    {
    bool conversion_needed = false;

    // Check the universe and use v4 file or v6 file.
    std::string universe = conn_.getParameter("universe");
    if (universe == "4") {
        std::string file4 = initLeaseFilePath(V4);
        if (!file4.empty()) {
            conversion_needed = loadLeasesFromFiles<Lease4,
                                                 CSVLeaseFile4>(file4,
                                                                lease_file4_,
                                                                storage4_);
        }
    } else {
        std::string file6 = initLeaseFilePath(V6);
        if (!file6.empty()) {
            conversion_needed = loadLeasesFromFiles<Lease6,
                                                 CSVLeaseFile6>(file6,
                                                                lease_file6_,
                                                                storage6_);
        }
    }

    // If lease persistence have been disabled for both v4 and v6,
    // issue a warning. It is ok not to write leases to disk when
    // doing testing, but it should not be done in normal server
    // operation.
   if (!persistLeases(V4) && !persistLeases(V6)) {
        LOG_WARN(dhcpsrv_logger, DHCPSRV_MEMFILE_NO_STORAGE);
    } else  {
        if (conversion_needed) {
            LOG_WARN(dhcpsrv_logger, DHCPRSV_MEMFILE_CONVERTING_LEASE_FILES)
                    .arg(MAJOR_VERSION).arg(MINOR_VERSION);
        }
        lfcSetup(conversion_needed);
    }

}

Memfile_LeaseMgr::~Memfile_LeaseMgr() {
    if (lease_file4_) {
        lease_file4_->close();
        lease_file4_.reset();
    }
    if (lease_file6_) {
        lease_file6_->close();
        lease_file6_.reset();
    }
}

std::string
Memfile_LeaseMgr::getDBVersion() {
    std::stringstream tmp;
    tmp << "Memfile backend " << MAJOR_VERSION;
    tmp << "." << MINOR_VERSION;
    return (tmp.str());
}

bool
Memfile_LeaseMgr::addLease(const Lease4Ptr& lease) {
    LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL,
              DHCPSRV_MEMFILE_ADD_ADDR4).arg(lease->addr_.toText());

    if (getLease4(lease->addr_)) {
        // there is a lease with specified address already
        return (false);
    }

    // Try to write a lease to disk first. If this fails, the lease will
    // not be inserted to the memory and the disk and in-memory data will
    // remain consistent.
    if (persistLeases(V4)) {
        lease_file4_->append(*lease);
    }

    storage4_.insert(lease);
    return (true);
}

bool
Memfile_LeaseMgr::addLease(const Lease6Ptr& lease) {
    LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL,
              DHCPSRV_MEMFILE_ADD_ADDR6).arg(lease->addr_.toText());

    if (getLease6(lease->type_, lease->addr_)) {
        // there is a lease with specified address already
        return (false);
    }

    // Try to write a lease to disk first. If this fails, the lease will
    // not be inserted to the memory and the disk and in-memory data will
    // remain consistent.
    if (persistLeases(V6)) {
        lease_file6_->append(*lease);
    }

    storage6_.insert(lease);
    return (true);
}

Lease4Ptr
Memfile_LeaseMgr::getLease4(const isc::asiolink::IOAddress& addr) const {
    LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL,
              DHCPSRV_MEMFILE_GET_ADDR4).arg(addr.toText());

    const Lease4StorageAddressIndex& idx = storage4_.get<AddressIndexTag>();
    Lease4StorageAddressIndex::iterator l = idx.find(addr);
    if (l == idx.end()) {
        return (Lease4Ptr());
    } else {
        return (Lease4Ptr(new Lease4(**l)));
    }
}

Lease4Collection
Memfile_LeaseMgr::getLease4(const HWAddr& hwaddr) const {
    LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL,
              DHCPSRV_MEMFILE_GET_HWADDR).arg(hwaddr.toText());
    Lease4Collection collection;
    const Lease4StorageAddressIndex& idx = storage4_.get<AddressIndexTag>();
    for(Lease4StorageAddressIndex::const_iterator lease = idx.begin();
        lease != idx.end(); ++lease) {

        // Every Lease4 has a hardware address, so we can compare it
        if ( (*(*lease)->hwaddr_) == hwaddr) {
            collection.push_back((*lease));
        }
    }

    return (collection);
}

Lease4Ptr
Memfile_LeaseMgr::getLease4(const HWAddr& hwaddr, SubnetID subnet_id) const {
    LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL,
              DHCPSRV_MEMFILE_GET_SUBID_HWADDR).arg(subnet_id)
        .arg(hwaddr.toText());

    // Get the index by HW Address and Subnet Identifier.
    const Lease4StorageHWAddressSubnetIdIndex& idx =
        storage4_.get<HWAddressSubnetIdIndexTag>();
    // Try to find the lease using HWAddr and subnet id.
    Lease4StorageHWAddressSubnetIdIndex::const_iterator lease =
        idx.find(boost::make_tuple(hwaddr.hwaddr_, subnet_id));
    // Lease was not found. Return empty pointer to the caller.
    if (lease == idx.end()) {
        return (Lease4Ptr());
    }

    // Lease was found. Return it to the caller.
    return (Lease4Ptr(new Lease4(**lease)));
}

Lease4Collection
Memfile_LeaseMgr::getLease4(const ClientId& client_id) const {
    LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL,
              DHCPSRV_MEMFILE_GET_CLIENTID).arg(client_id.toText());
    Lease4Collection collection;
    const Lease4StorageAddressIndex& idx = storage4_.get<AddressIndexTag>();
    for(Lease4StorageAddressIndex::const_iterator lease = idx.begin();
        lease != idx.end(); ++ lease) {

        // client-id is not mandatory in DHCPv4. There can be a lease that does
        // not have a client-id. Dereferencing null pointer would be a bad thing
        if((*lease)->client_id_ && *(*lease)->client_id_ == client_id) {
            collection.push_back((*lease));
        }
    }

    return (collection);
}

Lease4Ptr
Memfile_LeaseMgr::getLease4(const ClientId& client_id,
                            const HWAddr& hwaddr,
                            SubnetID subnet_id) const {
    LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL,
              DHCPSRV_MEMFILE_GET_CLIENTID_HWADDR_SUBID).arg(client_id.toText())
                                                        .arg(hwaddr.toText())
                                                        .arg(subnet_id);

    // Get the index by client id, HW address and subnet id.
    const Lease4StorageClientIdHWAddressSubnetIdIndex& idx =
        storage4_.get<ClientIdHWAddressSubnetIdIndexTag>();
    // Try to get the lease using client id, hardware address and subnet id.
    Lease4StorageClientIdHWAddressSubnetIdIndex::const_iterator lease =
        idx.find(boost::make_tuple(client_id.getClientId(), hwaddr.hwaddr_,
                                   subnet_id));

    if (lease == idx.end()) {
        // Lease was not found. Return empty pointer to the caller.
        return (Lease4Ptr());
    }

    // Lease was found. Return it to the caller.
    return (*lease);
}

Lease4Ptr
Memfile_LeaseMgr::getLease4(const ClientId& client_id,
                            SubnetID subnet_id) const {
    LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL,
              DHCPSRV_MEMFILE_GET_SUBID_CLIENTID).arg(subnet_id)
              .arg(client_id.toText());

    // Get the index by client and subnet id.
    const Lease4StorageClientIdSubnetIdIndex& idx =
        storage4_.get<ClientIdSubnetIdIndexTag>();
    // Try to get the lease using client id and subnet id.
    Lease4StorageClientIdSubnetIdIndex::const_iterator lease =
        idx.find(boost::make_tuple(client_id.getClientId(), subnet_id));
    // Lease was not found. Return empty pointer to the caller.
    if (lease == idx.end()) {
        return (Lease4Ptr());
    }
    // Lease was found. Return it to the caller.
    return (Lease4Ptr(new Lease4(**lease)));
}

Lease6Ptr
Memfile_LeaseMgr::getLease6(Lease::Type type,
                            const isc::asiolink::IOAddress& addr) const {
    LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL,
              DHCPSRV_MEMFILE_GET_ADDR6)
        .arg(addr.toText())
        .arg(Lease::typeToText(type));
    Lease6Storage::iterator l = storage6_.find(addr);
    if (l == storage6_.end() || !(*l) || ((*l)->type_ != type)) {
        return (Lease6Ptr());
    } else {
        return (Lease6Ptr(new Lease6(**l)));
    }
}

Lease6Collection
Memfile_LeaseMgr::getLeases6(Lease::Type type,
                            const DUID& duid, uint32_t iaid) const {
    LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL,
              DHCPSRV_MEMFILE_GET_IAID_DUID)
        .arg(iaid)
        .arg(duid.toText())
        .arg(Lease::typeToText(type));

    // Get the index by DUID, IAID, lease type.
    const Lease6StorageDuidIaidTypeIndex& idx = storage6_.get<DuidIaidTypeIndexTag>();
    // Try to get the lease using the DUID, IAID and lease type.
    std::pair<Lease6StorageDuidIaidTypeIndex::const_iterator,
              Lease6StorageDuidIaidTypeIndex::const_iterator> l =
        idx.equal_range(boost::make_tuple(duid.getDuid(), iaid, type));
    Lease6Collection collection;
    for(Lease6StorageDuidIaidTypeIndex::const_iterator lease =
            l.first; lease != l.second; ++lease) {
        collection.push_back(Lease6Ptr(new Lease6(**lease)));
    }

    return (collection);
}

Lease6Collection
Memfile_LeaseMgr::getLeases6(Lease::Type type,
                             const DUID& duid, uint32_t iaid,
                             SubnetID subnet_id) const {
    LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL,
              DHCPSRV_MEMFILE_GET_IAID_SUBID_DUID)
        .arg(iaid)
        .arg(subnet_id)
        .arg(duid.toText())
        .arg(Lease::typeToText(type));

    // Get the index by DUID, IAID, lease type.
    const Lease6StorageDuidIaidTypeIndex& idx = storage6_.get<DuidIaidTypeIndexTag>();
    // Try to get the lease using the DUID, IAID and lease type.
    std::pair<Lease6StorageDuidIaidTypeIndex::const_iterator,
              Lease6StorageDuidIaidTypeIndex::const_iterator> l =
        idx.equal_range(boost::make_tuple(duid.getDuid(), iaid, type));
    Lease6Collection collection;
    for(Lease6StorageDuidIaidTypeIndex::const_iterator lease =
            l.first; lease != l.second; ++lease) {
        // Filter out the leases which subnet id doesn't match.
        if((*lease)->subnet_id_ == subnet_id) {
            collection.push_back(Lease6Ptr(new Lease6(**lease)));
        }
    }

    return (collection);
}

void
Memfile_LeaseMgr::getExpiredLeases6(Lease6Collection& expired_leases,
                                    const size_t max_leases) const {
    LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL, DHCPSRV_MEMFILE_GET_EXPIRED4)
        .arg(max_leases);

    // Obtain the index which segragates leases by state and time.
    const Lease6StorageExpirationIndex& index = storage6_.get<ExpirationIndexTag>();

    // Retrieve leases which are not reclaimed and which haven't expired. The
    // 'less-than' operator will be used for both components of the index. So,
    // for the 'state' 'false' is less than 'true'. Also the leases with
    // expiration time lower than current time will be returned.
    Lease6StorageExpirationIndex::const_iterator ub =
        index.upper_bound(boost::make_tuple(false, time(NULL)));

    // Copy only the number of leases indicated by the max_leases parameter.
    for (Lease6StorageExpirationIndex::const_iterator lease = index.begin();
         (lease != ub) && ((max_leases == 0) || (std::distance(index.begin(), lease) <
                                                 max_leases));
         ++lease) {
        expired_leases.push_back(Lease6Ptr(new Lease6(**lease)));
    }
}

void
Memfile_LeaseMgr::getExpiredLeases4(Lease4Collection& expired_leases,
                                    const size_t max_leases) const {
    LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL, DHCPSRV_MEMFILE_GET_EXPIRED6)
        .arg(max_leases);

    // Obtain the index which segragates leases by state and time.
    const Lease4StorageExpirationIndex& index = storage4_.get<ExpirationIndexTag>();

    // Retrieve leases which are not reclaimed and which haven't expired. The
    // 'less-than' operator will be used for both components of the index. So,
    // for the 'state' 'false' is less than 'true'. Also the leases with
    // expiration time lower than current time will be returned.
    Lease4StorageExpirationIndex::const_iterator ub =
        index.upper_bound(boost::make_tuple(false, time(NULL)));

    // Copy only the number of leases indicated by the max_leases parameter.
    for (Lease4StorageExpirationIndex::const_iterator lease = index.begin();
         (lease != ub) && ((max_leases == 0) || (std::distance(index.begin(), lease) <
                                                 max_leases));
         ++lease) {
        expired_leases.push_back(Lease4Ptr(new Lease4(**lease)));
    }
}

void
Memfile_LeaseMgr::updateLease4(const Lease4Ptr& lease) {
    LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL,
              DHCPSRV_MEMFILE_UPDATE_ADDR4).arg(lease->addr_.toText());

    // Obtain 'by address' index.
    Lease4StorageAddressIndex& index = storage4_.get<AddressIndexTag>();

    // Lease must exist if it is to be updated.
    Lease4StorageAddressIndex::const_iterator lease_it = index.find(lease->addr_);
    if (lease_it == index.end()) {
        isc_throw(NoSuchLease, "failed to update the lease with address "
                  << lease->addr_ << " - no such lease");
    }

    // Try to write a lease to disk first. If this fails, the lease will
    // not be inserted to the memory and the disk and in-memory data will
    // remain consistent.
    if (persistLeases(V4)) {
        lease_file4_->append(*lease);
    }

    // Use replace() to re-index leases.
    index.replace(lease_it, Lease4Ptr(new Lease4(*lease)));
}

void
Memfile_LeaseMgr::updateLease6(const Lease6Ptr& lease) {
    LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL,
              DHCPSRV_MEMFILE_UPDATE_ADDR6).arg(lease->addr_.toText());

    // Obtain 'by address' index.
    Lease6StorageAddressIndex& index = storage6_.get<AddressIndexTag>();

    // Lease must exist if it is to be updated.
    Lease6StorageAddressIndex::const_iterator lease_it = index.find(lease->addr_);
    if (lease_it == index.end()) {
        isc_throw(NoSuchLease, "failed to update the lease with address "
                  << lease->addr_ << " - no such lease");
    }

    // Try to write a lease to disk first. If this fails, the lease will
    // not be inserted to the memory and the disk and in-memory data will
    // remain consistent.
    if (persistLeases(V6)) {
        lease_file6_->append(*lease);
    }

    // Use replace() to re-index leases.
    index.replace(lease_it, Lease6Ptr(new Lease6(*lease)));
}

bool
Memfile_LeaseMgr::deleteLease(const isc::asiolink::IOAddress& addr) {
    LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL,
              DHCPSRV_MEMFILE_DELETE_ADDR).arg(addr.toText());
    if (addr.isV4()) {
        // v4 lease
        Lease4Storage::iterator l = storage4_.find(addr);
        if (l == storage4_.end()) {
            // No such lease
            return (false);
        } else {
            if (persistLeases(V4)) {
                // Copy the lease. The valid lifetime needs to be modified and
                // we don't modify the original lease.
                Lease4 lease_copy = **l;
                // Setting valid lifetime to 0 means that lease is being
                // removed.
                lease_copy.valid_lft_ = 0;
                lease_file4_->append(lease_copy);
            }
            storage4_.erase(l);
            return (true);
        }

    } else {
        // v6 lease
        Lease6Storage::iterator l = storage6_.find(addr);
        if (l == storage6_.end()) {
            // No such lease
            return (false);
        } else {
            if (persistLeases(V6)) {
                // Copy the lease. The lifetimes need to be modified and we
                // don't modify the original lease.
                Lease6 lease_copy = **l;
                // Setting lifetimes to 0 means that lease is being removed.
                lease_copy.valid_lft_ = 0;
                lease_copy.preferred_lft_ = 0;
                lease_file6_->append(lease_copy);
            }

            storage6_.erase(l);
            return (true);
        }
    }
}

uint64_t
Memfile_LeaseMgr::deleteExpiredReclaimedLeases4(const uint32_t secs) {
    LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL,
              DHCPSRV_MEMFILE_DELETE_EXPIRED_RECLAIMED4)
        .arg(secs);
    return (deleteExpiredReclaimedLeases<
            Lease4StorageExpirationIndex, Lease4
            >(secs, V4, storage4_, lease_file4_));
}

uint64_t
Memfile_LeaseMgr::deleteExpiredReclaimedLeases6(const uint32_t secs) {
    LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL,
              DHCPSRV_MEMFILE_DELETE_EXPIRED_RECLAIMED6)
        .arg(secs);
    return (deleteExpiredReclaimedLeases<
            Lease6StorageExpirationIndex, Lease6
            >(secs, V6, storage6_, lease_file6_));
}

template<typename IndexType, typename LeaseType, typename StorageType,
         typename LeaseFileType>
uint64_t
Memfile_LeaseMgr::deleteExpiredReclaimedLeases(const uint32_t secs,
                                               const Universe& universe,
                                               StorageType& storage,
                                               LeaseFileType& lease_file) const {
    // Obtain the index which segragates leases by state and time.
    IndexType& index = storage.template get<ExpirationIndexTag>();

    // This returns the first element which is greater than the specified
    // tuple (true, time(NULL) - secs). However, the range between the
    // beginnng of the index and returned element also includes all the
    // elements for which the first value is false (lease state is NOT
    // reclaimed), because false < true. All elements between the
    // beginning of the index and the element returned, for which the
    // first value is true, represent the reclaimed leases which should
    // be deleted, because their expiration time + secs has occured earlier
    // than current time.
    typename IndexType::const_iterator upper_limit =
        index.upper_bound(boost::make_tuple(true, time(NULL) - secs));

    // Now, we have to exclude all elements of the index which represent
    // leases in the state other than reclaimed - with the first value
    // in the index equal to false. Note that elements in the index are
    // ordered from the lower to the higher ones. So, all elements with
    // the first value of false are placed before the elements with the
    // value of true. Hence, we have to find the first element which
    // contains value of true. The time value is the lowest possible.
    typename IndexType::const_iterator lower_limit =
        index.upper_bound(boost::make_tuple(true, std::numeric_limits<int64_t>::min()));

    // If there are some elements in this range, delete them.
    uint64_t num_leases = static_cast<uint64_t>(std::distance(lower_limit, upper_limit));
    if (num_leases > 0) {

        LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL,
                  DHCPSRV_MEMFILE_DELETE_EXPIRED_RECLAIMED_START)
            .arg(num_leases);

        // If lease persistence is enabled, we also have to mark leases
        // as deleted in the lease file. We do this by setting the
        // lifetime to 0.
        if (persistLeases(universe)) {
            for (typename IndexType::const_iterator lease = lower_limit;
                 lease != upper_limit; ++lease) {
                // Copy lease to not affect the lease in the container.
                LeaseType lease_copy(**lease);
                // Set the valid lifetime to 0 to indicate the removal
                // of the lease.
                lease_copy.valid_lft_ = 0;
                lease_file->append(lease_copy);
            }
        }

        // Erase leases from memory.
        index.erase(lower_limit, upper_limit);
    }
    // Return number of leases deleted.
    return (num_leases);
}


std::string
Memfile_LeaseMgr::getDescription() const {
    return (std::string("In memory database with leases stored in a CSV file."));
}

void
Memfile_LeaseMgr::commit() {
    LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL, DHCPSRV_MEMFILE_COMMIT);
}

void
Memfile_LeaseMgr::rollback() {
    LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL,
              DHCPSRV_MEMFILE_ROLLBACK);
}

std::string
Memfile_LeaseMgr::appendSuffix(const std::string& file_name,
                               const LFCFileType& file_type) {
    std::string name(file_name);
    switch (file_type) {
    case FILE_INPUT:
        name += ".1";
        break;
    case FILE_PREVIOUS:
        name += ".2";
        break;
    case FILE_OUTPUT:
        name += ".output";
        break;
    case FILE_FINISH:
        name += ".completed";
        break;
    case FILE_PID:
        name += ".pid";
        break;
    default:
        // Do not append any suffix for the FILE_CURRENT.
        ;
    }

    return (name);
}

std::string
Memfile_LeaseMgr::getDefaultLeaseFilePath(Universe u) const {
    std::ostringstream s;
    s << CfgMgr::instance().getDataDir() << "/kea-leases";
    s << (u == V4 ? "4" : "6");
    s << ".csv";
    return (s.str());
}

std::string
Memfile_LeaseMgr::getLeaseFilePath(Universe u) const {
    if (u == V4) {
        return (lease_file4_ ? lease_file4_->getFilename() : "");
    }

    return (lease_file6_ ? lease_file6_->getFilename() : "");
}

bool
Memfile_LeaseMgr::persistLeases(Universe u) const {
    // Currently, if the lease file IO is not created, it means that writes to
    // disk have been explicitly disabled by the administrator. At some point,
    // there may be a dedicated ON/OFF flag implemented to control this.
    if (u == V4 && lease_file4_) {
        return (true);
    }

    return (u == V6 && lease_file6_);
}

std::string
Memfile_LeaseMgr::initLeaseFilePath(Universe u) {
    std::string persist_val;
    try {
        persist_val = conn_.getParameter("persist");
    } catch (const Exception&) {
        // If parameter persist hasn't been specified, we use a default value
        // 'yes'.
        persist_val = "true";
    }
    // If persist_val is 'false' we will not store leases to disk, so let's
    // return empty file name.
    if (persist_val == "false") {
        return ("");

    } else if (persist_val != "true") {
        isc_throw(isc::BadValue, "invalid value 'persist="
                  << persist_val << "'");
    }

    std::string lease_file;
    try {
        lease_file = conn_.getParameter("name");
    } catch (const Exception&) {
        lease_file = getDefaultLeaseFilePath(u);
    }
    return (lease_file);
}

template<typename LeaseObjectType, typename LeaseFileType, typename StorageType>
bool Memfile_LeaseMgr::loadLeasesFromFiles(const std::string& filename,
                                           boost::shared_ptr<LeaseFileType>& lease_file,
                                           StorageType& storage) {
    // Check if the instance of the LFC is running right now. If it is
    // running, we refuse to load leases as the LFC may be writing to the
    // lease files right now. When the user retries server configuration
    // it should go through.
    /// @todo Consider applying a timeout for an LFC and retry when this
    /// timeout elapses.
    PIDFile pid_file(appendSuffix(filename, FILE_PID));
    if (pid_file.check()) {
        isc_throw(DbOpenError, "unable to load leases from files while the "
                  "lease file cleanup is in progress");
    }

    storage.clear();

    // Load the leasefile.completed, if exists.
    bool conversion_needed = false;
    lease_file.reset(new LeaseFileType(std::string(filename + ".completed")));
    if (lease_file->exists()) {
        LeaseFileLoader::load<LeaseObjectType>(*lease_file, storage,
                                               MAX_LEASE_ERRORS);
        conversion_needed = conversion_needed || lease_file->needsConversion();
    } else {
        // If the leasefile.completed doesn't exist, let's load the leases
        // from leasefile.2 and leasefile.1, if they exist.
        lease_file.reset(new LeaseFileType(appendSuffix(filename, FILE_PREVIOUS)));
        if (lease_file->exists()) {
            LeaseFileLoader::load<LeaseObjectType>(*lease_file, storage,
                                                   MAX_LEASE_ERRORS);
            conversion_needed =  conversion_needed || lease_file->needsConversion();
        }

        lease_file.reset(new LeaseFileType(appendSuffix(filename, FILE_INPUT)));
        if (lease_file->exists()) {
            LeaseFileLoader::load<LeaseObjectType>(*lease_file, storage,
                                                   MAX_LEASE_ERRORS);
            conversion_needed =  conversion_needed || lease_file->needsConversion();
        }
    }

    // Always load leases from the primary lease file. If the lease file
    // doesn't exist it will be created by the LeaseFileLoader. Note
    // that the false value passed as the last parameter to load
    // function causes the function to leave the file open after
    // it is parsed. This file will be used by the backend to record
    // future lease updates.
    lease_file.reset(new LeaseFileType(filename));
    LeaseFileLoader::load<LeaseObjectType>(*lease_file, storage,
                                           MAX_LEASE_ERRORS, false);
    conversion_needed =  conversion_needed || lease_file->needsConversion();

    return (conversion_needed);
}


bool
Memfile_LeaseMgr::isLFCRunning() const {
    return (lfc_setup_->isRunning());
}

int
Memfile_LeaseMgr::getLFCExitStatus() const {
    return (lfc_setup_->getExitStatus());
}

void
Memfile_LeaseMgr::lfcCallback() {
    LOG_INFO(dhcpsrv_logger, DHCPSRV_MEMFILE_LFC_START);

    // Check if we're in the v4 or v6 space and use the appropriate file.
    if (lease_file4_) {
        lfcExecute(lease_file4_);

    } else if (lease_file6_) {
        lfcExecute(lease_file6_);
    }
}

void
Memfile_LeaseMgr::lfcSetup(bool conversion_needed) {
    std::string lfc_interval_str = "0";
    try {
        lfc_interval_str = conn_.getParameter("lfc-interval");
    } catch (const std::exception&) {
        // Ignore and default to 0.
    }

    uint32_t lfc_interval = 0;
    try {
        lfc_interval = boost::lexical_cast<uint32_t>(lfc_interval_str);
    } catch (boost::bad_lexical_cast&) {
        isc_throw(isc::BadValue, "invalid value of the lfc-interval "
                  << lfc_interval_str << " specified");
    }

    if (lfc_interval > 0 || conversion_needed) {
        lfc_setup_.reset(new LFCSetup(boost::bind(&Memfile_LeaseMgr::lfcCallback, this)));
        lfc_setup_->setup(lfc_interval, lease_file4_, lease_file6_, conversion_needed);
    }
}

template<typename LeaseFileType>
void Memfile_LeaseMgr::lfcExecute(boost::shared_ptr<LeaseFileType>& lease_file) {
    bool do_lfc = true;

    // Check the status of the LFC instance.
    // If the finish file exists or the copy of the lease file exists it
    // is an indication that another LFC instance may be in progress or
    // may be stalled. In that case we don't want to rotate the current
    // lease file to avoid overriding the contents of the existing file.
    CSVFile lease_file_finish(appendSuffix(lease_file->getFilename(), FILE_FINISH));
    CSVFile lease_file_copy(appendSuffix(lease_file->getFilename(), FILE_INPUT));
    if (!lease_file_finish.exists() && !lease_file_copy.exists()) {
        // Close the current file so as we can move it to the copy file.
        lease_file->close();
        // Move the current file to the copy file. Remember the result
        // because we don't want to run LFC if the rename failed.
        do_lfc = (rename(lease_file->getFilename().c_str(),
                         lease_file_copy.getFilename().c_str()) == 0);

        if (!do_lfc) {
            LOG_ERROR(dhcpsrv_logger, DHCPSRV_MEMFILE_LFC_LEASE_FILE_RENAME_FAIL)
                .arg(lease_file->getFilename())
                .arg(lease_file_copy.getFilename())
                .arg(strerror(errno));
        }

        // Regardless if we successfully moved the current file or not,
        // we need to re-open the current file for the server to write
        // new lease updates. If the file has been successfully moved,
        // this will result in creation of the new file. Otherwise,
        // an existing file will be opened.
        try {
            lease_file->open(true);

        } catch (const CSVFileError& ex) {
            // If we're unable to open the lease file this is a serious
            // error because the server will not be able to persist
            // leases.
            /// @todo We need to better address this error. It should
            /// trigger an alarm (once we have a monitoring system in
            /// place) so as an administrator can correct it. In
            /// practice it should be very rare that this happens and
            /// is most likely related to a human error, e.g. changing
            /// file permissions.
            LOG_ERROR(dhcpsrv_logger, DHCPSRV_MEMFILE_LFC_LEASE_FILE_REOPEN_FAIL)
                .arg(lease_file->getFilename())
                .arg(ex.what());
            // Reset the pointer to the file so as the backend doesn't
            // try to write leases to disk.
            lease_file.reset();
            do_lfc = false;
        }
    }
    // Once the files have been rotated, or untouched if another LFC had
    // not finished, a new process is started.
    if (do_lfc) {
        lfc_setup_->execute();
    }
}

AddressStatsQuery4Ptr
Memfile_LeaseMgr::startAddressStatsQuery4() {
    AddressStatsQuery4Ptr query(new MemfileAddressStatsQuery4(storage4_));
    query->start();
    return(query);
}

} // end of namespace isc::dhcp
} // end of namespace isc