[3601] Memfile_LeaseMgr now runs LFC automatically if lease files need upgrading

Memfile_LeaseMgr loads the lease files during instantiation. If it detects
that any of the files loaded are out of date, it will invoke LFC directly,
whether or not LFC is enabled.

src/lib/dhcpsrv/dhcpsrv_messages.mes
    added two new log messages:
        DHCPRSV_MEMFILE_UPGRADING_LEASE_FILES
        DHCPSRV_MEMFILE_NEEDS_UPGRADING

src/lib/dhcpsrv/lease_file_loader.h
    LeaseFileLoader::load() - emits DHCPSRV_MEMFILE_NEEDS_UPGRADING log
    when an out of date file is detected

src/lib/dhcpsrv/memfile_lease_mgr.cc
    LFCSetup::setup() - added run_once_now parameter, which causes the
    method to invoke the LFC callback immediately, regardless of the value
    of LFC interval.
    Memfile_LeaseMgr::Memfile_LeaseMgr() - added logic to track if
    files loaded need upgrading and pass that into lfcSetup()

    Memfile_LeaseMgr::loadLeasesFromFiles() - change to return a boolean
    true if any of the files loaded need upgrading.

    Memfile_LeaseMgr::lfcSetup() - added upgrade_needed parameter, which
    is passed through to LFCSetup::setup() as "run_once_now"

src/lib/dhcpsrv/dhcpsrv_messages.mes

@@ -352,6 +352,13 @@ timer used for lease file cleanup scheduling. This is highly unlikely
 and indicates programming error. The message include the reason for this
 error.
 
+% DHCPSRV_MEMFILE_NEEDS_UPGRADING Lease file: %1 is schema version %2, it needs to be upgraded to current schema version, %3.
+A warning message issued when the schema of the lease file loaded by the server
+is pre-dates the current Memfile schema.  Note that the server converts the lease
+data from older schemas to the current schema as it is read, therefore the lease
+information in use by the server will be correct.  What remains is for the file
+itself to be rewritten using the current schema.
+
 % DHCPSRV_MEMFILE_NO_STORAGE running in non-persistent mode, leases will be lost after restart
 A warning message issued when writes of leases to disk have been disabled
 in the configuration. This mode is useful for some kinds of performance
@@ -377,6 +384,12 @@ lease from the memory file database for the specified address.
 A debug message issued when the server is attempting to update IPv6
 lease from the memory file database for the specified address.
 
+% DHCPRSV_MEMFILE_UPGRADING_LEASE_FILES Running LFC now, to upgrade lease files to current schema: %1.%2
+A warning message when the server has detected lease files that need to be upgraded,
+and is automatically running the LFC process to perform the upgrade.  This should
+only occur the first time the server is launched following a Kea upgrade in which
+the Memfile schema was updated.
+
 % DHCPSRV_MULTIPLE_RAW_SOCKETS_PER_IFACE current configuration will result in opening multiple brodcast capable sockets on some interfaces and some DHCP messages may be duplicated
 A warning message issued when the current configuration indicates that multiple
 sockets, capable of receiving brodcast traffic, will be opened on some of the

src/lib/dhcpsrv/lease_file_loader.h

@@ -154,6 +154,13 @@ public:
             }
         }
 
+        if (lease_file.needsUpgrading()) {
+            LOG_WARN(dhcpsrv_logger, DHCPSRV_MEMFILE_NEEDS_UPGRADING)
+                     .arg(lease_file.getFilename())
+                     .arg(lease_file.getInputSchemaVersion())
+                     .arg(lease_file.getSchemaVersion());
+        }
+
         if (close_file_on_exit) {
             lease_file.close();
         }

src/lib/dhcpsrv/memfile_lease_mgr.cc

@@ -90,9 +90,13 @@ public:
     /// or NULL. If this is NULL, the @c lease_file6 must be non-null.
     /// @param lease_file6 A pointer to the DHCPv6 lease file to be cleaned up
     /// or NULL. If this is NULL, the @c lease_file4 must be non-null.
+    /// @param run_once_now A flag that causes LFC to be invoked immediately,
+    /// regardless of the value of lfc_interval.  This is primarily used to
+    /// cause lease file schema upgrades upon startup.
     void setup(const uint32_t lfc_interval,
                const boost::shared_ptr<CSVLeaseFile4>& lease_file4,
-               const boost::shared_ptr<CSVLeaseFile6>& lease_file6);
+               const boost::shared_ptr<CSVLeaseFile6>& lease_file6,
+               bool run_once_now = false);
 
     /// @brief Spawns a new process.
     void execute();
@@ -155,58 +159,67 @@ LFCSetup::~LFCSetup() {
 void
 LFCSetup::setup(const uint32_t lfc_interval,
                 const boost::shared_ptr<CSVLeaseFile4>& lease_file4,
-                const boost::shared_ptr<CSVLeaseFile6>& lease_file6) {
+                const boost::shared_ptr<CSVLeaseFile6>& lease_file6,
+                bool run_once_now) {
 
-    // If LFC is enabled, we have to setup the interval timer and prepare for
-    // executing the kea-lfc process.
-    if (lfc_interval > 0) {
-        std::string executable;
-        char* c_executable = getenv(KEA_LFC_EXECUTABLE_ENV_NAME);
-        if (c_executable == NULL) {
-            executable = KEA_LFC_EXECUTABLE;
+    // If to nothing to do, punt
+    if (lfc_interval == 0 && run_once_now == false) {
+        return;
+    }
 
-        } else {
-            executable = c_executable;
-        }
+    // Start preparing the command line for kea-lfc.
+    std::string executable;
+    char* c_executable = getenv(KEA_LFC_EXECUTABLE_ENV_NAME);
+    if (c_executable == NULL) {
+        executable = KEA_LFC_EXECUTABLE;
+    } else {
+        executable = c_executable;
+    }
 
-        // Start preparing the command line for kea-lfc.
-
-        // Gather the base file name.
-        std::string lease_file = lease_file4 ? lease_file4->getFilename() :
-            lease_file6->getFilename();
-
-        // Create the other names by appending suffixes to the base name.
-        util::ProcessArgs args;
-        // Universe: v4 or v6.
-        args.push_back(lease_file4 ? "-4" : "-6");
-        // Previous file.
-        args.push_back("-x");
-        args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
-                                                      Memfile_LeaseMgr::FILE_PREVIOUS));
-        // Input file.
-        args.push_back("-i");
-        args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
-                                                      Memfile_LeaseMgr::FILE_INPUT));
-        // Output file.
-        args.push_back("-o");
-        args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
-                                                      Memfile_LeaseMgr::FILE_OUTPUT));
-        // Finish file.
-        args.push_back("-f");
-        args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
-                                                      Memfile_LeaseMgr::FILE_FINISH));
-        // PID file.
-        args.push_back("-p");
-        args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
-                                                      Memfile_LeaseMgr::FILE_PID));
-
-        // The configuration file is currently unused.
-        args.push_back("-c");
-        args.push_back("ignored-path");
-
-        // Create the process (do not start it yet).
-        process_.reset(new util::ProcessSpawn(executable, args));
+    // Gather the base file name.
+    std::string lease_file = lease_file4 ? lease_file4->getFilename() :
+                                           lease_file6->getFilename();
+
+    // Create the other names by appending suffixes to the base name.
+    util::ProcessArgs args;
+    // Universe: v4 or v6.
+    args.push_back(lease_file4 ? "-4" : "-6");
+
+    // Previous file.
+    args.push_back("-x");
+    args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
+                                                  Memfile_LeaseMgr::FILE_PREVIOUS));
+    // Input file.
+    args.push_back("-i");
+    args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
+                                                  Memfile_LeaseMgr::FILE_INPUT));
+    // Output file.
+    args.push_back("-o");
+    args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
+                                                  Memfile_LeaseMgr::FILE_OUTPUT));
+    // Finish file.
+    args.push_back("-f");
+    args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
+                                                  Memfile_LeaseMgr::FILE_FINISH));
+    // PID file.
+    args.push_back("-p");
+    args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
+                                                  Memfile_LeaseMgr::FILE_PID));
+
+    // The configuration file is currently unused.
+    args.push_back("-c");
+    args.push_back("ignored-path");
+
+    // Create the process (do not start it yet).
+    process_.reset(new util::ProcessSpawn(executable, args));
+
+    // If we've been told to run it once now, invoke the callback directly.
+    if (run_once_now) {
+        callback_();
+    }
 
+    // If it's suposed to run periodically, setup that now.
+    if (lfc_interval > 0) {
         // Set the timer to call callback function periodically.
         LOG_INFO(dhcpsrv_logger, DHCPSRV_MEMFILE_LFC_SETUP).arg(lfc_interval);
 
@@ -253,19 +266,25 @@ const int Memfile_LeaseMgr::MINOR_VERSION;
 Memfile_LeaseMgr::Memfile_LeaseMgr(const DatabaseConnection::ParameterMap& parameters)
     : LeaseMgr(), lfc_setup_(), conn_(parameters)
     {
+    bool upgrade_needed = false;
+
     // Check the universe and use v4 file or v6 file.
     std::string universe = conn_.getParameter("universe");
     if (universe == "4") {
         std::string file4 = initLeaseFilePath(V4);
         if (!file4.empty()) {
-            loadLeasesFromFiles<Lease4, CSVLeaseFile4>(file4, lease_file4_,
-                                                       storage4_);
+            upgrade_needed = loadLeasesFromFiles<Lease4,
+                                                 CSVLeaseFile4>(file4,
+                                                                lease_file4_,
+                                                                storage4_);
         }
     } else {
         std::string file6 = initLeaseFilePath(V6);
         if (!file6.empty()) {
-            loadLeasesFromFiles<Lease6, CSVLeaseFile6>(file6, lease_file6_,
-                                                       storage6_);
+            upgrade_needed = loadLeasesFromFiles<Lease6,
+                                                 CSVLeaseFile6>(file6,
+                                                                lease_file6_,
+                                                                storage6_);
         }
     }
 
@@ -275,9 +294,12 @@ Memfile_LeaseMgr::Memfile_LeaseMgr(const DatabaseConnection::ParameterMap& param
     // operation.
    if (!persistLeases(V4) && !persistLeases(V6)) {
         LOG_WARN(dhcpsrv_logger, DHCPSRV_MEMFILE_NO_STORAGE);
-
     } else  {
-       lfcSetup();
+        if (upgrade_needed) {
+            LOG_WARN(dhcpsrv_logger, DHCPRSV_MEMFILE_UPGRADING_LEASE_FILES)
+                    .arg(MAJOR_VERSION).arg(MINOR_VERSION);
+        }
+        lfcSetup(upgrade_needed);
     }
 }
 
@@ -867,7 +889,7 @@ Memfile_LeaseMgr::initLeaseFilePath(Universe u) {
 }
 
 template<typename LeaseObjectType, typename LeaseFileType, typename StorageType>
-void Memfile_LeaseMgr::loadLeasesFromFiles(const std::string& filename,
+bool Memfile_LeaseMgr::loadLeasesFromFiles(const std::string& filename,
                                            boost::shared_ptr<LeaseFileType>& lease_file,
                                            StorageType& storage) {
     // Check if the instance of the LFC is running right now. If it is
@@ -885,11 +907,12 @@ void Memfile_LeaseMgr::loadLeasesFromFiles(const std::string& filename,
     storage.clear();
 
     // Load the leasefile.completed, if exists.
+    bool upgrade_needed = false;
     lease_file.reset(new LeaseFileType(std::string(filename + ".completed")));
     if (lease_file->exists()) {
         LeaseFileLoader::load<LeaseObjectType>(*lease_file, storage,
                                                MAX_LEASE_ERRORS);
-
+        upgrade_needed |= lease_file->needsUpgrading();
     } else {
         // If the leasefile.completed doesn't exist, let's load the leases
         // from leasefile.2 and leasefile.1, if they exist.
@@ -897,12 +920,14 @@ void Memfile_LeaseMgr::loadLeasesFromFiles(const std::string& filename,
         if (lease_file->exists()) {
             LeaseFileLoader::load<LeaseObjectType>(*lease_file, storage,
                                                    MAX_LEASE_ERRORS);
+            upgrade_needed |= lease_file->needsUpgrading();
         }
 
         lease_file.reset(new LeaseFileType(appendSuffix(filename, FILE_INPUT)));
         if (lease_file->exists()) {
             LeaseFileLoader::load<LeaseObjectType>(*lease_file, storage,
                                                    MAX_LEASE_ERRORS);
+            upgrade_needed |= lease_file->needsUpgrading();
         }
     }
 
@@ -915,6 +940,9 @@ void Memfile_LeaseMgr::loadLeasesFromFiles(const std::string& filename,
     lease_file.reset(new LeaseFileType(filename));
     LeaseFileLoader::load<LeaseObjectType>(*lease_file, storage,
                                            MAX_LEASE_ERRORS, false);
+    upgrade_needed |= lease_file->needsUpgrading();
+
+    return (upgrade_needed);
 }
 
 
@@ -942,7 +970,7 @@ Memfile_LeaseMgr::lfcCallback() {
 }
 
 void
-Memfile_LeaseMgr::lfcSetup() {
+Memfile_LeaseMgr::lfcSetup(bool upgrade_needed) {
     std::string lfc_interval_str = "0";
     try {
         lfc_interval_str = conn_.getParameter("lfc-interval");
@@ -958,9 +986,9 @@ Memfile_LeaseMgr::lfcSetup() {
                   << lfc_interval_str << " specified");
     }
 
-    if (lfc_interval > 0) {
+    if (lfc_interval > 0 || upgrade_needed) {
         lfc_setup_.reset(new LFCSetup(boost::bind(&Memfile_LeaseMgr::lfcCallback, this)));
-        lfc_setup_->setup(lfc_interval, lease_file4_, lease_file6_);
+        lfc_setup_->setup(lfc_interval, lease_file4_, lease_file6_, upgrade_needed);
     }
 }
 

src/lib/dhcpsrv/memfile_lease_mgr.h

@@ -553,7 +553,7 @@ private:
     /// @throw DbOpenError when it is found that the LFC is in progress.
     template<typename LeaseObjectType, typename LeaseFileType,
              typename StorageType>
-    void loadLeasesFromFiles(const std::string& filename,
+    bool loadLeasesFromFiles(const std::string& filename,
                              boost::shared_ptr<LeaseFileType>& lease_file,
                              StorageType& storage);
 
@@ -626,7 +626,10 @@ private:
     /// Kea build directory, the @c KEA_LFC_EXECUTABLE environmental
     /// variable should be set to hold an absolute path to the kea-lfc
     /// excutable.
-    void lfcSetup();
+    /// @param upgrade_needed flag that indicates input lease file(s) are
+    /// from an earlier schema version and need conversion.  This value is
+    /// passed through to LFCSetup::setup() via its run_once_now parameter.
+    void lfcSetup(bool upgrade_needed = false);
 
     /// @brief Performs a lease file cleanup for DHCPv4 or DHCPv6.
     ///