[master] MemfileLeaseMgr now supports upgrading/downgrading lease files

    Merged in branch 'trac3601'

doc/guide/admin.xml

@@ -150,6 +150,27 @@
         will create an empty lease file if one is not
         present. Necessary disk write permission is required.
       </para>
+      <section id="memfile-upgrade">
+        <title>Upgrading Memfile Lease Files from an Earlier Version of Kea</title>
+        <para>
+        There are no special steps required to upgrade memfile lease files
+        from an earlier version of Kea to a new version of Kea.
+
+        During startup the servers will check the schema version of the lease
+        files against their own.  If there is a mismatch, the servers will
+        automatically launch the LFC process to convert the files to the
+        server's schema version.  While this mechanism is primarily meant to
+        ease the process of upgrading to newer versions of Kea, it can also
+        be used for downgrading should the need arise.  When upgrading, any
+        values not present in the original lease files will be assigned
+        appropriate default values.  When downgrading, any data present in
+        the files but not in the server's schema will be dropped.
+
+        If you wish to convert the files manually, prior to starting the
+        servers you may do so by running the LFC process yourself.
+        See <xref linkend="kea-lfc"/> for more information.
+        </para>
+      </section>
       <!-- @todo: document lease file upgrades once they are implemented in kea-admin -->
     </section>
 

src/lib/dhcpsrv/csv_lease_file4.cc

@@ -22,14 +22,14 @@ namespace isc {
 namespace dhcp {
 
 CSVLeaseFile4::CSVLeaseFile4(const std::string& filename)
-    : CSVFile(filename) {
+    : VersionedCSVFile(filename) {
     initColumns();
 }
 
 void
 CSVLeaseFile4::open(const bool seek_to_end) {
     // Call the base class to open the file
-    CSVFile::open(seek_to_end);
+    VersionedCSVFile::open(seek_to_end);
 
     // and clear any statistics we may have
     clearStatistics();
@@ -62,7 +62,7 @@ CSVLeaseFile4::append(const Lease4& lease) {
     row.writeAt(getColumnIndex("state"), lease.state_);
 
     try {
-        CSVFile::append(row);
+        VersionedCSVFile::append(row);
     } catch (const std::exception&) {
         // Catch any errors so we can bump the error counter than rethrow it
         ++write_errs_;
@@ -85,7 +85,7 @@ CSVLeaseFile4::next(Lease4Ptr& lease) {
     try {
         // Get the row of CSV values.
         CSVRow row;
-        CSVFile::next(row);
+        VersionedCSVFile::next(row);
         // The empty row signals EOF.
         if (row == CSVFile::EMPTY_ROW()) {
             lease.reset();
@@ -137,16 +137,18 @@ CSVLeaseFile4::next(Lease4Ptr& lease) {
 
 void
 CSVLeaseFile4::initColumns() {
-    addColumn("address");
-    addColumn("hwaddr");
-    addColumn("client_id");
-    addColumn("valid_lifetime");
-    addColumn("expire");
-    addColumn("subnet_id");
-    addColumn("fqdn_fwd");
-    addColumn("fqdn_rev");
-    addColumn("hostname");
-    addColumn("state");
+    addColumn("address", "1.0");
+    addColumn("hwaddr", "1.0");
+    addColumn("client_id", "1.0");
+    addColumn("valid_lifetime", "1.0");
+    addColumn("expire", "1.0");
+    addColumn("subnet_id", "1.0");
+    addColumn("fqdn_fwd", "1.0");
+    addColumn("fqdn_rev", "1.0");
+    addColumn("hostname", "1.0");
+    addColumn("state", "2.0", "0");
+    // Any file with less than hostname is invalid
+    setMinimumValidColumns("hostname");
 }
 
 IOAddress

src/lib/dhcpsrv/csv_lease_file4.h

@@ -20,7 +20,7 @@
 #include <dhcpsrv/lease.h>
 #include <dhcpsrv/subnet.h>
 #include <dhcpsrv/lease_file_stats.h>
-#include <util/csv_file.h>
+#include <util/versioned_csv_file.h>
 #include <stdint.h>
 #include <string>
 #include <time.h>
@@ -39,7 +39,7 @@ namespace dhcp {
 /// validation (see http://kea.isc.org/ticket/2405). However, when #2405
 /// is implemented, the @c next function may need to be updated to use the
 /// validation capablity of @c Lease4.
-class CSVLeaseFile4 : public isc::util::CSVFile, public LeaseFileStats {
+class CSVLeaseFile4 : public isc::util::VersionedCSVFile, public LeaseFileStats {
 public:
 
     /// @brief Constructor.

src/lib/dhcpsrv/csv_lease_file6.cc

@@ -23,14 +23,14 @@ namespace isc {
 namespace dhcp {
 
 CSVLeaseFile6::CSVLeaseFile6(const std::string& filename)
-    : CSVFile(filename) {
+    : VersionedCSVFile(filename) {
     initColumns();
 }
 
 void
 CSVLeaseFile6::open(const bool seek_to_end) {
     // Call the base class to open the file
-    CSVFile::open(seek_to_end);
+    VersionedCSVFile::open(seek_to_end);
 
     // and clear any statistics we may have
     clearStatistics();
@@ -61,7 +61,7 @@ CSVLeaseFile6::append(const Lease6& lease) {
     }
     row.writeAt(getColumnIndex("state"), lease.state_);
     try {
-        CSVFile::append(row);
+        VersionedCSVFile::append(row);
     } catch (const std::exception&) {
         // Catch any errors so we can bump the error counter than rethrow it
         ++write_errs_;
@@ -84,7 +84,7 @@ CSVLeaseFile6::next(Lease6Ptr& lease) {
     try {
         // Get the row of CSV values.
         CSVRow row;
-        CSVFile::next(row);
+        VersionedCSVFile::next(row);
         // The empty row signals EOF.
         if (row == CSVFile::EMPTY_ROW()) {
             lease.reset();
@@ -122,20 +122,23 @@ CSVLeaseFile6::next(Lease6Ptr& lease) {
 
 void
 CSVLeaseFile6::initColumns() {
-    addColumn("address");
-    addColumn("duid");
-    addColumn("valid_lifetime");
-    addColumn("expire");
-    addColumn("subnet_id");
-    addColumn("pref_lifetime");
-    addColumn("lease_type");
-    addColumn("iaid");
-    addColumn("prefix_len");
-    addColumn("fqdn_fwd");
-    addColumn("fqdn_rev");
-    addColumn("hostname");
-    addColumn("hwaddr");
-    addColumn("state");
+    addColumn("address", "1.0");
+    addColumn("duid", "1.0");
+    addColumn("valid_lifetime", "1.0");
+    addColumn("expire", "1.0");
+    addColumn("subnet_id", "1.0");
+    addColumn("pref_lifetime", "1.0");
+    addColumn("lease_type", "1.0");
+    addColumn("iaid", "1.0");
+    addColumn("prefix_len", "1.0");
+    addColumn("fqdn_fwd", "1.0");
+    addColumn("fqdn_rev", "1.0");
+    addColumn("hostname", "1.0");
+    addColumn("hwaddr", "2.0");
+    addColumn("state", "3.0", "0");
+
+    // Any file with less than hostname is invalid
+    setMinimumValidColumns("hostname");
 }
 
 Lease::Type

src/lib/dhcpsrv/csv_lease_file6.h

@@ -20,7 +20,7 @@
 #include <dhcpsrv/lease.h>
 #include <dhcpsrv/subnet.h>
 #include <dhcpsrv/lease_file_stats.h>
-#include <util/csv_file.h>
+#include <util/versioned_csv_file.h>
 #include <stdint.h>
 #include <string>
 
@@ -38,7 +38,7 @@ namespace dhcp {
 /// validation (see http://kea.isc.org/ticket/2405). However, when #2405
 /// is implemented, the @c next function may need to be updated to use the
 /// validation capablity of @c Lease6.
-class CSVLeaseFile6 : public isc::util::CSVFile, public LeaseFileStats {
+class CSVLeaseFile6 : public isc::util::VersionedCSVFile, public LeaseFileStats {
 public:
 
     /// @brief Constructor.

src/lib/dhcpsrv/dhcpsrv_messages.mes

@@ -220,6 +220,13 @@ with the specified address to the memory file backend database.
 The code has issued a commit call.  For the memory file database, this is
 a no-op.
 
+% DHCPRSV_MEMFILE_CONVERTING_LEASE_FILES running LFC now to convert lease files to the current schema: %1.%2
+A warning message issued when the server has detected lease files that need
+to be either upgraded or downgraded to match the server's schema, and that
+the server is automatically running the LFC process to perform the conversion.
+This should only occur the first time the server is launched following a Kea
+installation upgrade (or downgrade).
+
 % DHCPSRV_MEMFILE_DB opening memory file lease database: %1
 This informational message is logged when a DHCP server (either V4 or
 V6) is about to open a memory file lease database.  The parameters of
@@ -352,6 +359,21 @@ timer used for lease file cleanup scheduling. This is highly unlikely
 and indicates programming error. The message include the reason for this
 error.
 
+% DHCPSRV_MEMFILE_NEEDS_DOWNGRADING version of lease file: %1 schema is later than version %2.
+A warning message issued when the schema of the lease file loaded by the server
+is newer than the memfile schema of the server.  The server converts the lease
+data from newer schemas to its schema as it is read, therefore the lease
+information in use by the server will be correct. Note though, that any data
+data stored in newer schema fields will be dropped.  What remains is for the
+file itself to be rewritten using the current schema.
+
+% DHCPSRV_MEMFILE_NEEDS_UPGRADING version of lease file: %1 schema is earlier than version %2.
+A warning message issued when the schema of the lease file loaded by the server
+pre-dates the memfile schema of the server.  Note that the server converts the
+lease data from older schemas to the current schema as it is read, therefore
+the lease information in use by the server will be correct.  What remains is
+for the file itself to be rewritten using the current schema.
+
 % DHCPSRV_MEMFILE_NO_STORAGE running in non-persistent mode, leases will be lost after restart
 A warning message issued when writes of leases to disk have been disabled
 in the configuration. This mode is useful for some kinds of performance

src/lib/dhcpsrv/lease_file_loader.h

@@ -17,7 +17,7 @@
 
 #include <dhcpsrv/dhcpsrv_log.h>
 #include <dhcpsrv/memfile_lease_storage.h>
-#include <util/csv_file.h>
+#include <util/versioned_csv_file.h>
 
 #include <boost/shared_ptr.hpp>
 
@@ -154,6 +154,16 @@ public:
             }
         }
 
+        if (lease_file.needsConversion()) {
+            LOG_WARN(dhcpsrv_logger,
+                     (lease_file.getInputSchemaState()
+                      == util::VersionedCSVFile::NEEDS_UPGRADE
+                      ?  DHCPSRV_MEMFILE_NEEDS_UPGRADING
+                      : DHCPSRV_MEMFILE_NEEDS_DOWNGRADING))
+                     .arg(lease_file.getFilename())
+                     .arg(lease_file.getSchemaVersion());
+        }
+
         if (close_file_on_exit) {
             lease_file.close();
         }

src/lib/dhcpsrv/memfile_lease_mgr.cc

@@ -90,9 +90,13 @@ public:
     /// or NULL. If this is NULL, the @c lease_file6 must be non-null.
     /// @param lease_file6 A pointer to the DHCPv6 lease file to be cleaned up
     /// or NULL. If this is NULL, the @c lease_file4 must be non-null.
+    /// @param run_once_now A flag that causes LFC to be invoked immediately,
+    /// regardless of the value of lfc_interval.  This is primarily used to
+    /// cause lease file schema upgrades upon startup.
     void setup(const uint32_t lfc_interval,
                const boost::shared_ptr<CSVLeaseFile4>& lease_file4,
-               const boost::shared_ptr<CSVLeaseFile6>& lease_file6);
+               const boost::shared_ptr<CSVLeaseFile6>& lease_file6,
+               bool run_once_now = false);
 
     /// @brief Spawns a new process.
     void execute();
@@ -155,58 +159,67 @@ LFCSetup::~LFCSetup() {
 void
 LFCSetup::setup(const uint32_t lfc_interval,
                 const boost::shared_ptr<CSVLeaseFile4>& lease_file4,
-                const boost::shared_ptr<CSVLeaseFile6>& lease_file6) {
+                const boost::shared_ptr<CSVLeaseFile6>& lease_file6,
+                bool run_once_now) {
 
-    // If LFC is enabled, we have to setup the interval timer and prepare for
-    // executing the kea-lfc process.
-    if (lfc_interval > 0) {
-        std::string executable;
-        char* c_executable = getenv(KEA_LFC_EXECUTABLE_ENV_NAME);
-        if (c_executable == NULL) {
-            executable = KEA_LFC_EXECUTABLE;
+    // If to nothing to do, punt
+    if (lfc_interval == 0 && !run_once_now) {
+        return;
+    }
 
-        } else {
-            executable = c_executable;
-        }
+    // Start preparing the command line for kea-lfc.
+    std::string executable;
+    char* c_executable = getenv(KEA_LFC_EXECUTABLE_ENV_NAME);
+    if (c_executable == NULL) {
+        executable = KEA_LFC_EXECUTABLE;
+    } else {
+        executable = c_executable;
+    }
 
-        // Start preparing the command line for kea-lfc.
-
-        // Gather the base file name.
-        std::string lease_file = lease_file4 ? lease_file4->getFilename() :
-            lease_file6->getFilename();
-
-        // Create the other names by appending suffixes to the base name.
-        util::ProcessArgs args;
-        // Universe: v4 or v6.
-        args.push_back(lease_file4 ? "-4" : "-6");
-        // Previous file.
-        args.push_back("-x");
-        args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
-                                                      Memfile_LeaseMgr::FILE_PREVIOUS));
-        // Input file.
-        args.push_back("-i");
-        args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
-                                                      Memfile_LeaseMgr::FILE_INPUT));
-        // Output file.
-        args.push_back("-o");
-        args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
-                                                      Memfile_LeaseMgr::FILE_OUTPUT));
-        // Finish file.
-        args.push_back("-f");
-        args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
-                                                      Memfile_LeaseMgr::FILE_FINISH));
-        // PID file.
-        args.push_back("-p");
-        args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
-                                                      Memfile_LeaseMgr::FILE_PID));
-
-        // The configuration file is currently unused.
-        args.push_back("-c");
-        args.push_back("ignored-path");
-
-        // Create the process (do not start it yet).
-        process_.reset(new util::ProcessSpawn(executable, args));
+    // Gather the base file name.
+    std::string lease_file = lease_file4 ? lease_file4->getFilename() :
+                                           lease_file6->getFilename();
+
+    // Create the other names by appending suffixes to the base name.
+    util::ProcessArgs args;
+    // Universe: v4 or v6.
+    args.push_back(lease_file4 ? "-4" : "-6");
+
+    // Previous file.
+    args.push_back("-x");
+    args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
+                                                  Memfile_LeaseMgr::FILE_PREVIOUS));
+    // Input file.
+    args.push_back("-i");
+    args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
+                                                  Memfile_LeaseMgr::FILE_INPUT));
+    // Output file.
+    args.push_back("-o");
+    args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
+                                                  Memfile_LeaseMgr::FILE_OUTPUT));
+    // Finish file.
+    args.push_back("-f");
+    args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
+                                                  Memfile_LeaseMgr::FILE_FINISH));
+    // PID file.
+    args.push_back("-p");
+    args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
+                                                  Memfile_LeaseMgr::FILE_PID));
+
+    // The configuration file is currently unused.
+    args.push_back("-c");
+    args.push_back("ignored-path");
+
+    // Create the process (do not start it yet).
+    process_.reset(new util::ProcessSpawn(executable, args));
+
+    // If we've been told to run it once now, invoke the callback directly.
+    if (run_once_now) {
+        callback_();
+    }
 
+    // If it's suposed to run periodically, setup that now.
+    if (lfc_interval > 0) {
         // Set the timer to call callback function periodically.
         LOG_INFO(dhcpsrv_logger, DHCPSRV_MEMFILE_LFC_SETUP).arg(lfc_interval);
 
@@ -253,19 +266,25 @@ const int Memfile_LeaseMgr::MINOR_VERSION;
 Memfile_LeaseMgr::Memfile_LeaseMgr(const DatabaseConnection::ParameterMap& parameters)
     : LeaseMgr(), lfc_setup_(), conn_(parameters)
     {
+    bool conversion_needed = false;
+
     // Check the universe and use v4 file or v6 file.
     std::string universe = conn_.getParameter("universe");
     if (universe == "4") {
         std::string file4 = initLeaseFilePath(V4);
         if (!file4.empty()) {
-            loadLeasesFromFiles<Lease4, CSVLeaseFile4>(file4, lease_file4_,
-                                                       storage4_);
+            conversion_needed = loadLeasesFromFiles<Lease4,
+                                                 CSVLeaseFile4>(file4,
+                                                                lease_file4_,
+                                                                storage4_);
         }
     } else {
         std::string file6 = initLeaseFilePath(V6);
         if (!file6.empty()) {
-            loadLeasesFromFiles<Lease6, CSVLeaseFile6>(file6, lease_file6_,
-                                                       storage6_);
+            conversion_needed = loadLeasesFromFiles<Lease6,
+                                                 CSVLeaseFile6>(file6,
+                                                                lease_file6_,
+                                                                storage6_);
         }
     }
 
@@ -275,9 +294,12 @@ Memfile_LeaseMgr::Memfile_LeaseMgr(const DatabaseConnection::ParameterMap& param
     // operation.
    if (!persistLeases(V4) && !persistLeases(V6)) {
         LOG_WARN(dhcpsrv_logger, DHCPSRV_MEMFILE_NO_STORAGE);
-
     } else  {
-       lfcSetup();
+        if (conversion_needed) {
+            LOG_WARN(dhcpsrv_logger, DHCPRSV_MEMFILE_CONVERTING_LEASE_FILES)
+                    .arg(MAJOR_VERSION).arg(MINOR_VERSION);
+        }
+        lfcSetup(conversion_needed);
     }
 }
 
@@ -867,7 +889,7 @@ Memfile_LeaseMgr::initLeaseFilePath(Universe u) {
 }
 
 template<typename LeaseObjectType, typename LeaseFileType, typename StorageType>
-void Memfile_LeaseMgr::loadLeasesFromFiles(const std::string& filename,
+bool Memfile_LeaseMgr::loadLeasesFromFiles(const std::string& filename,
                                            boost::shared_ptr<LeaseFileType>& lease_file,
                                            StorageType& storage) {
     // Check if the instance of the LFC is running right now. If it is
@@ -885,11 +907,12 @@ void Memfile_LeaseMgr::loadLeasesFromFiles(const std::string& filename,
     storage.clear();
 
     // Load the leasefile.completed, if exists.
+    bool conversion_needed = false;
     lease_file.reset(new LeaseFileType(std::string(filename + ".completed")));
     if (lease_file->exists()) {
         LeaseFileLoader::load<LeaseObjectType>(*lease_file, storage,
                                                MAX_LEASE_ERRORS);
-
+        conversion_needed = conversion_needed || lease_file->needsConversion();
     } else {
         // If the leasefile.completed doesn't exist, let's load the leases
         // from leasefile.2 and leasefile.1, if they exist.
@@ -897,12 +920,14 @@ void Memfile_LeaseMgr::loadLeasesFromFiles(const std::string& filename,
         if (lease_file->exists()) {
             LeaseFileLoader::load<LeaseObjectType>(*lease_file, storage,
                                                    MAX_LEASE_ERRORS);
+            conversion_needed =  conversion_needed || lease_file->needsConversion();
         }
 
         lease_file.reset(new LeaseFileType(appendSuffix(filename, FILE_INPUT)));
         if (lease_file->exists()) {
             LeaseFileLoader::load<LeaseObjectType>(*lease_file, storage,
                                                    MAX_LEASE_ERRORS);
+            conversion_needed =  conversion_needed || lease_file->needsConversion();
         }
     }
 
@@ -915,6 +940,9 @@ void Memfile_LeaseMgr::loadLeasesFromFiles(const std::string& filename,
     lease_file.reset(new LeaseFileType(filename));
     LeaseFileLoader::load<LeaseObjectType>(*lease_file, storage,
                                            MAX_LEASE_ERRORS, false);
+    conversion_needed =  conversion_needed || lease_file->needsConversion();
+
+    return (conversion_needed);
 }
 
 
@@ -942,7 +970,7 @@ Memfile_LeaseMgr::lfcCallback() {
 }
 
 void
-Memfile_LeaseMgr::lfcSetup() {
+Memfile_LeaseMgr::lfcSetup(bool conversion_needed) {
     std::string lfc_interval_str = "0";
     try {
         lfc_interval_str = conn_.getParameter("lfc-interval");
@@ -958,9 +986,9 @@ Memfile_LeaseMgr::lfcSetup() {
                   << lfc_interval_str << " specified");
     }
 
-    if (lfc_interval > 0) {
+    if (lfc_interval > 0 || conversion_needed) {
         lfc_setup_.reset(new LFCSetup(boost::bind(&Memfile_LeaseMgr::lfcCallback, this)));
-        lfc_setup_->setup(lfc_interval, lease_file4_, lease_file6_);
+        lfc_setup_->setup(lfc_interval, lease_file4_, lease_file6_, conversion_needed);
     }
 }
 

src/lib/dhcpsrv/memfile_lease_mgr.h

@@ -118,6 +118,18 @@ public:
 
     /// @brief The sole lease manager constructor
     ///
+    /// This method:
+    /// - Initializes the new instance based on the parameters given
+    /// - Loads (or creates) the appropriate lease file(s)
+    /// - Initiates the periodic scheduling of the LFC (if enabled)
+    ///
+    /// If any of the files loaded require conversion to the current schema
+    /// (upgrade or downgrade), @c lfcSetup() will be invoked with its
+    /// @c run_once_now parameter set to true.  This causes lfcSetup() to
+    /// invoke the LFC process immediately regardless of whether LFC is
+    /// enabled. This ensures that any files which need conversion are
+    /// converted automatically.
+    ///
     /// dbconfig is a generic way of passing parameters. Parameters
     /// are passed in the "name=value" format, separated by spaces.
     /// Values may be enclosed in double quotes, if needed.
@@ -549,11 +561,14 @@ private:
     /// @tparam LeaseFileType @c CSVLeaseFile4 or @c CSVLeaseFile6.
     /// @tparam StorageType @c Lease4Storage or @c Lease6Storage.
     ///
+    /// @return Returns true if any of the files loaded need conversion from
+    /// an older or newer schema.
+    ///
     /// @throw CSVFileError when parsing any of the lease files fails.
     /// @throw DbOpenError when it is found that the LFC is in progress.
     template<typename LeaseObjectType, typename LeaseFileType,
              typename StorageType>
-    void loadLeasesFromFiles(const std::string& filename,
+    bool loadLeasesFromFiles(const std::string& filename,
                              boost::shared_ptr<LeaseFileType>& lease_file,
                              StorageType& storage);
 
@@ -626,7 +641,11 @@ private:
     /// Kea build directory, the @c KEA_LFC_EXECUTABLE environmental
     /// variable should be set to hold an absolute path to the kea-lfc
     /// excutable.
-    void lfcSetup();
+    /// @param conversion_needed flag that indicates input lease file(s) are
+    /// schema do not match the current schema (older or newer), and need
+    /// conversion. This value is passed through to LFCSetup::setup() via its
+    /// run_once_now parameter.
+    void lfcSetup(bool conversion_needed = false);
 
     /// @brief Performs a lease file cleanup for DHCPv4 or DHCPv6.
     ///

src/lib/dhcpsrv/tests/csv_lease_file4_unittest.cc

@@ -18,8 +18,6 @@
 #include <dhcpsrv/csv_lease_file4.h>
 #include <dhcpsrv/lease.h>
 #include <dhcpsrv/tests/lease_file_io.h>
-#include <boost/scoped_ptr.hpp>
-#include <boost/shared_ptr.hpp>
 #include <gtest/gtest.h>
 #include <sstream>
 
@@ -125,22 +123,22 @@ TEST_F(CSVLeaseFile4Test, parse) {
     writeSampleFile();
 
     // Open the lease file.
-    boost::scoped_ptr<CSVLeaseFile4> lf(new CSVLeaseFile4(filename_));
-    ASSERT_NO_THROW(lf->open());
+    CSVLeaseFile4 lf(filename_);
+    ASSERT_NO_THROW(lf.open());
 
     // Verify the counters are cleared
     {
     SCOPED_TRACE("Check stats are empty");
-    checkStats(*lf, 0, 0, 0, 0, 0, 0);
+    checkStats(lf, 0, 0, 0, 0, 0, 0);
     }
 
     Lease4Ptr lease;
     // Reading first read should be successful.
     {
     SCOPED_TRACE("First lease valid");
-    EXPECT_TRUE(lf->next(lease));
+    EXPECT_TRUE(lf.next(lease));
     ASSERT_TRUE(lease);
-    checkStats(*lf, 1, 1, 0, 0, 0, 0);
+    checkStats(lf, 1, 1, 0, 0, 0, 0);
 
     // Verify that the lease attributes are correct.
     EXPECT_EQ("192.0.2.1", lease->addr_.toText());
@@ -159,17 +157,17 @@ TEST_F(CSVLeaseFile4Test, parse) {
     // Second lease is malformed - HW address is empty.
     {
     SCOPED_TRACE("Second lease malformed");
-    EXPECT_FALSE(lf->next(lease));
-    checkStats(*lf, 2, 1, 1, 0, 0, 0);
+    EXPECT_FALSE(lf.next(lease));
+    checkStats(lf, 2, 1, 1, 0, 0, 0);
     }
 
     // Even though parsing previous lease failed, reading the next lease should be
     // successful.
     {
     SCOPED_TRACE("Third lease valid");
-    EXPECT_TRUE(lf->next(lease));
+    EXPECT_TRUE(lf.next(lease));
     ASSERT_TRUE(lease);
-    checkStats(*lf, 3, 2, 1, 0, 0, 0);
+    checkStats(lf, 3, 2, 1, 0, 0, 0);
 
     // Verify that the third lease is correct.
     EXPECT_EQ("192.0.3.15", lease->addr_.toText());
@@ -190,28 +188,28 @@ TEST_F(CSVLeaseFile4Test, parse) {
     // lease pointer should be NULL.
     {
     SCOPED_TRACE("Fifth read empty");
-    EXPECT_TRUE(lf->next(lease));
+    EXPECT_TRUE(lf.next(lease));
     EXPECT_FALSE(lease);
-    checkStats(*lf, 4, 2, 1, 0, 0, 0);
+    checkStats(lf, 4, 2, 1, 0, 0, 0);
     }
 
     // We should be able to do it again.
     {
     SCOPED_TRACE("Sixth read empty");
-    EXPECT_TRUE(lf->next(lease));
+    EXPECT_TRUE(lf.next(lease));
     EXPECT_FALSE(lease);
-    checkStats(*lf, 5, 2, 1, 0, 0, 0);
+    checkStats(lf, 5, 2, 1, 0, 0, 0);
     }
 }
 
 // This test checks creation of the lease file and writing leases.
 TEST_F(CSVLeaseFile4Test, recreate) {
-    boost::scoped_ptr<CSVLeaseFile4> lf(new CSVLeaseFile4(filename_));
-    ASSERT_NO_THROW(lf->recreate());
+    CSVLeaseFile4 lf(filename_);
+    ASSERT_NO_THROW(lf.recreate());
     ASSERT_TRUE(io_.exists());
 
     // Verify the counters are cleared
-    checkStats(*lf, 0, 0, 0, 0, 0, 0);
+    checkStats(lf, 0, 0, 0, 0, 0, 0);
 
     // Create first lease, with NULL client id.
     Lease4Ptr lease(new Lease4(IOAddress("192.0.3.2"),
@@ -222,8 +220,8 @@ TEST_F(CSVLeaseFile4Test, recreate) {
     lease->state_ = Lease::STATE_EXPIRED_RECLAIMED;
     {
     SCOPED_TRACE("First write");
-    ASSERT_NO_THROW(lf->append(*lease));
-    checkStats(*lf, 0, 0, 0, 1, 1, 0);
+    ASSERT_NO_THROW(lf.append(*lease));
+    checkStats(lf, 0, 0, 0, 1, 1, 0);
     }
 
     // Create second lease, with non-NULL client id.
@@ -233,12 +231,12 @@ TEST_F(CSVLeaseFile4Test, recreate) {
                            100, 60, 90, 0, 7));
     {
     SCOPED_TRACE("Second write");
-    ASSERT_NO_THROW(lf->append(*lease));
-    checkStats(*lf, 0, 0, 0, 2, 2, 0);
+    ASSERT_NO_THROW(lf.append(*lease));
+    checkStats(lf, 0, 0, 0, 2, 2, 0);
     }
 
     // Close the lease file.
-    lf->close();
+    lf.close();
     // Check that the contents of the csv file are correct.
     EXPECT_EQ("address,hwaddr,client_id,valid_lifetime,expire,subnet_id,"
               "fqdn_fwd,fqdn_rev,hostname,state\n"
@@ -248,6 +246,155 @@ TEST_F(CSVLeaseFile4Test, recreate) {
               io_.readFile());
 }
 
+// Verifies that a schema 1.0 file with records from
+// schema 1.0 and 2.0 loads correctly.
+TEST_F(CSVLeaseFile4Test, mixedSchemaload) {
+    // Create mixed schema file
+    io_.writeFile(
+                  // schema 1.0 header
+                  "address,hwaddr,client_id,valid_lifetime,expire,subnet_id,"
+                  "fqdn_fwd,fqdn_rev,hostname\n"
+                  // schema 1.0 record
+                  "192.0.2.1,06:07:08:09:1a:bc,,200,200,8,1,1,"
+                  "one.example.com\n"
+                  // schema 2.0 record - has state
+                  "192.0.2.2,06:07:08:09:2a:bc,,200,200,8,1,1,"
+                  "two.example.com,1\n"
+                  // schema 2.0 record - has state
+                  "192.0.2.3,06:07:08:09:3a:bc,,200,200,8,1,1,"
+                  "three.example.com,2\n"
+                   );
+
+    // Open the lease file.
+    CSVLeaseFile4 lf(filename_);
+    ASSERT_NO_THROW(lf.open());
+
+    Lease4Ptr lease;
+
+    // Reading first read should be successful.
+    {
+    SCOPED_TRACE("First lease valid");
+    EXPECT_TRUE(lf.next(lease));
+    ASSERT_TRUE(lease);
+
+    // Verify that the lease attributes are correct.
+    EXPECT_EQ("192.0.2.1", lease->addr_.toText());
+    HWAddr hwaddr1(*lease->hwaddr_);
+    EXPECT_EQ("06:07:08:09:1a:bc", hwaddr1.toText(false));
+    EXPECT_FALSE(lease->client_id_);
+    EXPECT_EQ(200, lease->valid_lft_);
+    EXPECT_EQ(0, lease->cltt_);
+    EXPECT_EQ(8, lease->subnet_id_);
+    EXPECT_TRUE(lease->fqdn_fwd_);
+    EXPECT_TRUE(lease->fqdn_rev_);
+    EXPECT_EQ("one.example.com", lease->hostname_);
+    // Verify that added state is DEFAULT
+    EXPECT_EQ(Lease::STATE_DEFAULT, lease->state_);
+    }
+
+    {
+    SCOPED_TRACE("Second lease valid");
+    EXPECT_TRUE(lf.next(lease));
+    ASSERT_TRUE(lease);
+
+    // Verify that the lease attributes are correct.
+    EXPECT_EQ("192.0.2.2", lease->addr_.toText());
+    HWAddr hwaddr1(*lease->hwaddr_);
+    EXPECT_EQ("06:07:08:09:2a:bc", hwaddr1.toText(false));
+    EXPECT_FALSE(lease->client_id_);
+    EXPECT_EQ(200, lease->valid_lft_);
+    EXPECT_EQ(0, lease->cltt_);
+    EXPECT_EQ(8, lease->subnet_id_);
+    EXPECT_TRUE(lease->fqdn_fwd_);
+    EXPECT_TRUE(lease->fqdn_rev_);
+    EXPECT_EQ("two.example.com", lease->hostname_);
+    EXPECT_EQ(Lease::STATE_DECLINED, lease->state_);
+    }
+
+    {
+    SCOPED_TRACE("Third lease valid");
+    EXPECT_TRUE(lf.next(lease));
+    ASSERT_TRUE(lease);
+
+    // Verify that the third lease is correct.
+    EXPECT_EQ("192.0.2.3", lease->addr_.toText());
+    HWAddr hwaddr1(*lease->hwaddr_);
+    EXPECT_EQ("06:07:08:09:3a:bc", hwaddr1.toText(false));
+    EXPECT_FALSE(lease->client_id_);
+    EXPECT_EQ(200, lease->valid_lft_);
+    EXPECT_EQ(0, lease->cltt_);
+    EXPECT_EQ(8, lease->subnet_id_);
+    EXPECT_TRUE(lease->fqdn_fwd_);
+    EXPECT_TRUE(lease->fqdn_rev_);
+    EXPECT_EQ("three.example.com", lease->hostname_);
+    EXPECT_EQ(Lease::STATE_EXPIRED_RECLAIMED, lease->state_);
+    }
+}
+
+
+// Verifies that a lease file with fewer header columns than the
+// minimum allowed will not open.
+TEST_F(CSVLeaseFile4Test, tooFewHeaderColumns) {
+    // Create 1.0 file
+    io_.writeFile("address,hwaddr,client_id,valid_lifetime,expire,subnet_id,"
+                  "fqdn_fwd,fqdn_rev\n");
+
+    // Open the lease file.
+    CSVLeaseFile4 lf(filename_);
+    ASSERT_THROW(lf.open(), CSVFileError);
+}
+
+// Verifies that a lease file with an unrecognized column header
+// will not open.
+TEST_F(CSVLeaseFile4Test, invalidHeaderColumn) {
+    // Create 1.0 file
+    io_.writeFile("address,hwaddr,BOGUS,valid_lifetime,expire,subnet_id,"
+                  "fqdn_fwd,fqdn_rev,hostname,state\n");
+
+    // Open the lease file.
+    CSVLeaseFile4 lf(filename_);
+    ASSERT_THROW(lf.open(), CSVFileError);
+}
+
+// Verifies that a lease file with more header columns than defined
+// columns will downgrade.
+TEST_F(CSVLeaseFile4Test, downGrade) {
+    // Create 2.0 PLUS a column file
+    io_.writeFile("address,hwaddr,client_id,valid_lifetime,expire,subnet_id,"
+                  "fqdn_fwd,fqdn_rev,hostname,state,FUTURE_COL\n"
+
+                  "192.0.2.3,06:07:08:09:3a:bc,,200,200,8,1,1,"
+                  "three.example.com,2,BOGUS\n");
+
+    // Lease file should open and report as needing downgrade.
+    CSVLeaseFile4 lf(filename_);
+    ASSERT_NO_THROW(lf.open());
+    EXPECT_TRUE(lf.needsConversion());
+    EXPECT_EQ(util::VersionedCSVFile::NEEDS_DOWNGRADE,
+              lf.getInputSchemaState());
+    Lease4Ptr lease;
+
+    {
+    SCOPED_TRACE("First lease valid");
+    EXPECT_TRUE(lf.next(lease));
+    ASSERT_TRUE(lease);
+
+    // Verify that the third lease is correct.
+    EXPECT_EQ("192.0.2.3", lease->addr_.toText());
+    HWAddr hwaddr1(*lease->hwaddr_);
+    EXPECT_EQ("06:07:08:09:3a:bc", hwaddr1.toText(false));
+    EXPECT_FALSE(lease->client_id_);
+    EXPECT_EQ(200, lease->valid_lft_);
+    EXPECT_EQ(0, lease->cltt_);
+    EXPECT_EQ(8, lease->subnet_id_);
+    EXPECT_TRUE(lease->fqdn_fwd_);
+    EXPECT_TRUE(lease->fqdn_rev_);
+    EXPECT_EQ("three.example.com", lease->hostname_);
+    EXPECT_EQ(Lease::STATE_EXPIRED_RECLAIMED, lease->state_);
+    }
+}
+
+
 /// @todo Currently we don't check invalid lease attributes, such as invalid
 /// lease type, invalid preferred lifetime vs valid lifetime etc. The Lease6
 /// should be extended with the function that validates lease attributes. Once

src/lib/dhcpsrv/tests/csv_lease_file6_unittest.cc

@@ -18,8 +18,6 @@
 #include <dhcpsrv/csv_lease_file6.h>
 #include <dhcpsrv/lease.h>
 #include <dhcpsrv/tests/lease_file_io.h>
-#include <boost/scoped_ptr.hpp>
-#include <boost/shared_ptr.hpp>
 #include <gtest/gtest.h>
 #include <sstream>
 
@@ -126,22 +124,22 @@ TEST_F(CSVLeaseFile6Test, parse) {
     writeSampleFile();
 
     // Open the lease file.
-    boost::scoped_ptr<CSVLeaseFile6> lf(new CSVLeaseFile6(filename_));
-    ASSERT_NO_THROW(lf->open());
+    CSVLeaseFile6 lf(filename_);
+    ASSERT_NO_THROW(lf.open());
 
     // Verify the counters are cleared
     {
     SCOPED_TRACE("Check stats are empty");
-    checkStats(*lf, 0, 0, 0, 0, 0, 0);
+    checkStats(lf, 0, 0, 0, 0, 0, 0);
     }
 
     Lease6Ptr lease;
     // Reading first read should be successful.
     {
     SCOPED_TRACE("First lease valid");
-    EXPECT_TRUE(lf->next(lease));
+    EXPECT_TRUE(lf.next(lease));
     ASSERT_TRUE(lease);
-    checkStats(*lf, 1, 1, 0, 0, 0, 0);
+    checkStats(lf, 1, 1, 0, 0, 0, 0);
 
     // Verify that the lease attributes are correct.
     EXPECT_EQ("2001:db8:1::1", lease->addr_.toText());
@@ -162,17 +160,17 @@ TEST_F(CSVLeaseFile6Test, parse) {
     // Second lease is malformed - DUID is empty.
     {
     SCOPED_TRACE("Second lease malformed");
-    EXPECT_FALSE(lf->next(lease));
-    checkStats(*lf, 2, 1, 1, 0, 0, 0);
+    EXPECT_FALSE(lf.next(lease));
+    checkStats(lf, 2, 1, 1, 0, 0, 0);
     }
 
     // Even, parsing previous lease failed, reading the next lease should be
     // successful.
     {
     SCOPED_TRACE("Third lease valid");
-    EXPECT_TRUE(lf->next(lease));
+    EXPECT_TRUE(lf.next(lease));
     ASSERT_TRUE(lease);
-    checkStats(*lf, 3, 2, 1, 0, 0, 0);
+    checkStats(lf, 3, 2, 1, 0, 0, 0);
 
     // Verify that the third lease is correct.
     EXPECT_EQ("2001:db8:2::10", lease->addr_.toText());
@@ -193,9 +191,9 @@ TEST_F(CSVLeaseFile6Test, parse) {
     // Reading the fourth lease should be successful.
     {
     SCOPED_TRACE("Fourth lease valid");
-    EXPECT_TRUE(lf->next(lease));
+    EXPECT_TRUE(lf.next(lease));
     ASSERT_TRUE(lease);
-    checkStats(*lf, 4, 3, 1, 0, 0, 0);
+    checkStats(lf, 4, 3, 1, 0, 0, 0);
 
     // Verify that the lease is correct.
     EXPECT_EQ("3000:1::", lease->addr_.toText());
@@ -217,30 +215,30 @@ TEST_F(CSVLeaseFile6Test, parse) {
     // lease pointer should be NULL.
     {
     SCOPED_TRACE("Fifth read empty");
-    EXPECT_TRUE(lf->next(lease));
+    EXPECT_TRUE(lf.next(lease));
     EXPECT_FALSE(lease);
-    checkStats(*lf, 5, 3, 1, 0, 0, 0);
+    checkStats(lf, 5, 3, 1, 0, 0, 0);
     }
 
     // We should be able to do it again.
     {
     SCOPED_TRACE("Sixth read empty");
-    EXPECT_TRUE(lf->next(lease));
+    EXPECT_TRUE(lf.next(lease));
     EXPECT_FALSE(lease);
-    checkStats(*lf, 6, 3, 1, 0, 0, 0);
+    checkStats(lf, 6, 3, 1, 0, 0, 0);
     }
 }
 
 // This test checks creation of the lease file and writing leases.
 TEST_F(CSVLeaseFile6Test, recreate) {
-    boost::scoped_ptr<CSVLeaseFile6> lf(new CSVLeaseFile6(filename_));
-    ASSERT_NO_THROW(lf->recreate());
+    CSVLeaseFile6 lf(filename_);
+    ASSERT_NO_THROW(lf.recreate());
     ASSERT_TRUE(io_.exists());
 
     // Verify the counters are cleared
     {
     SCOPED_TRACE("Check stats are empty");
-    checkStats(*lf, 0, 0, 0, 0, 0, 0);
+    checkStats(lf, 0, 0, 0, 0, 0, 0);
     }
 
     Lease6Ptr lease(new Lease6(Lease::TYPE_NA, IOAddress("2001:db8:1::1"),
@@ -250,8 +248,8 @@ TEST_F(CSVLeaseFile6Test, recreate) {
     lease->cltt_ = 0;
     {
     SCOPED_TRACE("First write");
-    ASSERT_NO_THROW(lf->append(*lease));
-    checkStats(*lf, 0, 0, 0, 1, 1, 0);
+    ASSERT_NO_THROW(lf.append(*lease));
+    checkStats(lf, 0, 0, 0, 1, 1, 0);
     }
 
     lease.reset(new Lease6(Lease::TYPE_NA, IOAddress("2001:db8:2::10"),
@@ -261,8 +259,8 @@ TEST_F(CSVLeaseFile6Test, recreate) {
     lease->cltt_ = 0;
     {
     SCOPED_TRACE("Second write");
-    ASSERT_NO_THROW(lf->append(*lease));
-    checkStats(*lf, 0, 0, 0, 2, 2, 0);
+    ASSERT_NO_THROW(lf.append(*lease));
+    checkStats(lf, 0, 0, 0, 2, 2, 0);
     }
 
     lease.reset(new Lease6(Lease::TYPE_PD, IOAddress("3000:1:1::"),
@@ -272,8 +270,8 @@ TEST_F(CSVLeaseFile6Test, recreate) {
     lease->cltt_ = 0;
     {
     SCOPED_TRACE("Third write");
-    ASSERT_NO_THROW(lf->append(*lease));
-    checkStats(*lf, 0, 0, 0, 3, 3, 0);
+    ASSERT_NO_THROW(lf.append(*lease));
+    checkStats(lf, 0, 0, 0, 3, 3, 0);
     }
 
     EXPECT_EQ("address,duid,valid_lifetime,expire,subnet_id,pref_lifetime,"
@@ -288,6 +286,180 @@ TEST_F(CSVLeaseFile6Test, recreate) {
               io_.readFile());
 }
 
+// Verifies that a 1.0 schema file with records from
+// schema 1.0, 2.0, and 3.0 loads correctly.
+TEST_F(CSVLeaseFile6Test, mixedSchemaLoad) {
+    // Create a mixed schema file
+    io_.writeFile(
+             // schema 1.0 header
+              "address,duid,valid_lifetime,expire,subnet_id,pref_lifetime,"
+              "lease_type,iaid,prefix_len,fqdn_fwd,fqdn_rev,hostname\n"
+              // schema 1.0 record
+              "2001:db8:1::1,00:01:02:03:04:05:06:0a:0b:0c:0d:0e:01,"
+              "200,200,8,100,0,7,0,1,1,one.example.com\n"
+
+              // schema 2.0 record - has hwaddr
+              "2001:db8:1::2,00:01:02:03:04:05:06:0a:0b:0c:0d:0e:02,"
+              "200,200,8,100,0,7,0,1,1,two.example.com,01:02:03:04:05\n"
+
+              // schema 3.0 record - has hwaddr and state
+              "2001:db8:1::3,00:01:02:03:04:05:06:0a:0b:0c:0d:0e:03,"
+              "200,200,8,100,0,7,0,1,1,three.example.com,0a:0b:0c:0d:0e,1\n");
+
+    // Open the lease file.
+    CSVLeaseFile6 lf(filename_);
+    ASSERT_NO_THROW(lf.open());
+
+    Lease6Ptr lease;
+    {
+    SCOPED_TRACE("First lease valid");
+    EXPECT_TRUE(lf.next(lease));
+    ASSERT_TRUE(lease);
+
+    // Verify that the lease attributes are correct.
+    EXPECT_EQ("2001:db8:1::1", lease->addr_.toText());
+    ASSERT_TRUE(lease->duid_);
+    EXPECT_EQ("00:01:02:03:04:05:06:0a:0b:0c:0d:0e:01", lease->duid_->toText());
+    EXPECT_EQ(200, lease->valid_lft_);
+    EXPECT_EQ(0, lease->cltt_);
+    EXPECT_EQ(8, lease->subnet_id_);
+    EXPECT_EQ(100, lease->preferred_lft_);
+    EXPECT_EQ(Lease::TYPE_NA, lease->type_);
+    EXPECT_EQ(7, lease->iaid_);
+    EXPECT_EQ(0, lease->prefixlen_);
+    EXPECT_TRUE(lease->fqdn_fwd_);
+    EXPECT_TRUE(lease->fqdn_rev_);
+    EXPECT_EQ("one.example.com", lease->hostname_);
+    // Verify that added HWaddr is empty
+    EXPECT_FALSE(lease->hwaddr_);
+    // Verify that added state is STATE_DEFAULT
+    EXPECT_EQ(Lease::STATE_DEFAULT, lease->state_);
+    }
+
+    {
+    SCOPED_TRACE("Second lease valid");
+    EXPECT_TRUE(lf.next(lease));
+    ASSERT_TRUE(lease);
+
+    // Verify that the lease attributes are correct.
+    EXPECT_EQ("2001:db8:1::2", lease->addr_.toText());
+    ASSERT_TRUE(lease->duid_);
+    EXPECT_EQ("00:01:02:03:04:05:06:0a:0b:0c:0d:0e:02", lease->duid_->toText());
+    EXPECT_EQ(200, lease->valid_lft_);
+    EXPECT_EQ(0, lease->cltt_);
+    EXPECT_EQ(8, lease->subnet_id_);
+    EXPECT_EQ(100, lease->preferred_lft_);
+    EXPECT_EQ(Lease::TYPE_NA, lease->type_);
+    EXPECT_EQ(7, lease->iaid_);
+    EXPECT_EQ(0, lease->prefixlen_);
+    EXPECT_TRUE(lease->fqdn_fwd_);
+    EXPECT_TRUE(lease->fqdn_rev_);
+    EXPECT_EQ("two.example.com", lease->hostname_);
+    ASSERT_TRUE(lease->hwaddr_);
+    EXPECT_EQ("01:02:03:04:05", lease->hwaddr_->toText(false));
+    // Verify that added state is STATE_DEFAULT
+    EXPECT_EQ(Lease::STATE_DEFAULT, lease->state_);
+    }
+
+    {
+    SCOPED_TRACE("Third lease valid");
+    EXPECT_TRUE(lf.next(lease));
+    ASSERT_TRUE(lease);
+
+    // Verify that the lease attributes are correct.
+    EXPECT_EQ("2001:db8:1::3", lease->addr_.toText());
+    ASSERT_TRUE(lease->duid_);
+    EXPECT_EQ("00:01:02:03:04:05:06:0a:0b:0c:0d:0e:03", lease->duid_->toText());
+    EXPECT_EQ(200, lease->valid_lft_);
+    EXPECT_EQ(0, lease->cltt_);
+    EXPECT_EQ(8, lease->subnet_id_);
+    EXPECT_EQ(100, lease->preferred_lft_);
+    EXPECT_EQ(Lease::TYPE_NA, lease->type_);
+    EXPECT_EQ(7, lease->iaid_);
+    EXPECT_EQ(0, lease->prefixlen_);
+    EXPECT_TRUE(lease->fqdn_fwd_);
+    EXPECT_TRUE(lease->fqdn_rev_);
+    EXPECT_EQ("three.example.com", lease->hostname_);
+    ASSERT_TRUE(lease->hwaddr_);
+    EXPECT_EQ("0a:0b:0c:0d:0e", lease->hwaddr_->toText(false));
+    EXPECT_EQ(Lease::STATE_DECLINED, lease->state_);
+    }
+
+}
+
+// Verifies that a lease file with fewer header columns than the
+// minimum allowed will not open.
+TEST_F(CSVLeaseFile6Test, tooFewHeaderColumns) {
+    io_.writeFile("address,duid,valid_lifetime,expire,subnet_id,pref_lifetime,"
+              "lease_type,iaid,prefix_len,fqdn_fwd,fqdn_rev\n");
+
+    // Open should fail.
+    CSVLeaseFile6 lf(filename_);
+    ASSERT_THROW(lf.open(), CSVFileError);
+}
+
+// Verifies that a lease file with an unrecognized column header
+// will not open.
+TEST_F(CSVLeaseFile6Test, invalidHeaderColumn) {
+    io_.writeFile("address,BOGUS,valid_lifetime,expire,subnet_id,pref_lifetime,"
+              "lease_type,iaid,prefix_len,fqdn_fwd,fqdn_rev,hostname,"
+              "hwaddr,state\n");
+
+    // Open should fail.
+    CSVLeaseFile6 lf(filename_);
+    ASSERT_THROW(lf.open(), CSVFileError);
+}
+
+// Verifies that a lease file with more header columns than defined
+// columns will open as needing a downgrade.
+TEST_F(CSVLeaseFile6Test, downGrade) {
+    // Create a mixed schema file
+    io_.writeFile(
+             // schema 1.0 header
+              "address,duid,valid_lifetime,expire,subnet_id,pref_lifetime,"
+              "lease_type,iaid,prefix_len,fqdn_fwd,fqdn_rev,hostname,"
+              "hwaddr,state,FUTURE_COL\n"
+
+              // schema 3.0 record - has hwaddr and state
+              "2001:db8:1::3,00:01:02:03:04:05:06:0a:0b:0c:0d:0e:03,"
+              "200,200,8,100,0,7,0,1,1,three.example.com,0a:0b:0c:0d:0e,1,"
+              "BOGUS\n");
+
+    // Open should succeed in the event someone is downgrading.
+    CSVLeaseFile6 lf(filename_);
+    ASSERT_NO_THROW(lf.open());
+    EXPECT_TRUE(lf.needsConversion());
+    EXPECT_EQ(util::VersionedCSVFile::NEEDS_DOWNGRADE,
+              lf.getInputSchemaState());
+
+
+    Lease6Ptr lease;
+    {
+    SCOPED_TRACE("First lease valid");
+    EXPECT_TRUE(lf.next(lease));
+    ASSERT_TRUE(lease);
+
+    // Verify that the lease attributes are correct.
+    EXPECT_EQ("2001:db8:1::3", lease->addr_.toText());
+    ASSERT_TRUE(lease->duid_);
+    EXPECT_EQ("00:01:02:03:04:05:06:0a:0b:0c:0d:0e:03", lease->duid_->toText());
+    EXPECT_EQ(200, lease->valid_lft_);
+    EXPECT_EQ(0, lease->cltt_);
+    EXPECT_EQ(8, lease->subnet_id_);
+    EXPECT_EQ(100, lease->preferred_lft_);
+    EXPECT_EQ(Lease::TYPE_NA, lease->type_);
+    EXPECT_EQ(7, lease->iaid_);
+    EXPECT_EQ(0, lease->prefixlen_);
+    EXPECT_TRUE(lease->fqdn_fwd_);
+    EXPECT_TRUE(lease->fqdn_rev_);
+    EXPECT_EQ("three.example.com", lease->hostname_);
+    ASSERT_TRUE(lease->hwaddr_);
+    EXPECT_EQ("0a:0b:0c:0d:0e", lease->hwaddr_->toText(false));
+    EXPECT_EQ(Lease::STATE_DECLINED, lease->state_);
+    }
+}
+
+
 /// @todo Currently we don't check invalid lease attributes, such as invalid
 /// lease type, invalid preferred lifetime vs valid lifetime etc. The Lease6
 /// should be extended with the function that validates lease attributes. Once

src/lib/dhcpsrv/tests/memfile_lease_mgr_unittest.cc

@@ -1388,4 +1388,137 @@ TEST_F(MemfileLeaseMgrTest, load6LFCInProgress) {
     ASSERT_NO_THROW(lease_mgr.reset(new NakedMemfileLeaseMgr(pmap)));
 }
 
+// Verifies that LFC is automatically run during MemfileLeasemMgr construction
+// when the lease file(s) being loaded need to be upgraded.
+TEST_F(MemfileLeaseMgrTest, leaseUpgrade4) {
+    // Create header strings for each schema
+    std::string header_1_0 =
+        "address,hwaddr,client_id,valid_lifetime,expire,"
+        "subnet_id,fqdn_fwd,fqdn_rev,hostname\n";
+
+    std::string header_2_0 =
+        "address,hwaddr,client_id,valid_lifetime,expire,"
+        "subnet_id,fqdn_fwd,fqdn_rev,hostname,state\n";
+
+    // Create 1.0 Schema current lease file with two entries for
+    // the same lease
+    std::string current_file_contents = header_1_0 +
+        "192.0.2.2,02:02:02:02:02:02,,200,200,8,1,1,\n"
+        "192.0.2.2,02:02:02:02:02:02,,200,800,8,1,1,\n";
+    LeaseFileIO current_file(getLeaseFilePath("leasefile4_0.csv"));
+    current_file.writeFile(current_file_contents);
+
+    // Create 1.0 Schema previous lease file, with two entries for
+    // a another lease
+    std::string previous_file_contents = header_1_0 +
+        "192.0.2.3,03:03:03:03:03:03,,200,200,8,1,1,\n"
+        "192.0.2.3,03:03:03:03:03:03,,200,800,8,1,1,\n";
+    LeaseFileIO previous_file(getLeaseFilePath("leasefile4_0.csv.2"));
+    previous_file.writeFile(previous_file_contents);
+
+    // Create the backend.
+    DatabaseConnection::ParameterMap pmap;
+    pmap["type"] = "memfile";
+    pmap["universe"] = "4";
+    pmap["name"] = getLeaseFilePath("leasefile4_0.csv");
+    pmap["lfc-interval"] = "0";
+    boost::scoped_ptr<NakedMemfileLeaseMgr> lease_mgr(new NakedMemfileLeaseMgr(pmap));
+
+    // Since lease files are loaded during lease manager
+    // constructor, LFC should get launched automatically.
+    // The new lease file should be 2.0 schema and have no entries
+    ASSERT_TRUE(current_file.exists());
+    EXPECT_EQ(header_2_0, current_file.readFile());
+
+    // Wait for the LFC process to complete and
+    // make sure it has returned an exit status of 0.
+    ASSERT_TRUE(waitForProcess(*lease_mgr, 2));
+    ASSERT_EQ(0, lease_mgr->getLFCExitStatus())
+        << "Executing the LFC process failed: make sure that"
+        " the kea-lfc program has been compiled.";
+
+    // The LFC should have created a 2.0 schema completion file with the
+    // one entry for each lease and moved it to leasefile4_0.csv.2
+    LeaseFileIO input_file(getLeaseFilePath("leasefile4_0.csv.2"), false);
+    ASSERT_TRUE(input_file.exists());
+
+    // Verify cleaned, converted contents
+    std::string result_file_contents = header_2_0 +
+        "192.0.2.2,02:02:02:02:02:02,,200,800,8,1,1,,0\n"
+        "192.0.2.3,03:03:03:03:03:03,,200,800,8,1,1,,0\n";
+    EXPECT_EQ(result_file_contents, input_file.readFile());
+}
+
+TEST_F(MemfileLeaseMgrTest, leaseUpgrade6) {
+    // Create header strings for all three schemas
+    std::string header_1_0 =
+        "address,duid,valid_lifetime,expire,subnet_id,"
+        "pref_lifetime,lease_type,iaid,prefix_len,fqdn_fwd,"
+        "fqdn_rev,hostname\n";
+
+    std::string header_2_0 =
+        "address,duid,valid_lifetime,expire,subnet_id,"
+        "pref_lifetime,lease_type,iaid,prefix_len,fqdn_fwd,"
+        "fqdn_rev,hostname,hwaddr\n";
+
+    std::string header_3_0 =
+        "address,duid,valid_lifetime,expire,subnet_id,"
+        "pref_lifetime,lease_type,iaid,prefix_len,fqdn_fwd,"
+        "fqdn_rev,hostname,hwaddr,state\n";
+
+    // The current lease file is schema 1.0 and has two entries for
+    // the same lease
+    std::string current_file_contents = header_1_0 +
+        "2001:db8:1::1,00:01:02:03:04:05:06:0a:0b:0c:0d:0e:0f,200,200,"
+        "8,100,0,7,0,1,1,,\n"
+        "2001:db8:1::1,00:01:02:03:04:05:06:0a:0b:0c:0d:0e:0f,200,800,"
+        "8,100,0,7,0,1,1,,\n";
+    LeaseFileIO current_file(getLeaseFilePath("leasefile6_0.csv"));
+    current_file.writeFile(current_file_contents);
+
+    // The previous lease file is schema 2.0 and has two entries for
+    // a different lease
+    std::string previous_file_contents = header_2_0 +
+        "2001:db8:1::2,01:01:01:01:01:01:01:01:01:01:01:01:01,200,200,"
+        "8,100,0,7,0,1,1,,11:22:33:44:55\n"
+        "2001:db8:1::2,01:01:01:01:01:01:01:01:01:01:01:01:01,200,800,"
+        "8,100,0,7,0,1,1,,11:22:33:44:55\n";
+    LeaseFileIO previous_file(getLeaseFilePath("leasefile6_0.csv.2"));
+    previous_file.writeFile(previous_file_contents);
+
+    // Create the backend.
+    DatabaseConnection::ParameterMap pmap;
+    pmap["type"] = "memfile";
+    pmap["universe"] = "6";
+    pmap["name"] = getLeaseFilePath("leasefile6_0.csv");
+    pmap["lfc-interval"] = "0";
+    boost::scoped_ptr<NakedMemfileLeaseMgr> lease_mgr(new NakedMemfileLeaseMgr(pmap));
+
+    // Since lease files are loaded during lease manager
+    // constructor, LFC should get launched automatically.
+    // The new lease file should been 3.0 and contain no leases.
+    ASSERT_TRUE(current_file.exists());
+    EXPECT_EQ(header_3_0, current_file.readFile());
+
+    // Wait for the LFC process to complete and
+    // make sure it has returned an exit status of 0.
+    ASSERT_TRUE(waitForProcess(*lease_mgr, 2));
+    ASSERT_EQ(0, lease_mgr->getLFCExitStatus())
+        << "Executing the LFC process failed: make sure that"
+        " the kea-lfc program has been compiled.";
+
+    // The LFC should have created a 3.0 schema cleaned file with one entry
+    // for each lease as leasefile6_0.csv.2
+    LeaseFileIO input_file(getLeaseFilePath("leasefile6_0.csv.2"), false);
+    ASSERT_TRUE(input_file.exists());
+
+    // Verify cleaned, converted contents
+    std::string result_file_contents = header_3_0 +
+        "2001:db8:1::1,00:01:02:03:04:05:06:0a:0b:0c:0d:0e:0f,200,800,"
+        "8,100,0,7,0,1,1,,,0\n"
+        "2001:db8:1::2,01:01:01:01:01:01:01:01:01:01:01:01:01,200,800,"
+        "8,100,0,7,0,1,1,,11:22:33:44:55,0\n";
+    EXPECT_EQ(result_file_contents, input_file.readFile());
+}
+
 }; // end of anonymous namespace

src/lib/util/Makefile.am

@@ -23,6 +23,7 @@ libkea_util_la_SOURCES += range_utilities.h
 libkea_util_la_SOURCES += signal_set.cc signal_set.h
 libkea_util_la_SOURCES += stopwatch.cc stopwatch.h
 libkea_util_la_SOURCES += stopwatch_impl.cc stopwatch_impl.h
+libkea_util_la_SOURCES += versioned_csv_file.h versioned_csv_file.cc
 libkea_util_la_SOURCES += watch_socket.cc watch_socket.h
 libkea_util_la_SOURCES += encode/base16_from_binary.h
 libkea_util_la_SOURCES += encode/base32hex.h encode/base64.h

src/lib/util/csv_file.cc

@@ -65,6 +65,12 @@ CSVRow::writeAt(const size_t at, const char* value) {
     values_[at] = value;
 }
 
+void
+CSVRow::trim(const size_t count) {
+    checkIndex(count);
+    values_.resize(values_.size() - count);
+}
+
 std::ostream& operator<<(std::ostream& os, const CSVRow& row) {
     os << row.render();
     return (os);
@@ -296,9 +302,9 @@ CSVFile::open(const bool seek_to_end) {
 
             // Check the header against the columns specified for the CSV file.
             if (!validateHeader(header)) {
-
                 isc_throw(CSVFileError, "invalid header '" << header
-                          << "' in CSV file '" << filename_ << "'");
+                          << "' in CSV file '" << filename_ << "': "
+                          << getReadMsg());
             }
 
             // Everything is good, so if we haven't added any columns yet,

src/lib/util/csv_file.h

@@ -117,6 +117,14 @@ public:
     /// @c CSVRow::getValuesCount.
     std::string readAt(const size_t at) const;
 
+    /// @brief Trims a given number of elements from the end of a row
+    ///
+    /// @param number of elements to trim
+    ///
+    /// @throw CSVFileError if the number to trim is larger than
+    /// then the number of elements
+    void trim(const size_t count);
+
     /// @brief Retrieves a value from the internal container.
     ///
     /// This method is reads a value from the internal container and converts
@@ -404,7 +412,7 @@ public:
     /// Otherwise, this function will write the header to the file.
     /// In order to write rows to opened file, the @c append function
     /// should be called.
-    void recreate();
+    virtual void recreate();
 
     /// @brief Sets error message after row validation.
     ///
@@ -469,8 +477,6 @@ protected:
     /// This function is called internally by @ref CSVFile::open. Derived classes
     /// may add extra validation steps.
     ///
-    /// @todo There should be a support for optional columns (see ticket #3626).
-    ///
     /// @param header A row holding a header.
     /// @return true if header matches the columns; false otherwise.
     virtual bool validateHeader(const CSVRow& header);

src/lib/util/tests/Makefile.am

@@ -50,6 +50,7 @@ run_unittests_SOURCES += time_utilities_unittest.cc
 run_unittests_SOURCES += range_utilities_unittest.cc
 run_unittests_SOURCES += signal_set_unittest.cc
 run_unittests_SOURCES += stopwatch_unittest.cc
+run_unittests_SOURCES += versioned_csv_file_unittest.cc
 run_unittests_SOURCES += watch_socket_unittests.cc
 
 

src/lib/util/tests/csv_file_unittest.cc

@@ -109,6 +109,35 @@ TEST(CSVRow, append) {
     EXPECT_EQ("alpha,beta,gamma,delta,epsilon", text);
 }
 
+// This test checks that a row can be trimmed of
+// a given number of elements
+TEST(CSVRow, trim) {
+    CSVRow row("zero,one,two,three,four");
+    ASSERT_EQ(5, row.getValuesCount());
+    EXPECT_EQ("zero", row.readAt(0));
+    EXPECT_EQ("one", row.readAt(1));
+    EXPECT_EQ("two", row.readAt(2));
+    EXPECT_EQ("three", row.readAt(3));
+    EXPECT_EQ("four", row.readAt(4));
+
+    ASSERT_THROW(row.trim(10), CSVFileError);
+
+    // Verify that we can erase just one
+    ASSERT_NO_THROW(row.trim(1));
+    ASSERT_EQ(4, row.getValuesCount());
+    EXPECT_EQ("zero", row.readAt(0));
+    EXPECT_EQ("one", row.readAt(1));
+    EXPECT_EQ("two", row.readAt(2));
+    EXPECT_EQ("three", row.readAt(3));
+
+    // Verfiy we can trim more than one
+    ASSERT_NO_THROW(row.trim(2));
+    ASSERT_EQ(2, row.getValuesCount());
+    EXPECT_EQ("zero", row.readAt(0));
+    EXPECT_EQ("one", row.readAt(1));
+}
+
+
 /// @brief Test fixture class for testing operations on CSV file.
 ///
 /// It implements basic operations on files, such as reading writing

src/lib/util/tests/versioned_csv_file_unittest.cc

@@ -0,0 +1,509 @@
+// Copyright (C) 2015 Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <config.h>
+#include <util/versioned_csv_file.h>
+#include <boost/scoped_ptr.hpp>
+#include <gtest/gtest.h>
+#include <fstream>
+#include <sstream>
+#include <string>
+
+#include <boost/algorithm/string/classification.hpp>
+#include <boost/algorithm/string/constants.hpp>
+#include <boost/algorithm/string/split.hpp>
+
+namespace {
+
+using namespace isc::util;
+
+/// @brief Test fixture class for testing operations on VersionedCSVFile.
+///
+/// It implements basic operations on files, such as reading writing
+/// file removal and checking presence of the file. This is used by
+/// unit tests to verify correctness of the file created by the
+/// CSVFile class.
+class VersionedCSVFileTest : public ::testing::Test {
+public:
+
+    /// @brief Constructor.
+    ///
+    /// Sets the path to the CSV file used throughout the tests.
+    /// The name of the file is test.csv and it is located in the
+    /// current build folder.
+    ///
+    /// It also deletes any dangling files after previous tests.
+    VersionedCSVFileTest();
+
+    /// @brief Destructor.
+    ///
+    /// Deletes the test CSV file if any.
+    virtual ~VersionedCSVFileTest();
+
+    /// @brief Prepends the absolute path to the file specified
+    /// as an argument.
+    ///
+    /// @param filename Name of the file.
+    /// @return Absolute path to the test file.
+    static std::string absolutePath(const std::string& filename);
+
+    /// @brief Check if test file exists on disk.
+    bool exists() const;
+
+    /// @brief Reads whole CSV file.
+    ///
+    /// @return Contents of the file.
+    std::string readFile() const;
+
+    /// @brief Removes existing file (if any).
+    int removeFile() const;
+
+    /// @brief Creates file with contents.
+    ///
+    /// @param contents Contents of the file.
+    void writeFile(const std::string& contents) const;
+
+    /// @brief Absolute path to the file used in the tests.
+    std::string testfile_;
+
+};
+
+VersionedCSVFileTest::VersionedCSVFileTest()
+    : testfile_(absolutePath("test.csv")) {
+    static_cast<void>(removeFile());
+}
+
+VersionedCSVFileTest::~VersionedCSVFileTest() {
+    static_cast<void>(removeFile());
+}
+
+std::string
+VersionedCSVFileTest::absolutePath(const std::string& filename) {
+    std::ostringstream s;
+    s << TEST_DATA_BUILDDIR << "/" << filename;
+    return (s.str());
+}
+
+bool
+VersionedCSVFileTest::exists() const {
+    std::ifstream fs(testfile_.c_str());
+    bool ok = fs.good();
+    fs.close();
+    return (ok);
+}
+
+std::string
+VersionedCSVFileTest::readFile() const {
+    std::ifstream fs(testfile_.c_str());
+    if (!fs.is_open()) {
+        return ("");
+    }
+    std::string contents((std::istreambuf_iterator<char>(fs)),
+                         std::istreambuf_iterator<char>());
+    fs.close();
+    return (contents);
+}
+
+int
+VersionedCSVFileTest::removeFile() const {
+    return (remove(testfile_.c_str()));
+}
+
+void
+VersionedCSVFileTest::writeFile(const std::string& contents) const {
+    std::ofstream fs(testfile_.c_str(), std::ofstream::out);
+    if (fs.is_open()) {
+        fs << contents;
+        fs.close();
+    }
+}
+
+// This test checks that the function which is used to add columns of the
+// CSV file works as expected.
+TEST_F(VersionedCSVFileTest, addColumn) {
+    boost::scoped_ptr<VersionedCSVFile> csv(new VersionedCSVFile(testfile_));
+
+    // Verify that we're not allowed to open it without the schema
+    ASSERT_THROW(csv->open(), VersionedCSVFileError);
+
+    // Add two columns.
+    ASSERT_NO_THROW(csv->addColumn("animal", "1.0", ""));
+    ASSERT_NO_THROW(csv->addColumn("color", "2.0", "blue"));
+
+    // Make sure we can't add duplicates.
+    EXPECT_THROW(csv->addColumn("animal", "1.0", ""), CSVFileError);
+    EXPECT_THROW(csv->addColumn("color", "2.0", "blue"), CSVFileError);
+
+    // But we should still be able to add unique columns.
+    EXPECT_NO_THROW(csv->addColumn("age", "3.0", "21"));
+
+    // Assert that the file is opened, because the rest of the test relies
+    // on this.
+    ASSERT_NO_THROW(csv->recreate());
+    ASSERT_TRUE(exists());
+
+    // We should have 3 defined columns
+    // Input Header should match defined columns on new files
+    // Valid columns should match defined columns on new files
+    // Minium valid columns wasn't set. (Remember it's optional)
+    EXPECT_EQ(3, csv->getColumnCount());
+    EXPECT_EQ(3, csv->getInputHeaderCount());
+    EXPECT_EQ(3, csv->getValidColumnCount());
+    EXPECT_EQ(0, csv->getMinimumValidColumns());
+
+    // Schema versions for new files should always match
+    EXPECT_EQ("3.0", csv->getInputSchemaVersion());
+    EXPECT_EQ("3.0", csv->getSchemaVersion());
+
+    // Input Schema State should be current for new files
+    EXPECT_EQ(VersionedCSVFile::CURRENT, csv->getInputSchemaState());
+    EXPECT_FALSE(csv->needsConversion());
+
+    // Make sure we can't add columns (even unique) when the file is open.
+    ASSERT_THROW(csv->addColumn("zoo", "3.0", ""), CSVFileError);
+
+    // Close the file.
+    ASSERT_NO_THROW(csv->close());
+    // And check that now it is possible to add the column.
+    EXPECT_NO_THROW(csv->addColumn("zoo", "3.0", ""));
+}
+
+// Verifies that a current schema version file loads correctly.
+TEST_F(VersionedCSVFileTest, currentSchemaTest) {
+
+    // Create our versioned file, with three columns
+    boost::scoped_ptr<VersionedCSVFile> csv(new VersionedCSVFile(testfile_));
+    ASSERT_NO_THROW(csv->addColumn("animal", "2.0", ""));
+    ASSERT_NO_THROW(csv->addColumn("color", "2.0", "grey"));
+    ASSERT_NO_THROW(csv->addColumn("age", "2.0", "0"));
+
+    // Write a file compliant with the current schema version.
+    writeFile("animal,color,age\n"
+              "cat,black,2\n"
+              "lion,yellow,17\n"
+              "dog,brown,5\n");
+
+    // Header should pass validation and allow the open to succeed.
+    ASSERT_NO_THROW(csv->open());
+
+    // For schema current file We should have:
+    // 3 defined columns
+    // 3 columns total found in the header
+    // 3 valid columns found in the header
+    // Minium valid columns wasn't set. (Remember it's optional)
+    EXPECT_EQ(3, csv->getColumnCount());
+    EXPECT_EQ(3, csv->getInputHeaderCount());
+    EXPECT_EQ(3, csv->getValidColumnCount());
+    EXPECT_EQ(0, csv->getMinimumValidColumns());
+
+    // Input schema and current schema should both be  2.0
+    EXPECT_EQ("2.0", csv->getInputSchemaVersion());
+    EXPECT_EQ("2.0", csv->getSchemaVersion());
+
+    // Input Schema State should be CURRENT
+    EXPECT_EQ(VersionedCSVFile::CURRENT, csv->getInputSchemaState());
+    EXPECT_FALSE(csv->needsConversion());
+
+    // First row is correct.
+    CSVRow row;
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("cat", row.readAt(0));
+    EXPECT_EQ("black", row.readAt(1));
+    EXPECT_EQ("2", row.readAt(2));
+
+    // Second row is correct.
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("lion", row.readAt(0));
+    EXPECT_EQ("yellow", row.readAt(1));
+    EXPECT_EQ("17", row.readAt(2));
+
+    // Third row is correct.
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("dog", row.readAt(0));
+    EXPECT_EQ("brown", row.readAt(1));
+    EXPECT_EQ("5", row.readAt(2));
+}
+
+
+// Verifies the basic ability to upgrade valid files.
+// It starts with a version 1.0 file and updates
+// it through two schema evolutions.
+TEST_F(VersionedCSVFileTest, upgradeOlderVersions) {
+
+    // Create version 1.0 schema  CSV file
+    writeFile("animal\n"
+              "cat\n"
+              "lion\n"
+              "dog\n");
+
+    // Create our versioned file, with two columns, one for each
+    // schema version
+    boost::scoped_ptr<VersionedCSVFile> csv(new VersionedCSVFile(testfile_));
+    ASSERT_NO_THROW(csv->addColumn("animal", "1.0", ""));
+    ASSERT_NO_THROW(csv->addColumn("color", "2.0", "blue"));
+
+    // Header should pass validation and allow the open to succeed.
+    ASSERT_NO_THROW(csv->open());
+
+    // We should have:
+    // 2 defined columns
+    // 1 column found in the header
+    // 1 valid column in the header
+    // Minium valid columns wasn't set. (Remember it's optional)
+    EXPECT_EQ(2, csv->getColumnCount());
+    EXPECT_EQ(1, csv->getInputHeaderCount());
+    EXPECT_EQ(1, csv->getValidColumnCount());
+    EXPECT_EQ(0, csv->getMinimumValidColumns());
+
+    // Input schema should be 1.0, while our current schema should be 2.0
+    EXPECT_EQ("1.0", csv->getInputSchemaVersion());
+    EXPECT_EQ("2.0", csv->getSchemaVersion());
+
+    // Input Schema State should be NEEDS_UPGRADE
+    EXPECT_EQ(VersionedCSVFile::NEEDS_UPGRADE, csv->getInputSchemaState());
+    EXPECT_TRUE(csv->needsConversion());
+
+    // First row is correct.
+    CSVRow row;
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("cat", row.readAt(0));
+    EXPECT_EQ("blue", row.readAt(1));
+
+    // Second row is correct.
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("lion", row.readAt(0));
+    EXPECT_EQ("blue", row.readAt(1));
+
+    // Third row is correct.
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("dog", row.readAt(0));
+    EXPECT_EQ("blue", row.readAt(1));
+
+    // Now, let's try to append something to this file.
+    CSVRow row_write(2);
+    row_write.writeAt(0, "bird");
+    row_write.writeAt(1, "yellow");
+    ASSERT_NO_THROW(csv->append(row_write));
+
+    // Close the file
+    ASSERT_NO_THROW(csv->flush());
+    ASSERT_NO_THROW(csv->close());
+
+
+    // Check the the file contents are correct.
+    EXPECT_EQ("animal\n"
+              "cat\n"
+              "lion\n"
+              "dog\n"
+              "bird,yellow\n",
+              readFile());
+
+    // Create a third schema by adding a column
+    ASSERT_NO_THROW(csv->addColumn("age", "3.0", "21"));
+    ASSERT_EQ(3, csv->getColumnCount());
+
+    // Header should pass validation and allow the open to succeed
+    ASSERT_NO_THROW(csv->open());
+
+    // We should have:
+    // 3 defined columns
+    // 1 column found in the header
+    // 1 valid column in the header
+    // Minium valid columns wasn't set. (Remember it's optional)
+    EXPECT_EQ(3, csv->getColumnCount());
+    EXPECT_EQ(1, csv->getInputHeaderCount());
+    EXPECT_EQ(1, csv->getValidColumnCount());
+    EXPECT_EQ(0, csv->getMinimumValidColumns());
+
+    // Make sure schema versions are accurate
+    EXPECT_EQ("1.0", csv->getInputSchemaVersion());
+    EXPECT_EQ("3.0", csv->getSchemaVersion());
+
+    // Input Schema State should be NEEDS_UPGRADE
+    EXPECT_EQ(VersionedCSVFile::NEEDS_UPGRADE, csv->getInputSchemaState());
+    EXPECT_TRUE(csv->needsConversion());
+
+    // First row is correct.
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("cat", row.readAt(0));
+    EXPECT_EQ("blue", row.readAt(1));
+    EXPECT_EQ("21", row.readAt(2));
+
+    // Second row is correct.
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("lion", row.readAt(0));
+    EXPECT_EQ("blue", row.readAt(1));
+    EXPECT_EQ("21", row.readAt(2));
+
+    // Third row is correct.
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("dog", row.readAt(0));
+    EXPECT_EQ("blue", row.readAt(1));
+    EXPECT_EQ("21", row.readAt(2));
+
+    // Fourth row is correct.
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("bird", row.readAt(0));
+    EXPECT_EQ("yellow", row.readAt(1));
+    EXPECT_EQ("21", row.readAt(2));
+}
+
+TEST_F(VersionedCSVFileTest, minimumValidColumn) {
+    // Create version 1.0 schema  CSV file
+    writeFile("animal\n"
+              "cat\n"
+              "lion\n"
+              "dog\n");
+
+    // Create our versioned file, with three columns, one for each
+    // schema version
+    boost::scoped_ptr<VersionedCSVFile> csv(new VersionedCSVFile(testfile_));
+    ASSERT_NO_THROW(csv->addColumn("animal", "1.0", ""));
+    ASSERT_NO_THROW(csv->addColumn("color", "2.0", "blue"));
+    ASSERT_NO_THROW(csv->addColumn("age", "3.0", "21"));
+
+    // Verify we can't set minimum columns with a non-existent column
+    EXPECT_THROW(csv->setMinimumValidColumns("bogus"), VersionedCSVFileError);
+
+    // Set the minimum number of columns to "color"
+    csv->setMinimumValidColumns("color");
+    EXPECT_EQ(2, csv->getMinimumValidColumns());
+
+    // Header validation should fail, too few columns
+    ASSERT_THROW(csv->open(), CSVFileError);
+
+    // Set the minimum number of columns to 1.  File should parse now.
+    csv->setMinimumValidColumns("animal");
+    EXPECT_EQ(1, csv->getMinimumValidColumns());
+    ASSERT_NO_THROW(csv->open());
+
+    // First row is correct.
+    CSVRow row;
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("cat", row.readAt(0));
+    EXPECT_EQ("blue", row.readAt(1));
+    EXPECT_EQ("21", row.readAt(2));
+
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("lion", row.readAt(0));
+    EXPECT_EQ("blue", row.readAt(1));
+    EXPECT_EQ("21", row.readAt(2));
+
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("dog", row.readAt(0));
+    EXPECT_EQ("blue", row.readAt(1));
+    EXPECT_EQ("21", row.readAt(2));
+}
+
+TEST_F(VersionedCSVFileTest, invalidHeaderColumn) {
+
+    // Create our version 2.0 schema file
+    boost::scoped_ptr<VersionedCSVFile> csv(new VersionedCSVFile(testfile_));
+    ASSERT_NO_THROW(csv->addColumn("animal", "1.0", ""));
+    ASSERT_NO_THROW(csv->addColumn("color", "2.0", "blue"));
+
+    // Create a file with the correct number of columns but a wrong column name
+    writeFile("animal,colour\n"
+              "cat,red\n"
+              "lion,green\n");
+
+    // Header validation should fail, we have an invalid column
+    ASSERT_THROW(csv->open(), CSVFileError);
+}
+
+TEST_F(VersionedCSVFileTest, downGrading) {
+    // Create our version 2.0 schema file
+    boost::scoped_ptr<VersionedCSVFile> csv(new VersionedCSVFile(testfile_));
+    ASSERT_NO_THROW(csv->addColumn("animal", "1.0", ""));
+    ASSERT_NO_THROW(csv->addColumn("color", "2.0", "blue"));
+
+    // Create schema 2.0 file PLUS an extra column
+    writeFile("animal,color,age\n"
+              "cat,red,5\n"
+              "lion,green,8\n");
+
+    // Header should validate and file should open.
+    ASSERT_NO_THROW(csv->open());
+
+    // We should have:
+    // 2 defined columns
+    // 3 columns found in the header
+    // 2 valid columns in the header
+    // Minium valid columns wasn't set. (Remember it's optional)
+    EXPECT_EQ(2, csv->getColumnCount());
+    EXPECT_EQ(3, csv->getInputHeaderCount());
+    EXPECT_EQ(2, csv->getValidColumnCount());
+    EXPECT_EQ(0, csv->getMinimumValidColumns());
+
+    // Input schema and current schema should both be 2.0
+    EXPECT_EQ("2.0", csv->getInputSchemaVersion());
+    EXPECT_EQ("2.0", csv->getSchemaVersion());
+
+    // Input Schema State should be NEEDS_DOWNGRADE
+    EXPECT_EQ(VersionedCSVFile::NEEDS_DOWNGRADE, csv->getInputSchemaState());
+    EXPECT_TRUE(csv->needsConversion());
+
+    // First row is correct.
+    CSVRow row;
+    EXPECT_TRUE(csv->next(row));
+    EXPECT_EQ("cat", row.readAt(0));
+    EXPECT_EQ("red", row.readAt(1));
+
+    // No data beyond the second column
+    EXPECT_THROW(row.readAt(2), CSVFileError);
+
+    // Second row is correct.
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("lion", row.readAt(0));
+    EXPECT_EQ("green", row.readAt(1));
+
+    // No data beyond the second column
+    EXPECT_THROW(row.readAt(2), CSVFileError);
+}
+
+
+TEST_F(VersionedCSVFileTest, rowChecking) {
+    // Create version 2.0 schema CSV file with a
+    // - valid header
+    // - row 0 has too many values
+    // - row 1 is valid
+    // - row 3 is too few values
+    writeFile("animal,color\n"
+              "cat,red,bogus_row_value\n"
+              "lion,green\n"
+              "too_few\n");
+
+    // Create our versioned file, with two columns, one for each
+    // schema version
+    boost::scoped_ptr<VersionedCSVFile> csv(new VersionedCSVFile(testfile_));
+    csv->addColumn("animal", "1.0", "");
+    csv->addColumn("color", "2.0", "blue");
+
+    // Header validation should pass, so we can open
+    ASSERT_NO_THROW(csv->open());
+
+    CSVRow row;
+    // First row has too many
+    EXPECT_FALSE(csv->next(row));
+
+    // Second row is valid
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("lion", row.readAt(0));
+    EXPECT_EQ("green", row.readAt(1));
+
+    // Third row has too few
+    EXPECT_FALSE(csv->next(row));
+}
+
+} // end of anonymous namespace

src/lib/util/versioned_csv_file.cc

@@ -0,0 +1,251 @@
+// Copyright (C) 2015 Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <util/versioned_csv_file.h>
+
+namespace isc {
+namespace util {
+
+VersionedCSVFile::VersionedCSVFile(const std::string& filename)
+    : CSVFile(filename), columns_(0), valid_column_count_(0),
+      minimum_valid_columns_(0), input_header_count_(0),
+      input_schema_state_(CURRENT) {
+}
+
+VersionedCSVFile::~VersionedCSVFile() {
+}
+
+void
+VersionedCSVFile::addColumn(const std::string& name,
+                            const std::string& version,
+                            const std::string& default_value) {
+    CSVFile::addColumn(name);
+    columns_.push_back(VersionedColumnPtr(new VersionedColumn(name, version,
+                                                              default_value)));
+}
+
+void
+VersionedCSVFile::setMinimumValidColumns(const std::string& column_name) {
+    int index = getColumnIndex(column_name);
+    if (index <  0) {
+        isc_throw(VersionedCSVFileError,
+                  "setMinimumValidColumns: " << column_name << " is defined");
+    }
+
+    minimum_valid_columns_ = index + 1;
+}
+
+size_t
+VersionedCSVFile::getMinimumValidColumns() const {
+    return (minimum_valid_columns_);
+}
+
+size_t
+VersionedCSVFile::getValidColumnCount() const {
+    return (valid_column_count_);
+}
+
+size_t
+VersionedCSVFile::getInputHeaderCount() const {
+    return (input_header_count_);
+}
+
+void
+VersionedCSVFile::open(const bool seek_to_end) {
+    if (getColumnCount() == 0) {
+        isc_throw(VersionedCSVFileError,
+                  "no schema has been defined, cannot open CSV file :"
+                  << getFilename());
+    }
+
+    CSVFile::open(seek_to_end);
+}
+
+void
+VersionedCSVFile::recreate() {
+    if (getColumnCount() == 0) {
+        isc_throw(VersionedCSVFileError,
+                  "no schema has been defined, cannot create CSV file :"
+                  << getFilename());
+    }
+
+    CSVFile::recreate();
+    // For new files they always match.
+    input_header_count_ = valid_column_count_ = getColumnCount();
+}
+
+VersionedCSVFile::InputSchemaState
+VersionedCSVFile::getInputSchemaState() const {
+    return (input_schema_state_);
+}
+
+bool
+VersionedCSVFile::needsConversion() const {
+    return (input_schema_state_ != CURRENT);
+}
+
+std::string
+VersionedCSVFile::getInputSchemaVersion() const {
+    if (getValidColumnCount() > 0) {
+        return (getVersionedColumn(getValidColumnCount() - 1)->version_);
+    }
+
+    return ("undefined");
+}
+
+std::string
+VersionedCSVFile::getSchemaVersion() const {
+    if (getColumnCount() > 0) {
+        return (getVersionedColumn(getColumnCount() - 1)->version_);
+    }
+
+    return ("undefined");
+}
+
+const VersionedColumnPtr&
+VersionedCSVFile::getVersionedColumn(const size_t index) const {
+    if (index >= getColumnCount()) {
+        isc_throw(isc::OutOfRange, "versioned column index " << index
+                  << " out of range;  CSV file : " << getFilename()
+                  << " only has " << getColumnCount() << " columns ");
+    }
+
+    return (columns_[index]);
+}
+
+bool
+VersionedCSVFile::next(CSVRow& row) {
+    setReadMsg("success");
+    // Use base class to physical read the row, but skip its row
+    // validation
+    CSVFile::next(row, true);
+    if (row == CSVFile::EMPTY_ROW()) {
+        return(true);
+    }
+
+    bool row_valid = true;
+    switch(getInputSchemaState()) {
+        case CURRENT:
+            // All rows must match than the current schema
+            if (row.getValuesCount() != getColumnCount()) {
+                columnCountError(row, "must match current schema");
+                row_valid = false;
+            }
+            break;
+
+        case NEEDS_UPGRADE:
+            // The input header met the minimum column count but
+            // is less than the current schema so:
+            // Rows must not be shorter than the valid column count
+            // and not longer than the current schema
+            if (row.getValuesCount() < getValidColumnCount()) {
+                columnCountError(row, "too few columns to upgrade");
+                row_valid = false;
+            } else if (row.getValuesCount() > getColumnCount()) {
+                columnCountError(row, "too many columns to upgrade");
+                row_valid = false;
+            } else {
+                // Add any missing values
+                for (size_t index = row.getValuesCount();
+                     index < getColumnCount(); ++index) {
+                    row.append(columns_[index]->default_value_);
+                }
+            }
+            break;
+
+        case NEEDS_DOWNGRADE:
+            // The input header exceeded current schema so:
+            // Rows may be as long as input header but not shorter than
+            // the the current schema
+            if (row.getValuesCount() < getColumnCount()) {
+                columnCountError(row, "too few columns to downgrade");
+            } else if (row.getValuesCount() > getInputHeaderCount()) {
+                columnCountError(row, "too many columns to downgrade");
+            } else {
+                // Toss any the extra columns
+                row.trim(row.getValuesCount() - getColumnCount());
+            }
+            break;
+    }
+
+    return (row_valid);
+}
+
+void
+VersionedCSVFile::columnCountError(const CSVRow& row,
+                                  const std::string& reason) {
+    std::ostringstream s;
+    s <<  "Invalid number of columns: "
+      << row.getValuesCount()  << " in row: '" << row
+      << "', file: '" << getFilename() << "' : " << reason;
+      setReadMsg(s.str());
+}
+
+bool
+VersionedCSVFile::validateHeader(const CSVRow& header) {
+    if (getColumnCount() == 0) {
+        isc_throw(VersionedCSVFileError,
+                  "cannot validate header, no schema has been defined");
+    }
+
+    input_header_count_ = header.getValuesCount();
+
+    // Iterate over the number of columns in the header, testing
+    // each against the defined column in the same position.
+    // If there is a mismatch, bail.
+    size_t i = 0;
+    for (  ; i < getInputHeaderCount() && i < getColumnCount(); ++i) {
+        if (getColumnName(i) != header.readAt(i)) {
+            std::ostringstream s;
+            s << " - header contains an invalid column: '"
+              << header.readAt(i) << "'";
+            setReadMsg(s.str());
+            return (false);
+        }
+    }
+
+    // If we found too few valid columns, then we cannot convert this
+    // file.  It's too old, too corrupt, or not a Kea file.
+    if (i < getMinimumValidColumns()) {
+        std::ostringstream s;
+        s << " - header has only " << i << " valid column(s), "
+          << "it must have at least " << getMinimumValidColumns();
+        setReadMsg(s.str());
+        return (false);
+    }
+
+    // Remember the number of valid columns we found.  When this number
+    // is less than the number of defined columns, then we have an older
+    // version of the lease file.  We'll need this value to validate
+    // and upgrade data rows.
+    valid_column_count_ = i;
+
+    if (getValidColumnCount() < getColumnCount()) {
+        input_schema_state_ = NEEDS_UPGRADE;
+    } else if (getInputHeaderCount() > getColumnCount()) {
+        // If there are more values in the header than defined columns
+        // then, we'll drop the extra.  This allows someone to attempt to
+        // downgrade if need be.
+        input_schema_state_ = NEEDS_DOWNGRADE;
+        std::ostringstream s;
+        s << " - header has " << getInputHeaderCount() - getColumnCount()
+          << " extra column(s), these will be ignored";
+        setReadMsg(s.str());
+    }
+
+    return (true);
+}
+
+} // end of isc::util namespace
+} // end of isc namespace

src/lib/util/versioned_csv_file.h

@@ -0,0 +1,326 @@
+// Copyright (C) 2015 Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#ifndef VERSIONED_CSV_FILE_H
+#define VERSIONED_CSV_FILE_H
+
+#include <util/csv_file.h>
+
+namespace isc {
+namespace util {
+
+/// @brief Exception thrown when an error occurs during CSV file processing.
+class VersionedCSVFileError : public Exception {
+public:
+    VersionedCSVFileError(const char* file, size_t line, const char* what) :
+        isc::Exception(file, line, what) { };
+};
+
+/// @brief Contains the metadata for a single column in a file.
+class VersionedColumn {
+public:
+    /// @brief Constructor
+    ///
+    /// @param name Name of the column.
+    /// @param version Text representation of the schema version in which
+    /// this column first appeared.
+    /// @param default_value The value the column should be assigned if it
+    /// is not present in a data row. It defaults to an empty string, ""
+    VersionedColumn(const std::string& name, const std::string& version,
+               const std::string& default_value = "")
+        : name_(name), version_(version), default_value_(default_value) {
+    };
+
+    /// @brief Destructor
+    virtual ~VersionedColumn(){};
+
+    /// @brief Name of the column.
+    std::string name_;
+
+    /// @brief Text representation of the schema version in which
+    /// this column first appeared.
+    std::string version_;
+
+    /// @brief default_value The value the column should be assigned if it
+    /// is not present in a data row.
+    std::string default_value_;
+};
+
+/// @brief Defines a smart pointer to VersionedColumn
+typedef boost::shared_ptr<VersionedColumn> VersionedColumnPtr;
+
+/// @brief Implements a CSV file that supports multiple versions of
+/// the file's "schema".  This allows files with older schemas to be
+/// upgraded to newer schemas as they are being read.  The file's schema
+/// is defined through a list of column descriptors, or @ref
+/// isc::util::VersionedColumn(s). Each descriptor contains metadata describing
+/// the column, consisting of the column's name, the version label in which
+/// the column was added to the schema, and a default value to be used if the
+/// column is missing from the file.  Note that the column descriptors are
+/// defined in the order they occur in the file, when reading a row from left
+/// to right.  This also assumes that when new version of the schema evolves,
+/// all new columns are added at the end of the row.  In other words, the
+/// order of the columns reflects not only the order in which they occur
+/// in a row but also the order they were added to the schema.  Conceptually,
+/// the entire list of columns defined constitutes the current schema.  Earlier
+/// schema versions are therefore subsets of this list.   Creating the schema
+/// is done by calling VersionedCSVfile::addColumn() for each column.  Note
+/// that the schema must be defined prior to opening the file.
+///
+/// The first row of the file is always the header row and is a comma-separated
+/// list of the names of the column in the file.  This row is used when
+/// opening the file via @ref VersionedCSVFile::open(), to identify its schema
+/// version so that it may be be read correctly.  This is done by comparing
+/// the column found in the header to the columns defined in the schema. The
+/// columns must match both by name and the order in which they occur.
+///
+/// -# If there are fewer columns in the header than in the schema, the file
+/// is presumed to be an earlier schema version and will be upgraded as it is
+/// read.  There is an ability to mark a specific column as being the minimum
+/// column which must be present, see @ref VersionedCSVFile::
+/// setMinimumValidColumns().  If the header columns do not match up to this
+/// minimum column, the file is presumed to be too old to upgrade and the
+/// open will fail.  A valid, upgradable file will have an input schema
+/// state of VersionedCSVFile::NEEDS_UPGRADE.
+///
+/// -# If there is a mismatch between a found column name and the column name
+/// defined for that position in the row, the file is presumed to be invalid
+/// and the open will fail.
+///
+/// -# If the content of the header matches exactly the columns defined in
+/// the schema, the file is considered to match the schema exactly and the
+/// input schema state will VersionedCSVFile::CURRENT.
+///
+/// -# If there columns in the header beyond all of the columns defined in
+/// the schema (i.e the schema is a subset of the header), then the file
+/// is presumed to be from a newer version of Kea and can be downgraded. The
+/// input schema state fo the file will be set to
+/// VersionedCSVFile::NEEDS_DOWNGRADE.
+///
+/// After successfully opening a file,  rows are read one at a time via
+/// @ref VersionedCSVFile::next() and handled according to the input schema
+/// state.   Each data row is expected to have at least the same number of
+/// columns as were found in the header. Any row which as fewer values is
+/// discarded as invalid.  Similarly, any row which is found to have more
+/// values than were found in the header is discarded as invalid.
+///
+/// When upgrading a row, the values for each missing column is filled in
+/// with the default value specified by that column's descriptor.  When
+/// downgrading a row, extraneous values are dropped from the row.
+///
+/// It is important to note that upgrading or downgrading a file does NOT
+/// alter the physical file itself.  Rather the conversion occurs after the
+/// raw data has been read but before it is passed to caller.
+///
+/// Also note that there is currently no support for writing out a file in
+/// anything other than the current schema.
+class VersionedCSVFile : public CSVFile {
+public:
+
+    /// @brief Possible input file schema states.
+    /// Used to categorize the input file's schema, relative to the defined
+    /// schema.
+    enum InputSchemaState {
+        CURRENT,
+        NEEDS_UPGRADE,
+        NEEDS_DOWNGRADE
+    };
+
+    /// @brief Constructor.
+    ///
+    /// @param filename CSV file name.
+    VersionedCSVFile(const std::string& filename);
+
+    /// @brief Destructor
+    virtual ~VersionedCSVFile();
+
+    /// @brief Adds metadata for a single column to the schema.
+    ///
+    /// This method appends a new column description to the file's schema.
+    /// Note this does not cause anything to be written to the physical file.
+    /// The name of the column will be placed in the CSV header when new file
+    /// is created by calling @c recreate or @c open function.
+    ///
+    /// @param name Name of the column.
+    /// @param version  Text representation of the schema version in which
+    /// this column first appeared.
+    /// @param default_value value the missing column should be given during
+    /// an upgrade.  It defaults to an empty string, ""
+    ///
+    /// @throw CSVFileError if a column with the specified name exists.
+    void addColumn(const std::string& col_name, const std::string& version,
+                   const std::string& default_value = "");
+
+    /// @brief Sets the minimum number of valid columns based on a given column
+    ///
+    /// @param column_name Name of the column which positionally represents
+    /// the minimum columns which must be present in a file and to be
+    /// considered valid.
+    void setMinimumValidColumns(const std::string& column_name);
+
+    /// @brief Returns the minimum number of columns which must be present
+    /// for the file to be considered valid.
+    size_t getMinimumValidColumns() const;
+
+    /// @brief Returns the number of columns found in the input header
+    size_t getInputHeaderCount() const;
+
+    /// @brief Returns the number of valid columns found in the header
+    /// For newly created files this will always match the number of defined
+    /// columns (i.e. getColumnCount()).  For existing files, this will be
+    /// the number of columns in the header that match the defined columnns.
+    /// When this number is less than getColumnCount() it means the input file
+    /// is from an earlier schema.  This value is zero until the file has
+    /// been opened.
+    size_t getValidColumnCount() const;
+
+    /// @brief Opens existing file or creates a new one.
+    ///
+    /// This function will try to open existing file if this file has size
+    /// greater than 0. If the file doesn't exist or has size of 0, the
+    /// file is recreated. If the existing file has been opened, the header
+    /// is parsed and and validated against the schema.
+    /// By default, the data pointer in the file is set to the beginning of
+    /// the first data row. In order to retrieve the row contents the @c next
+    /// function should be called. If a @c seek_to_end parameter is set to
+    /// true, the file will be opened and the internal pointer will be set
+    /// to the end of file.
+    ///
+    /// @param seek_to_end A boolean value which indicates if the intput and
+    /// output file pointer should be set at the end of file.
+    ///
+    /// @throw VersionedCSVFileError if schema has not been defined,
+    /// CSVFileError when IO operation fails, or header fails to validate.
+    virtual void open(const bool seek_to_end = false);
+
+    /// @brief Creates a new CSV file.
+    ///
+    /// The file creation will fail if there are no columns specified.
+    /// Otherwise, this function will write the header to the file.
+    /// In order to write rows to opened file, the @c append function
+    /// should be called.
+    ///
+    /// @throw VersionedCSVFileError if schema has not been defined
+    /// CSVFileError if an IO operation fails
+    virtual void recreate();
+
+    /// @brief Reads next row from the file file.
+    ///
+    /// This function will return the @c CSVRow object representing a
+    /// parsed row if parsing is successful. If the end of file has been
+    /// reached, the empty row is returned (a row containing no values).
+    ///
+    /// 1. If the row has fewer values than were found in the header it is
+    /// discarded as invalid.
+    ///
+    /// 2. If the row is found to have more values than are defined in the
+    /// schema it is discarded as invalid
+    ///
+    /// When a valid row has fewer than the defined number of columns, the
+    /// values for each missing column is filled in with the default value
+    /// specified by that column's descriptor.
+    ///
+    /// @param [out] row Object receiving the parsed CSV file.
+    /// @param skip_validation Do not perform validation.
+    ///
+    /// @return true if row has been read and validated; false if validation
+    /// failed.
+    bool next(CSVRow& row);
+
+    /// @brief Returns the schema version of the physical file
+    ///
+    /// @return text version of the schema found or string "undefined" if the
+    /// file has not been opened
+    std::string getInputSchemaVersion() const;
+
+    /// @brief text version of current schema supported by the file's metadata
+    ///
+    /// @return text version info assigned to the last column in the list of
+    /// defined column, or the string "undefined" if no columns have been
+    /// defined.
+    std::string getSchemaVersion() const;
+
+    /// @brief Fetch the column descriptor for a given index
+    ///
+    /// @param index index within the list of columns of the desired column
+    /// @return a pointer to the VersionedColumn at the given index
+    /// @trow OutOfRange exception if the index is invalid
+    const VersionedColumnPtr& getVersionedColumn(const size_t index) const;
+
+    /// @brief Fetches the state of the input file's schema
+    ///
+    /// Reflects that state of the input file's schema relative to the
+    /// defined schema as a enum, InputSchemaState.
+    ///
+    /// @return VersionedCSVFile::CURRENT if the input file schema matches
+    /// the defined schema, NEEDS_UPGRADE if the input file schema is older,
+    /// and NEEDS_DOWNGRADE if it is newer
+    enum InputSchemaState getInputSchemaState() const;
+
+    /// @brief Returns true if the input file schema state is not CURRENT
+    bool needsConversion() const;
+
+protected:
+
+    /// @brief Validates the header of a VersionedCSVFile
+    ///
+    /// This function is called internally when the reading in an existing
+    /// file.  It parses the header row of the file, comparing each value
+    /// in succession against the defined list of columns.  If the header
+    /// contains too few matching columns (i.e. less than @c
+    /// minimum_valid_columns_) or too many (more than the number of defined
+    /// columns), the file is presumed to be either too old, too new, or too
+    /// corrupt to process.  Otherwise it retains the number of valid columns
+    /// found and deems the header valid.
+    ///
+    /// @param header A row holding a header.
+    /// @return true if header matches the columns; false otherwise.
+    virtual bool validateHeader(const CSVRow& header);
+
+    /// @brief Convenience method for adding an error message
+    ///
+    /// Constructs an error message indicating that the number of columns
+    /// in a given row are wrong and why, then adds it readMsg.
+    ///
+    /// @param row The row in error
+    /// @param reason An explanation as to why the row column count is wrong
+    void columnCountError(const CSVRow& row, const std::string& reason);
+
+private:
+    /// @brief Holds the collection of column descriptors
+    std::vector<VersionedColumnPtr> columns_;
+
+    /// @brief Number of valid columns present in input file. If this is less
+    /// than the number of columns defined, this implies the input file is
+    /// from an earlier version of the code.
+    size_t valid_column_count_;
+
+    /// @brief Minimum number of valid columns an input file must contain.
+    /// If an input file does not meet this number it cannot be upgraded.
+    size_t minimum_valid_columns_;
+
+    /// @brief The number of columns found in the input header row
+    /// This value represent the number of columns present, in the header
+    /// valid or otherwise.
+    size_t input_header_count_;
+
+    /// @brief The state of the input schema in relation to the current schema
+    enum InputSchemaState input_schema_state_;
+};
+
+
+} // namespace isc::util
+} // namespace isc
+
+#endif // VERSIONED_CSV_FILE_H