[3601] Added isc::util::VersionedCSVFile

src/lib/util/versioned_csv_file.h
src/lib/util/versioned_csv_file.cc
    New files which implement VersionedCSVFile, CSV file which can
    support mulitple schema versions

src/lib/util/tests/versioned_csv_file_unittest.h
src/lib/util/tests/versioned_csv_file_unittest.cc
    new files for Unit tests for VersionedCSVFile

src/lib/util/Makefile.am
    added new files

src/lib/util/csv_file.cc
    includes read error message if header fails to validate

src/lib/util/csv_file.h
    removed @todo for 3626, no longer applicable

src/lib/util/tests/Makefile.am
    added versioned_csv_file_unittest.cc

src/lib/util/Makefile.am

@@ -23,6 +23,7 @@ libkea_util_la_SOURCES += range_utilities.h
 libkea_util_la_SOURCES += signal_set.cc signal_set.h
 libkea_util_la_SOURCES += stopwatch.cc stopwatch.h
 libkea_util_la_SOURCES += stopwatch_impl.cc stopwatch_impl.h
+libkea_util_la_SOURCES += versioned_csv_file.h versioned_csv_file.cc
 libkea_util_la_SOURCES += watch_socket.cc watch_socket.h
 libkea_util_la_SOURCES += encode/base16_from_binary.h
 libkea_util_la_SOURCES += encode/base32hex.h encode/base64.h

src/lib/util/csv_file.cc

@@ -296,9 +296,9 @@ CSVFile::open(const bool seek_to_end) {
 
             // Check the header against the columns specified for the CSV file.
             if (!validateHeader(header)) {
-
                 isc_throw(CSVFileError, "invalid header '" << header
-                          << "' in CSV file '" << filename_ << "'");
+                          << "' in CSV file '" << filename_ << "': "
+                          << getReadMsg());
             }
 
             // Everything is good, so if we haven't added any columns yet,

src/lib/util/csv_file.h

@@ -469,8 +469,6 @@ protected:
     /// This function is called internally by @ref CSVFile::open. Derived classes
     /// may add extra validation steps.
     ///
-    /// @todo There should be a support for optional columns (see ticket #3626).
-    ///
     /// @param header A row holding a header.
     /// @return true if header matches the columns; false otherwise.
     virtual bool validateHeader(const CSVRow& header);

src/lib/util/tests/Makefile.am

@@ -50,6 +50,7 @@ run_unittests_SOURCES += time_utilities_unittest.cc
 run_unittests_SOURCES += range_utilities_unittest.cc
 run_unittests_SOURCES += signal_set_unittest.cc
 run_unittests_SOURCES += stopwatch_unittest.cc
+run_unittests_SOURCES += versioned_csv_file_unittest.cc
 run_unittests_SOURCES += watch_socket_unittests.cc
 
 

src/lib/util/tests/versioned_csv_file_unittest.cc

@@ -0,0 +1,375 @@
+// Copyright (C) 2015 Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <config.h>
+#include <util/versioned_csv_file.h>
+#include <boost/scoped_ptr.hpp>
+#include <gtest/gtest.h>
+#include <fstream>
+#include <sstream>
+#include <string>
+
+#include <boost/algorithm/string/classification.hpp>
+#include <boost/algorithm/string/constants.hpp>
+#include <boost/algorithm/string/split.hpp>
+
+namespace {
+
+using namespace isc::util;
+
+/// @brief Test fixture class for testing operations on VersionedCSVFile.
+///
+/// It implements basic operations on files, such as reading writing
+/// file removal and checking presence of the file. This is used by
+/// unit tests to verify correctness of the file created by the
+/// CSVFile class.
+class VersionedCSVFileTest : public ::testing::Test {
+public:
+
+    /// @brief Constructor.
+    ///
+    /// Sets the path to the CSV file used throughout the tests.
+    /// The name of the file is test.csv and it is located in the
+    /// current build folder.
+    ///
+    /// It also deletes any dangling files after previous tests.
+    VersionedCSVFileTest();
+
+    /// @brief Destructor.
+    ///
+    /// Deletes the test CSV file if any.
+    virtual ~VersionedCSVFileTest();
+
+    /// @brief Prepends the absolute path to the file specified
+    /// as an argument.
+    ///
+    /// @param filename Name of the file.
+    /// @return Absolute path to the test file.
+    static std::string absolutePath(const std::string& filename);
+
+    /// @brief Check if test file exists on disk.
+    bool exists() const;
+
+    /// @brief Reads whole CSV file.
+    ///
+    /// @return Contents of the file.
+    std::string readFile() const;
+
+    /// @brief Removes existing file (if any).
+    int removeFile() const;
+
+    /// @brief Creates file with contents.
+    ///
+    /// @param contents Contents of the file.
+    void writeFile(const std::string& contents) const;
+
+    /// @brief Absolute path to the file used in the tests.
+    std::string testfile_;
+
+};
+
+VersionedCSVFileTest::VersionedCSVFileTest()
+    : testfile_(absolutePath("test.csv")) {
+    static_cast<void>(removeFile());
+}
+
+VersionedCSVFileTest::~VersionedCSVFileTest() {
+    static_cast<void>(removeFile());
+}
+
+std::string
+VersionedCSVFileTest::absolutePath(const std::string& filename) {
+    std::ostringstream s;
+    s << TEST_DATA_BUILDDIR << "/" << filename;
+    return (s.str());
+}
+
+bool
+VersionedCSVFileTest::exists() const {
+    std::ifstream fs(testfile_.c_str());
+    bool ok = fs.good();
+    fs.close();
+    return (ok);
+}
+
+std::string
+VersionedCSVFileTest::readFile() const {
+    std::ifstream fs(testfile_.c_str());
+    if (!fs.is_open()) {
+        return ("");
+    }
+    std::string contents((std::istreambuf_iterator<char>(fs)),
+                         std::istreambuf_iterator<char>());
+    fs.close();
+    return (contents);
+}
+
+int
+VersionedCSVFileTest::removeFile() const {
+    return (remove(testfile_.c_str()));
+}
+
+void
+VersionedCSVFileTest::writeFile(const std::string& contents) const {
+    std::ofstream fs(testfile_.c_str(), std::ofstream::out);
+    if (fs.is_open()) {
+        fs << contents;
+        fs.close();
+    }
+}
+
+// This test checks that the function which is used to add columns of the
+// CSV file works as expected.
+TEST_F(VersionedCSVFileTest, addColumn) {
+    boost::scoped_ptr<VersionedCSVFile> csv(new VersionedCSVFile(testfile_));
+
+    // Verify that we're not allowed to open it without the schema
+    ASSERT_THROW(csv->open(), VersionedCSVFileError);
+
+    // Add two columns.
+    ASSERT_NO_THROW(csv->addColumn("animal", "1.0", ""));
+    ASSERT_NO_THROW(csv->addColumn("color", "2.0", "blue"));
+
+    // Make sure we can't add duplicates.
+    EXPECT_THROW(csv->addColumn("animal", "1.0", ""), CSVFileError);
+    EXPECT_THROW(csv->addColumn("color", "2.0", "blue"), CSVFileError);
+
+    // But we should still be able to add unique columns.
+    EXPECT_NO_THROW(csv->addColumn("age", "3.0", "21"));
+
+    // Assert that the file is opened, because the rest of the test relies
+    // on this.
+    ASSERT_NO_THROW(csv->recreate());
+    ASSERT_TRUE(exists());
+
+    // Make sure we can't add columns (even unique) when the file is open.
+    ASSERT_THROW(csv->addColumn("zoo", "3.0", ""), CSVFileError);
+
+    // Close the file.
+    ASSERT_NO_THROW(csv->close());
+    // And check that now it is possible to add the column.
+    EXPECT_NO_THROW(csv->addColumn("zoo", "3.0", ""));
+}
+
+// Verifies the basic ability to upgrade valid files. 
+// It starts with a version 1.0 file and updates
+// it through two schema evolutions.
+TEST_F(VersionedCSVFileTest, upgradeOlderVersions) {
+
+    // Create version 1.0 schema  CSV file 
+    writeFile("animal\n"
+              "cat\n"
+              "lion\n"
+              "dog\n");
+
+    // Create our versioned file, with two columns, one for each
+    // schema version
+    boost::scoped_ptr<VersionedCSVFile> csv(new VersionedCSVFile(testfile_));
+    ASSERT_NO_THROW(csv->addColumn("animal", "1.0", ""));
+    ASSERT_NO_THROW(csv->addColumn("color", "2.0", "blue"));
+
+    // Header should pass validation and allow the open to succeed.
+    ASSERT_NO_THROW(csv->open());
+
+    // First row is correct.
+    CSVRow row;
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("cat", row.readAt(0));
+    EXPECT_EQ("blue", row.readAt(1));
+
+    // Second row is correct.
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("lion", row.readAt(0));
+    EXPECT_EQ("blue", row.readAt(1));
+
+    // Third row is correct.
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("dog", row.readAt(0));
+    EXPECT_EQ("blue", row.readAt(1));
+
+    // Now, let's try to append something to this file.
+    CSVRow row_write(2);
+    row_write.writeAt(0, "bird");
+    row_write.writeAt(1, "yellow");
+    ASSERT_NO_THROW(csv->append(row_write));
+
+    // Close the file
+    ASSERT_NO_THROW(csv->flush());
+    ASSERT_NO_THROW(csv->close());
+
+
+    // Check the the file contents are correct.
+    EXPECT_EQ("animal\n"
+              "cat\n"
+              "lion\n"
+              "dog\n"
+              "bird,yellow\n",
+              readFile());
+
+    // Create a third schema by adding a column
+    ASSERT_NO_THROW(csv->addColumn("age", "3.0", "21"));
+    ASSERT_EQ(3, csv->getColumnCount());
+
+    // Header should pass validation and allow the open to succeed
+    ASSERT_NO_THROW(csv->open());
+    ASSERT_EQ(3, csv->getColumnCount());
+
+    // First row is correct.
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("cat", row.readAt(0));
+    EXPECT_EQ("blue", row.readAt(1));
+    EXPECT_EQ("21", row.readAt(2));
+
+    // Second row is correct.
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("lion", row.readAt(0));
+    EXPECT_EQ("blue", row.readAt(1));
+    EXPECT_EQ("21", row.readAt(2));
+
+    // Third row is correct.
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("dog", row.readAt(0));
+    EXPECT_EQ("blue", row.readAt(1));
+    EXPECT_EQ("21", row.readAt(2));
+
+    ASSERT_EQ(3, csv->getColumnCount());
+
+    // Fourth row is correct.
+    if (!csv->next(row)) {
+        std::cout << "row error is : " << 
+           csv->getReadMsg() << std::endl; 
+    
+    }
+
+    EXPECT_EQ("bird", row.readAt(0));
+    EXPECT_EQ("yellow", row.readAt(1));
+    EXPECT_EQ("21", row.readAt(2));
+}
+
+TEST_F(VersionedCSVFileTest, minimumValidColumn) {
+    // Create version 1.0 schema  CSV file 
+    writeFile("animal\n"
+              "cat\n"
+              "lion\n"
+              "dog\n");
+
+    // Create our versioned file, with three columns, one for each
+    // schema version
+    boost::scoped_ptr<VersionedCSVFile> csv(new VersionedCSVFile(testfile_));
+    ASSERT_NO_THROW(csv->addColumn("animal", "1.0", ""));
+    ASSERT_NO_THROW(csv->addColumn("color", "2.0", "blue"));
+    ASSERT_NO_THROW(csv->addColumn("age", "3.0", "21"));
+
+    // Verify we can't set minimum columns with a non-existant column
+    EXPECT_THROW(csv->setMinimumValidColumns("bogus"), VersionedCSVFileError);
+
+    // Set the minimum number of columns to "color"
+    csv->setMinimumValidColumns("color");
+    EXPECT_EQ(2, csv->getMinimumValidColumns());
+
+    // Header validation should fail, too few columns
+    ASSERT_THROW(csv->open(), CSVFileError);
+
+    // Set the minimum number of columns to 1.  File should parse now.
+    csv->setMinimumValidColumns("animal");
+    EXPECT_EQ(1, csv->getMinimumValidColumns());
+    ASSERT_NO_THROW(csv->open());
+
+    // First row is correct.
+    CSVRow row;
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("cat", row.readAt(0));
+    EXPECT_EQ("blue", row.readAt(1));
+    EXPECT_EQ("21", row.readAt(2));
+
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("lion", row.readAt(0));
+    EXPECT_EQ("blue", row.readAt(1));
+    EXPECT_EQ("21", row.readAt(2));
+
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("dog", row.readAt(0));
+    EXPECT_EQ("blue", row.readAt(1));
+    EXPECT_EQ("21", row.readAt(2));
+}
+
+TEST_F(VersionedCSVFileTest, invalidHeaderColumn) {
+
+    // Create version 2.0 schema  CSV file 
+    writeFile("animal,colour\n"
+              "cat,red\n"
+              "lion,green\n");
+
+    // Create our versioned file, with three columns, one for each
+    // schema version
+    boost::scoped_ptr<VersionedCSVFile> csv(new VersionedCSVFile(testfile_));
+    ASSERT_NO_THROW(csv->addColumn("animal", "1.0", ""));
+    ASSERT_NO_THROW(csv->addColumn("color", "2.0", "blue"));
+
+    // Header validation should fail, we have an invalid column
+    ASSERT_THROW(csv->open(), CSVFileError);
+}
+
+TEST_F(VersionedCSVFileTest, tooManyHeaderColumns) {
+
+    // Create version 2.0 schema  CSV file 
+    writeFile("animal,color,age\n,"
+              "cat,red\n"
+              "lion,green\n");
+
+    // Create our versioned file, with three columns, one for each
+    // schema version
+    boost::scoped_ptr<VersionedCSVFile> csv(new VersionedCSVFile(testfile_));
+    ASSERT_NO_THROW(csv->addColumn("animal", "1.0", ""));
+    ASSERT_NO_THROW(csv->addColumn("color", "2.0", "blue"));
+
+    // Header validation should fail, we have too many columns 
+    ASSERT_THROW(csv->open(), CSVFileError);
+}
+
+
+TEST_F(VersionedCSVFileTest, rowChecking) {
+    // Create version 2.0 schema CSV file with a
+    // - valid header 
+    // - row 0 has too many values
+    // - row 1 is valid
+    // - row 3 is too few values
+    writeFile("animal,color\n"
+              "cat,red,bogus_row_value\n"
+              "lion,green\n"
+              "too_few\n");
+
+    // Create our versioned file, with two columns, one for each
+    // schema version
+    boost::scoped_ptr<VersionedCSVFile> csv(new VersionedCSVFile(testfile_));
+    csv->addColumn("animal", "1.0", "");
+    csv->addColumn("color", "2.0", "blue");
+
+    // Header validation should pass, so we can open
+    ASSERT_NO_THROW(csv->open());
+
+    CSVRow row;
+    // First row has too many
+    EXPECT_FALSE(csv->next(row));
+
+    // Second row is valid
+    ASSERT_TRUE(csv->next(row));
+    EXPECT_EQ("lion", row.readAt(0));
+    EXPECT_EQ("green", row.readAt(1));
+
+    // Third row has too few
+    EXPECT_FALSE(csv->next(row));
+}
+
+} // end of anonymous namespace

src/lib/util/versioned_csv_file.cc

@@ -0,0 +1,148 @@
+// Copyright (C) 2015 Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <util/versioned_csv_file.h>
+
+namespace isc {
+namespace util {
+
+VersionedCSVFile::VersionedCSVFile(const std::string& filename)
+    : CSVFile(filename), columns_(0), valid_column_count_(0),
+      minimum_valid_columns_(0) {
+}
+
+VersionedCSVFile::~VersionedCSVFile() {
+}
+
+void
+VersionedCSVFile::addColumn(const std::string& name,
+                            const std::string& version,
+                            const std::string& default_value) {
+    CSVFile::addColumn(name);
+    columns_.push_back(VersionedColumnPtr(new VersionedColumn(name, version,
+                                                              default_value)));
+}
+
+void
+VersionedCSVFile::setMinimumValidColumns(const std::string& column_name) {
+    int index = getColumnIndex(column_name);
+    if (index <  0) {
+        isc_throw(VersionedCSVFileError,
+                  "setMinimumValidColumns: " << column_name << " is defined");
+    }
+
+    minimum_valid_columns_ = index + 1;
+}
+
+size_t
+VersionedCSVFile::getMinimumValidColumns() {
+    return (minimum_valid_columns_);
+}
+
+void
+VersionedCSVFile::open(const bool seek_to_end) {
+    if (getColumnCount() == 0) {
+        isc_throw(VersionedCSVFileError,
+                  "no schema has been defined, cannot open file :"
+                  << getFilename());
+    }
+
+    CSVFile::open(seek_to_end);
+}
+
+bool
+VersionedCSVFile::next(CSVRow& row) {
+    CSVFile::next(row, true);
+    if (row == CSVFile::EMPTY_ROW()) {
+        return(true);
+    }
+
+    // If we're upgrading, valid_column_count_ will be less than
+    // defined column count.  If not they're the equal.  Either way
+    // each data row must have valid_column_count_ values or its
+    // an invalid row.
+    if (row.getValuesCount() < valid_column_count_) {
+        std::ostringstream s;
+        s << "the size of the row '" << row << "' has too few valid columns "
+          << valid_column_count_ << "' of the CSV file '"
+          << getFilename() << "'";
+        setReadMsg(s.str());
+        return (false);
+    }
+
+    // If we're upgrading, we need to add in any missing values
+    for (size_t index = row.getValuesCount(); index < getColumnCount();
+         ++index) {
+        row.append(columns_[index]->default_value_);
+    }
+
+    return (CSVFile::validate(row));
+}
+
+bool
+VersionedCSVFile::validateHeader(const CSVRow& header) {
+    // @todo does this ever make sense? What would be the point of a versioned
+    // file that has no defined columns?
+    if (getColumnCount() == 0) {
+        return (true);
+    }
+
+    // If there are more values in the header than defined columns
+    // then the lease file must be from a newer version, so bail out.
+    // @todo - we may wish to remove this constraint as it prohibits one
+    // from usig a newer schema file with older schema code.
+    if (header.getValuesCount() > getColumnCount()) {
+        std::ostringstream s;
+        s << " - header has " << header.getValuesCount()  << " column(s), "
+          << "it should not have more than " << getColumnCount();
+
+        setReadMsg(s.str());
+        return (false);
+    }
+
+    // Iterate over the number of columns in the header, testing
+    // each against the defined column in the same position.
+    // If there is a mismatch, bail.
+    size_t i = 0;
+    for (  ; i < header.getValuesCount(); ++i) {
+        if (getColumnName(i) != header.readAt(i)) {
+            std::ostringstream s;
+            s << " - header contains an invalid column: '"
+              << header.readAt(i) << "'";
+            setReadMsg(s.str());
+            return (false);
+        }
+    }
+
+    // If we found too few valid columns, then we cannot convert this
+    // file.  It's too old, too corrupt, or not a Kea file.
+    if (i < minimum_valid_columns_) {
+        std::ostringstream s;
+        s << " - header has only " << i << " valid column(s), "
+          << "it must have at least " << minimum_valid_columns_;
+        setReadMsg(s.str());
+        return (false);
+    }
+
+    // Remember the number of valid columns we found.  When this number
+    // is less than the number of defined columns, then we have an older
+    // version of the lease file.  We'll need this value to validate
+    // and upgrade data rows.
+    valid_column_count_ = i;
+
+    return (true);
+}
+
+} // end of isc::util namespace
+} // end of isc namespace

src/lib/util/versioned_csv_file.h

@@ -0,0 +1,235 @@
+// Copyright (C) 2015 Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#ifndef VERSIONED_CSV_FILE_H
+#define VERSIONED_CSV_FILE_H
+
+#include <util/csv_file.h>
+
+namespace isc {
+namespace util {
+
+/// @brief Exception thrown when an error occurs during CSV file processing.
+class VersionedCSVFileError : public Exception {
+public:
+    VersionedCSVFileError(const char* file, size_t line, const char* what) :
+        isc::Exception(file, line, what) { };
+};
+
+/// @brief Contains the metadata for a single column in a file.
+class VersionedColumn {
+public:
+    /// @brief Constructor
+    ///
+    /// @param name Name of the column.
+    /// @param version Text representation of the schema version in which
+    /// this column first appeared.
+    /// @param default_value The value the column should be assigned if it
+    /// is not present in a data row. It defaults to an empty string, ""
+    VersionedColumn(const std::string& name, const std::string& version,
+               const std::string& default_value = "")
+        : name_(name), version_(version), default_value_(default_value) {
+    };
+
+    /// @brief Destructor
+    virtual ~VersionedColumn(){};
+
+    /// @brief Name of the column.
+    std::string name_;
+
+    /// @brief Text representation of the schema version in which
+    /// this column first appeared.
+    std::string version_;
+
+    /// @brief default_value The value the column should be assigned if it
+    /// is not present in a data row.
+    std::string default_value_;
+};
+
+/// @brief Defines a smart pointer to VersionedColumn
+typedef boost::shared_ptr<VersionedColumn> VersionedColumnPtr;
+
+/// @brief Implements a CSV file that supports multiple versions of
+/// the file's "schema".  This allows files with older schemas to be
+/// upgraded to newer schemas as they are being read.  The file's schema
+/// is defined through a list of column descriptors, or @ref
+/// isc::util::VersionedColumn(s). Each descriptor contains metadata describing
+/// the column, consisting of the column's name, the version label in which
+/// the column was added to the schema, and a default value to be used if the
+/// column is missing from the file.  Note that the column descriptors are
+/// defined in the order they occur in the file, when reading a row from left
+/// to right.  This also assumes that when new version of the schema evolves,
+/// all new columns are added at the end of the row.  In other words, the
+/// order of the columns reflects not only the order in which they occur
+/// in a row but also the order they were added to the schema.  Conceptually,
+/// the entire list of columns defined constitutes the current schema.  Earlier
+/// schema versions are therefore subsets of this list.   Creating the schema
+/// is done by calling VersionedCSVfile::addColumn() for each column.  Note
+/// that the schema must be defined prior to opening the file.
+///
+/// The first row of the file is always the header row and is a comma-separated
+/// list of the names of the column in the file.  This row is used when
+/// opening the file via @ref VersionedCSVFile::open(), to identify its schema
+/// version so that it may be be read correctly.  This is done by comparing
+/// the column found in the header to the columns defined in the schema. The
+/// columns must match both by name and the order in which they occur.
+///
+/// 1. If there are fewer columns in the header than in the schema, the file
+/// is presumed to be an earlier schema version and will be upgraded as it is
+/// read.  There is an ability to mark a specific column as being the minimum
+/// column which must be present, see @ref VersionedCSVFile::
+/// setMinimumValidColumns().  If the header does contain match up to this
+/// minimum column, the file is presumed to be too old to upgrade and the
+/// open will fail.
+///
+/// 2. If there is a mismatch between a found column name and the column name
+/// defined for that position in the row, the file is presumed to be invalid
+/// and the open will fail.
+///
+/// After successfully opening a file,  rows are read one at a time via
+/// @ref VersionedCSVFile::next().  Each data row is expected to have at least
+/// the same number of columns as were found in the header. Any row which as
+/// fewer values is discarded as invalid.  Similarly, any row which is found
+/// to have more values than are defined in the schema is discarded as invalid
+/// (@todo, consider removing this constraint as it would prohibit reading a
+/// newer schema file with an older server).
+///
+/// When a row is found to have fewer than the defined number of columns,
+/// the values for each missing column is filled in with the default value
+/// specified by that column's descriptor.  In this manner rows from earlier
+/// schemas are upgraded to the current schema.
+///
+/// It is important to note that upgrading a file does NOT alter the physical
+/// file itself.  Rather the conversion occurs after the raw data has been read
+/// but before it is passed to caller.
+///
+/// Also note that there is currently no support for writing out a file in
+/// anything other than the current schema.
+class VersionedCSVFile : public CSVFile {
+public:
+
+    /// @brief Constructor.
+    ///
+    /// @param filename CSV file name.
+    VersionedCSVFile(const std::string& filename);
+
+    /// @brief Destructor
+    virtual ~VersionedCSVFile();
+
+    /// @brief Adds metadata for a single column to the schema.
+    ///
+    /// This method appends a new column description to the file's schema.
+    /// Note this does not cause anything to be written to the physical file.
+    /// The name of the column will be placed in the CSV header when new file
+    /// is created by calling @c recreate or @c open function.
+    ///
+    /// @param name Name of the column.
+    /// @param version  Text representation of the schema version in which
+    /// this column first appeared.
+    /// @param default_value value the missing column should be given during
+    /// an upgrade.  It defaults to an empty string, ""
+    ///
+    /// @throw CSVFileError if a column with the specified name exists.
+    void addColumn(const std::string& col_name, const std::string& version,
+                   const std::string& default_value = "");
+
+    /// @brief Sets the minimum number of valid columns based on a given column
+    ///
+    /// @param column_name Name of the column which positionally represents
+    /// the minimum columns which must be present in a file and to be
+    /// considered valid.
+    void setMinimumValidColumns(const std::string& column_name);
+
+    /// @brief Returns the minimum number of columns which must be present
+    /// for the file to be considered valid.
+    size_t getMinimumValidColumns();
+
+    /// @brief Opens existing file or creates a new one.
+    ///
+    /// This function will try to open existing file if this file has size
+    /// greater than 0. If the file doesn't exist or has size of 0, the
+    /// file is recreated. If the existing file has been opened, the header
+    /// is parsed and and validated against the schema.
+    /// By default, the data pointer in the file is set to the beginning of
+    /// the first data row. In order to retrieve the row contents the @c next
+    /// function should be called. If a @c seek_to_end parameter is set to
+    /// true, the file will be opened and the internal pointer will be set
+    /// to the end of file.
+    ///
+    /// @param seek_to_end A boolean value which indicates if the intput and
+    /// output file pointer should be set at the end of file.
+    ///
+    /// @throw VersionedCSVFileError if schema has not been defined,
+    /// CSVFileError when IO operation fails, or header fails to validate.
+    virtual void open(const bool seek_to_end = false);
+
+    /// @brief Reads next row from the file file.
+    ///
+    /// This function will return the @c CSVRow object representing a
+    /// parsed row if parsing is successful. If the end of file has been
+    /// reached, the empty row is returned (a row containing no values).
+    ///
+    /// 1. If the row has fewer values than were found in the header it is
+    /// discarded as invalid.
+    ///
+    /// 2. If the row is found to have more values than are defined in the
+    /// schema it is discarded as invalid
+    ///
+    /// When a valid row has fewer than the defined number of columns, the
+    /// values for each missing column is filled in with the default value
+    /// specified by that column's descriptor.
+    ///
+    /// @param [out] row Object receiving the parsed CSV file.
+    /// @param skip_validation Do not perform validation.
+    ///
+    /// @return true if row has been read and validated; false if validation
+    /// failed.
+    bool next(CSVRow& row);
+
+protected:
+
+    /// @brief Validates the header of a VersionedCSVFile
+    ///
+    /// This function is called internally when the reading in an existing
+    /// file.  It parses the header row of the file, comparing each value
+    /// in succession against the defined list of columns.  If the header
+    /// contains too few matching columns (i.e. less than @c
+    /// minimum_valid_columns_) or too many (more than the number of defined
+    /// columns), the file is presumed to be either too old, too new, or too
+    /// corrupt to process.  Otherwise it retains the number of valid columns
+    /// found and deems the header valid.
+    ///
+    /// @param header A row holding a header.
+    /// @return true if header matches the columns; false otherwise.
+    virtual bool validateHeader(const CSVRow& header);
+
+private:
+    /// @brief Holds the collection of column descriptors
+    std::vector<VersionedColumnPtr> columns_;
+
+    /// @brief Number of valid columns present in input file. If this is less
+    /// than the number of columns defined, this implies the input file is
+    /// from an earlier version of the code.
+    size_t valid_column_count_;
+
+    /// @brief Minimum number of valid columns an input file must contain.
+    /// If an input file does not meet this number it cannot be upgraded.
+    size_t minimum_valid_columns_;
+};
+
+
+} // namespace isc::util
+} // namespace isc
+
+#endif // VERSIONED_CSV_FILE_H