[3601] Added methods to VersionedCSVFile for tracking when updating is needed

src/lib/util/csv_file.h
    Made recreate() virtual

src/lib/util/versioned_csv_file.h
src/lib/util/versioned_csv_file.cc
    Added several methods to VersionedCSVFile:

    getValidColumnCount() - returns number of valid columns in header

    recreate() - wraps base class method, ensuring valid column
    count gets set to number of defined columns for new files

    needsUpgrading() - returns bool true if file schema is out of date

    getInputSchemaVersion() - returns schema version found in file

    getSchemaVersion() - returns current schema version

    getVersionedColumn() - returns the column definition for a given index

src/lib/util/tests/versioned_csv_file_unittest.cc
    Added checks for new methods to existing tests

src/lib/util/csv_file.h

@@ -404,7 +404,7 @@ public:
     /// Otherwise, this function will write the header to the file.
     /// In order to write rows to opened file, the @c append function
     /// should be called.
-    void recreate();
+    virtual void recreate();
 
     /// @brief Sets error message after row validation.
     ///

src/lib/util/tests/versioned_csv_file_unittest.cc

@@ -153,6 +153,22 @@ TEST_F(VersionedCSVFileTest, addColumn) {
     ASSERT_NO_THROW(csv->recreate());
     ASSERT_TRUE(exists());
 
+    // We should have 3 defined columns
+    EXPECT_EQ(3, csv->getColumnCount());
+
+    // Number valid columns should match defined columns
+    EXPECT_EQ(3, csv->getValidColumnCount());
+
+    // Minium valid columns wasn't set. (Remember it's optional)
+    EXPECT_EQ(0, csv->getMinimumValidColumns());
+
+    // Upgrade flag should be false
+    EXPECT_EQ(false, csv->needsUpgrading());
+
+    // Schema versions for new files should always match
+    EXPECT_EQ("3.0", csv->getInputSchemaVersion());
+    EXPECT_EQ("3.0", csv->getSchemaVersion());
+
     // Make sure we can't add columns (even unique) when the file is open.
     ASSERT_THROW(csv->addColumn("zoo", "3.0", ""), CSVFileError);
 
@@ -182,6 +198,22 @@ TEST_F(VersionedCSVFileTest, upgradeOlderVersions) {
     // Header should pass validation and allow the open to succeed.
     ASSERT_NO_THROW(csv->open());
 
+    // We should have 2 defined columns
+    EXPECT_EQ(2, csv->getColumnCount());
+
+    // We should have found 1 valid column in the header
+    EXPECT_EQ(1, csv->getValidColumnCount());
+
+    // Minium valid columns wasn't set. (Remember it's optional)
+    EXPECT_EQ(0, csv->getMinimumValidColumns());
+
+    // Upgrade flag should be true
+    EXPECT_EQ(true, csv->needsUpgrading());
+
+    // Input schema should be 1.0, while our current schema should be 2.0
+    EXPECT_EQ("1.0", csv->getInputSchemaVersion());
+    EXPECT_EQ("2.0", csv->getSchemaVersion());
+
     // First row is correct.
     CSVRow row;
     ASSERT_TRUE(csv->next(row));
@@ -223,7 +255,22 @@ TEST_F(VersionedCSVFileTest, upgradeOlderVersions) {
 
     // Header should pass validation and allow the open to succeed
     ASSERT_NO_THROW(csv->open());
-    ASSERT_EQ(3, csv->getColumnCount());
+
+    // We should have 2 defined columns
+    EXPECT_EQ(3, csv->getColumnCount());
+
+    // We should have found 1 valid column in the header
+    EXPECT_EQ(1, csv->getValidColumnCount());
+
+    // Minium valid columns wasn't set. (Remember it's optional)
+    EXPECT_EQ(0, csv->getMinimumValidColumns());
+
+    // Upgrade flag should be true
+    EXPECT_EQ(true, csv->needsUpgrading());
+
+    // Make sure schema versions are accurate
+    EXPECT_EQ("1.0", csv->getInputSchemaVersion());
+    EXPECT_EQ("3.0", csv->getSchemaVersion());
 
     // First row is correct.
     ASSERT_TRUE(csv->next(row));
@@ -243,15 +290,8 @@ TEST_F(VersionedCSVFileTest, upgradeOlderVersions) {
     EXPECT_EQ("blue", row.readAt(1));
     EXPECT_EQ("21", row.readAt(2));
 
-    ASSERT_EQ(3, csv->getColumnCount());
-
     // Fourth row is correct.
-    if (!csv->next(row)) {
-        std::cout << "row error is : " << 
-           csv->getReadMsg() << std::endl; 
-    
-    }
-
+    ASSERT_TRUE(csv->next(row));
     EXPECT_EQ("bird", row.readAt(0));
     EXPECT_EQ("yellow", row.readAt(1));
     EXPECT_EQ("21", row.readAt(2));

src/lib/util/versioned_csv_file.cc

@@ -46,23 +46,77 @@ VersionedCSVFile::setMinimumValidColumns(const std::string& column_name) {
 }
 
 size_t
-VersionedCSVFile::getMinimumValidColumns() {
+VersionedCSVFile::getMinimumValidColumns() const {
     return (minimum_valid_columns_);
 }
 
+size_t
+VersionedCSVFile::getValidColumnCount() const {
+    return (valid_column_count_);
+}
+
 void
 VersionedCSVFile::open(const bool seek_to_end) {
     if (getColumnCount() == 0) {
         isc_throw(VersionedCSVFileError,
-                  "no schema has been defined, cannot open file :"
+                  "no schema has been defined, cannot open CSV file :"
                   << getFilename());
     }
 
     CSVFile::open(seek_to_end);
 }
 
+void
+VersionedCSVFile::recreate() {
+    if (getColumnCount() == 0) {
+        isc_throw(VersionedCSVFileError,
+                  "no schema has been defined, cannot create CSV file :"
+                  << getFilename());
+    }
+
+    CSVFile::recreate();
+    // For new files they always match. 
+    valid_column_count_ = getColumnCount();
+}
+
+bool
+VersionedCSVFile::needsUpgrading() const {
+    return (getValidColumnCount() < getColumnCount());
+}
+
+std::string
+VersionedCSVFile::getInputSchemaVersion() const {
+    if (getValidColumnCount() > 0) {
+        return (getVersionedColumn(getValidColumnCount() - 1)->version_);
+    }
+
+    return ("undefined");
+}
+
+std::string
+VersionedCSVFile::getSchemaVersion() const {
+    if (getColumnCount() > 0) {
+        return (getVersionedColumn(getColumnCount() - 1)->version_);
+    }
+
+    return ("undefined");
+}
+
+const VersionedColumnPtr&
+VersionedCSVFile::getVersionedColumn(const size_t index) const {
+    if (index >= getColumnCount()) {
+        isc_throw(isc::OutOfRange, "versioned column index " << index 
+                  << " out of range;  CSV file : " << getFilename()
+                  << " only has " << getColumnCount() << " columns ");
+    }
+
+    return (columns_[index]);
+}
+
 bool
 VersionedCSVFile::next(CSVRow& row) {
+    // Use base class to physicall read the row, but skip its row
+    // validation
     CSVFile::next(row, true);
     if (row == CSVFile::EMPTY_ROW()) {
         return(true);
@@ -72,10 +126,10 @@ VersionedCSVFile::next(CSVRow& row) {
     // defined column count.  If not they're the equal.  Either way
     // each data row must have valid_column_count_ values or its
     // an invalid row.
-    if (row.getValuesCount() < valid_column_count_) {
+    if (row.getValuesCount() < getValidColumnCount()) {
         std::ostringstream s;
         s << "the size of the row '" << row << "' has too few valid columns "
-          << valid_column_count_ << "' of the CSV file '"
+          << getValidColumnCount() << "' of the CSV file '"
           << getFilename() << "'";
         setReadMsg(s.str());
         return (false);

src/lib/util/versioned_csv_file.h

@@ -153,7 +153,16 @@ public:
 
     /// @brief Returns the minimum number of columns which must be present
     /// for the file to be considered valid.
-    size_t getMinimumValidColumns();
+    size_t getMinimumValidColumns() const;
+
+    /// @brief Returns the number of valid columns found in the header
+    /// For newly created files this will always match the number of defined
+    /// columns (i.e. getColumnCount()).  For existing files, this will be
+    /// the number of columns in the header that match the defined columnns.
+    /// When this number is less than getColumnCount() it means the input file
+    /// is from an earlier schema.  This value is zero until the file has
+    /// been opened.
+    size_t getValidColumnCount() const;
 
     /// @brief Opens existing file or creates a new one.
     ///
@@ -174,6 +183,17 @@ public:
     /// CSVFileError when IO operation fails, or header fails to validate.
     virtual void open(const bool seek_to_end = false);
 
+    /// @brief Creates a new CSV file.
+    ///
+    /// The file creation will fail if there are no columns specified.
+    /// Otherwise, this function will write the header to the file.
+    /// In order to write rows to opened file, the @c append function
+    /// should be called.
+    ///
+    /// @throw VersionedCSVFileError if schema has not been defined
+    /// CSVFileError if an IO operation fails
+    virtual void recreate();
+
     /// @brief Reads next row from the file file.
     ///
     /// This function will return the @c CSVRow object representing a
@@ -197,6 +217,32 @@ public:
     /// failed.
     bool next(CSVRow& row);
 
+    /// @brief Returns the schema version of the physical file
+    ///
+    /// @return text version of the schema found or string "undefined" if the
+    /// file has not been opened
+    std::string getInputSchemaVersion() const;
+
+    /// @brief text version of current schema supported by the file's metadata
+    ///
+    /// @return text version info assigned to the last column in the list of
+    /// defined column, or the string "undefined" if no columns have been
+    /// defined.
+    std::string getSchemaVersion() const;
+
+    /// @brief Fetch the column descriptor for a given index
+    ///
+    /// @param index index within the list of columns of the desired column
+    /// @return a pointer to the VersionedColumn at the given index
+    /// @trow OutOfRange exception if the index is invalid
+    const VersionedColumnPtr& getVersionedColumn(const size_t index) const;
+
+    /// @brief Returns true if the opened file is needs to be upgraded
+    ///
+    /// @return true if the file's valid column count is greater than 0 and
+    /// is less than the defined number of columns
+    bool needsUpgrading() const;
+
 protected:
 
     /// @brief Validates the header of a VersionedCSVFile