Merge branch #1067

Conflicts:
	src/lib/datasrc/database.h

src/lib/datasrc/Makefile.am

@@ -21,7 +21,7 @@ libdatasrc_la_SOURCES += memory_datasrc.h memory_datasrc.cc
 libdatasrc_la_SOURCES += zone.h
 libdatasrc_la_SOURCES += result.h
 libdatasrc_la_SOURCES += logger.h logger.cc
-libdatasrc_la_SOURCES += client.h
+libdatasrc_la_SOURCES += client.h iterator.h
 libdatasrc_la_SOURCES += database.h database.cc
 libdatasrc_la_SOURCES += sqlite3_accessor.h sqlite3_accessor.cc
 nodist_libdatasrc_la_SOURCES = datasrc_messages.h datasrc_messages.cc

src/lib/datasrc/client.h

@@ -16,12 +16,19 @@
 #define __DATA_SOURCE_CLIENT_H 1
 
 #include <boost/noncopyable.hpp>
+#include <boost/shared_ptr.hpp>
+
+#include <exceptions/exceptions.h>
 
 #include <datasrc/zone.h>
 
 namespace isc {
 namespace datasrc {
 
+// The iterator.h is not included on purpose, most application won't need it
+class ZoneIterator;
+typedef boost::shared_ptr<ZoneIterator> ZoneIteratorPtr;
+
 /// \brief The base class of data source clients.
 ///
 /// This is an abstract base class that defines the common interface for
@@ -143,6 +150,36 @@ public:
     /// \param name A domain name for which the search is performed.
     /// \return A \c FindResult object enclosing the search result (see above).
     virtual FindResult findZone(const isc::dns::Name& name) const = 0;
+
+    /// \brief Returns an iterator to the given zone
+    ///
+    /// This allows for traversing the whole zone. The returned object can
+    /// provide the RRsets one by one.
+    ///
+    /// This throws DataSourceError when the zone does not exist in the
+    /// datasource.
+    ///
+    /// The default implementation throws isc::NotImplemented. This allows
+    /// for easy and fast deployment of minimal custom data sources, where
+    /// the user/implementator doesn't have to care about anything else but
+    /// the actual queries. Also, in some cases, it isn't possible to traverse
+    /// the zone from logic point of view (eg. dynamically generated zone
+    /// data).
+    ///
+    /// It is not fixed if a concrete implementation of this method can throw
+    /// anything else.
+    ///
+    /// \param name The name of zone apex to be traversed. It doesn't do
+    ///     nearest match as findZone.
+    /// \return Pointer to the iterator.
+    virtual ZoneIteratorPtr getIterator(const isc::dns::Name& name) const {
+        // This is here to both document the parameter in doxygen (therefore it
+        // needs a name) and avoid unused parameter warning.
+        static_cast<void>(name);
+
+        isc_throw(isc::NotImplemented,
+                  "Data source doesn't support iteration");
+    }
 };
 }
 }

src/lib/datasrc/database.cc

@@ -15,10 +15,12 @@
 #include <vector>
 
 #include <datasrc/database.h>
+#include <datasrc/data_source.h>
+#include <datasrc/iterator.h>
 
 #include <exceptions/exceptions.h>
 #include <dns/name.h>
-#include <dns/rrttl.h>
+#include <dns/rrclass.h>
 #include <dns/rdata.h>
 #include <dns/rdataclass.h>
 
@@ -27,7 +29,10 @@
 
 #include <boost/foreach.hpp>
 
-using isc::dns::Name;
+#include <string>
+
+using namespace isc::dns;
+using std::string;
 
 namespace isc {
 namespace datasrc {
@@ -109,7 +114,7 @@ void addOrCreate(isc::dns::RRsetPtr& rrset,
             if (ttl < rrset->getTTL()) {
                 rrset->setTTL(ttl);
             }
-            logger.info(DATASRC_DATABASE_FIND_TTL_MISMATCH)
+            logger.warn(DATASRC_DATABASE_FIND_TTL_MISMATCH)
                 .arg(db.getDBName()).arg(name).arg(cls)
                 .arg(type).arg(rrset->getTTL());
         }
@@ -401,5 +406,109 @@ DatabaseClient::Finder::getClass() const {
     return isc::dns::RRClass::IN();
 }
 
+namespace {
+
+/*
+ * This needs, beside of converting all data from textual representation, group
+ * together rdata of the same RRsets. To do this, we hold one row of data ahead
+ * of iteration. When we get a request to provide data, we create it from this
+ * data and load a new one. If it is to be put to the same rrset, we add it.
+ * Otherwise we just return what we have and keep the row as the one ahead
+ * for next time.
+ */
+class DatabaseIterator : public ZoneIterator {
+public:
+    DatabaseIterator(const DatabaseAccessor::IteratorContextPtr& context,
+             const RRClass& rrclass) :
+        context_(context),
+        class_(rrclass),
+        ready_(true)
+    {
+        // Prepare data for the next time
+        getData();
+    }
+
+    virtual isc::dns::ConstRRsetPtr getNextRRset() {
+        if (!ready_) {
+            isc_throw(isc::Unexpected, "Iterating past the zone end");
+        }
+        if (!data_ready_) {
+            // At the end of zone
+            ready_ = false;
+            LOG_DEBUG(logger, DBG_TRACE_DETAILED,
+                      DATASRC_DATABASE_ITERATE_END);
+            return (ConstRRsetPtr());
+        }
+        string name_str(name_), rtype_str(rtype_), ttl(ttl_);
+        Name name(name_str);
+        RRType rtype(rtype_str);
+        RRsetPtr rrset(new RRset(name, class_, rtype, RRTTL(ttl)));
+        while (data_ready_ && name_ == name_str && rtype_str == rtype_) {
+            if (ttl_ != ttl) {
+                if (ttl < ttl_) {
+                    ttl_ = ttl;
+                    rrset->setTTL(RRTTL(ttl));
+                }
+                LOG_WARN(logger, DATASRC_DATABASE_ITERATE_TTL_MISMATCH).
+                    arg(name_).arg(class_).arg(rtype_).arg(rrset->getTTL());
+            }
+            rrset->addRdata(rdata::createRdata(rtype, class_, rdata_));
+            getData();
+        }
+        LOG_DEBUG(logger, DBG_TRACE_DETAILED, DATASRC_DATABASE_ITERATE_NEXT).
+            arg(rrset->getName()).arg(rrset->getType());
+        return (rrset);
+    }
+private:
+    // Load next row of data
+    void getData() {
+        string data[DatabaseAccessor::COLUMN_COUNT];
+        data_ready_ = context_->getNext(data, DatabaseAccessor::COLUMN_COUNT);
+        name_ = data[DatabaseAccessor::NAME_COLUMN];
+        rtype_ = data[DatabaseAccessor::TYPE_COLUMN];
+        ttl_ = data[DatabaseAccessor::TTL_COLUMN];
+        rdata_ = data[DatabaseAccessor::RDATA_COLUMN];
+    }
+
+    // The context
+    const DatabaseAccessor::IteratorContextPtr context_;
+    // Class of the zone
+    RRClass class_;
+    // Status
+    bool ready_, data_ready_;
+    // Data of the next row
+    string name_, rtype_, rdata_, ttl_;
+};
+
+}
+
+ZoneIteratorPtr
+DatabaseClient::getIterator(const isc::dns::Name& name) const {
+    // Get the zone
+    std::pair<bool, int> zone(database_->getZone(name));
+    if (!zone.first) {
+        // No such zone, can't continue
+        isc_throw(DataSourceError, "Zone " + name.toText() +
+                  " can not be iterated, because it doesn't exist "
+                  "in this data source");
+    }
+    // Request the context
+    DatabaseAccessor::IteratorContextPtr
+        context(database_->getAllRecords(name, zone.second));
+    // It must not return NULL, that's a bug of the implementation
+    if (context == DatabaseAccessor::IteratorContextPtr()) {
+        isc_throw(isc::Unexpected, "Iterator context null at " +
+                  name.toText());
+    }
+    // Create the iterator and return it
+    // TODO: Once #1062 is merged with this, we need to get the
+    // actual zone class from the connection, as the DatabaseClient
+    // doesn't know it and the iterator needs it (so it wouldn't query
+    // it each time)
+    LOG_DEBUG(logger, DBG_TRACE_DETAILED, DATASRC_DATABASE_ITERATE).
+        arg(name);
+    return (ZoneIteratorPtr(new DatabaseIterator(context, RRClass::IN())));
+}
+
 }
 }

src/lib/datasrc/database.h

@@ -18,6 +18,7 @@
 #include <datasrc/client.h>
 
 #include <dns/name.h>
+#include <exceptions/exceptions.h>
 
 namespace isc {
 namespace datasrc {
@@ -76,6 +77,86 @@ public:
     virtual std::pair<bool, int> getZone(const isc::dns::Name& name) const = 0;
 
     /**
+     * \brief This holds the internal context of ZoneIterator for databases
+     *
+     * While the ZoneIterator implementation from DatabaseClient does all the
+     * translation from strings to DNS classes and validation, this class
+     * holds the pointer to where the database is at reading the data.
+     *
+     * It can either hold shared pointer to the connection which created it
+     * and have some kind of statement inside (in case single database
+     * connection can handle multiple concurrent SQL statements) or it can
+     * create a new connection (or, if it is more convenient, the connection
+     * itself can inherit both from DatabaseConnection and IteratorContext
+     * and just clone itself).
+     */
+    class IteratorContext : public boost::noncopyable {
+    public:
+        /**
+         * \brief Destructor
+         *
+         * Virtual destructor, so any descendand class is destroyed correctly.
+         */
+        virtual ~IteratorContext() { }
+
+        /**
+         * \brief Function to provide next resource record
+         *
+         * This function should provide data about the next resource record
+         * from the iterated zone. The data are not converted yet.
+         *
+         * \note The order of RRs is not strictly set, but the RRs for single
+         * RRset must not be interleaved with any other RRs (eg. RRsets must be
+         * "together").
+         *
+         * \param columns The data will be returned through here. The order
+         *     is specified by the RecordColumns enum.
+         * \param Size of the columns array. Must be equal to COLUMN_COUNT,
+         *     otherwise DataSourceError is thrown.
+         * \todo Do we consider databases where it is stored in binary blob
+         *     format?
+         * \throw DataSourceError if there's database-related error. If the
+         *     exception (or any other in case of derived class) is thrown,
+         *     the iterator can't be safely used any more.
+         */
+        virtual bool getNext(std::string columns[], size_t column_data) = 0;
+    };
+
+    typedef boost::shared_ptr<IteratorContext> IteratorContextPtr;
+
+    /**
+     * \brief Creates an iterator context for the whole zone.
+     *
+     * This should create a new iterator context to be used by
+     * DatabaseConnection's ZoneIterator. It can be created based on the name
+     * or the ID (returned from getZone()), what is more comfortable for the
+     * database implementation. Both are provided (and are guaranteed to match,
+     * the DatabaseClient first looks up the zone ID and then calls this).
+     *
+     * The default implementation throws isc::NotImplemented, to allow
+     * "minimal" implementations of the connection not supporting optional
+     * functionality.
+     *
+     * \param name The name of the zone.
+     * \param id The ID of the zone, returned from getZone().
+     * \return Newly created iterator context. Must not be NULL.
+     */
+    virtual IteratorContextPtr getAllRecords(const isc::dns::Name& name,
+                                             int id) const
+    {
+        /*
+         * This is a compromise. We need to document the parameters in doxygen,
+         * so they need a name, but then it complains about unused parameter.
+         * This is a NOP that "uses" the parameters.
+         */
+        static_cast<void>(name);
+        static_cast<void>(id);
+
+        isc_throw(isc::NotImplemented,
+                  "This database datasource can't be iterated");
+    }
+
+    /**
      * \brief Starts a new search for records of the given name in the given zone
      *
      * The data searched by this call can be retrieved with subsequent calls to
@@ -145,11 +226,12 @@ public:
         SIGTYPE_COLUMN = 2, ///< For RRSIG records, this contains the RRTYPE
                             ///< the RRSIG covers. In the current implementation,
                             ///< this field is ignored.
-        RDATA_COLUMN = 3    ///< Full text representation of the record's RDATA
+        RDATA_COLUMN = 3,   ///< Full text representation of the record's RDATA
+        NAME_COLUMN = 4     ///< The domain name of this RR
     };
 
     /// The number of fields the columns array passed to getNextRecord should have
-    static const size_t COLUMN_COUNT = 4;
+    static const size_t COLUMN_COUNT = 5;
 
     /**
      * \brief Returns a string identifying this dabase backend
@@ -356,6 +438,25 @@ public:
      */
     virtual FindResult findZone(const isc::dns::Name& name) const;
 
+    /**
+     * \brief Get the zone iterator
+     *
+     * The iterator allows going through the whole zone content. If the
+     * underlying DatabaseConnection is implemented correctly, it should
+     * be possible to have multiple ZoneIterators at once and query data
+     * at the same time.
+     *
+     * \exception DataSourceError if the zone doesn't exist.
+     * \exception isc::NotImplemented if the underlying DatabaseConnection
+     *     doesn't implement iteration. But in case it is not implemented
+     *     and the zone doesn't exist, DataSourceError is thrown.
+     * \exception Anything else the underlying DatabaseConnection might
+     *     want to throw.
+     * \param name The origin of the zone to iterate.
+     * \return Shared pointer to the iterator (it will never be NULL)
+     */
+    virtual ZoneIteratorPtr getIterator(const isc::dns::Name& name) const;
+
 private:
     /// \brief Our database.
     const boost::shared_ptr<DatabaseAccessor> database_;

src/lib/datasrc/datasrc_messages.mes

@@ -75,8 +75,9 @@ name and type in the database.
 
 % DATASRC_DATABASE_FIND_TTL_MISMATCH TTL values differ in %1 for elements of %2/%3/%4, setting to %5
 The datasource backend provided resource records for the given RRset with
-different TTL values. The TTL of the RRSET is set to the lowest value, which
-is printed in the log message.
+different TTL values. This isn't allowed on the wire and is considered
+an error, so we set it to the lowest value we found (but we don't modify the
+database). The data in database should be checked and fixed.
 
 % DATASRC_DATABASE_FIND_UNCAUGHT_ERROR uncaught general error retrieving data from datasource %1: %2
 There was an uncaught general exception while reading data from a datasource.
@@ -117,6 +118,23 @@ The data returned by the database backend contained data for the given domain
 name, and it either matches the type or has a relevant type. The RRset that is
 returned is printed.
 
+% DATASRC_DATABASE_ITERATE iterating zone %1
+The program is reading the whole zone, eg. not searching for data, but going
+through each of the RRsets there.
+
+% DATASRC_DATABASE_ITERATE_END iterating zone finished
+While iterating through the zone, the program reached end of the data.
+
+% DATASRC_DATABASE_ITERATE_NEXT next RRset in zone is %1/%2
+While iterating through the zone, the program extracted next RRset from it.
+The name and RRtype of the RRset is indicated in the message.
+
+% DATASRC_DATABASE_ITERATE_TTL_MISMATCH TTL values differ for RRs of %1/%2/%3, setting to %4
+While iterating through the zone, the time to live for RRs of the given RRset
+were found to be different. This isn't allowed on the wire and is considered
+an error, so we set it to the lowest value we found (but we don't modify the
+database). The data in database should be checked and fixed.
+
 % DATASRC_DO_QUERY handling query for '%1/%2'
 A debug message indicating that a query for the given name and RR type is being
 processed.

src/lib/datasrc/iterator.h

@@ -0,0 +1,61 @@
+// Copyright (C) 2011  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <dns/rrset.h>
+
+#include <boost/noncopyable.hpp>
+
+namespace isc {
+namespace datasrc {
+
+/**
+ * \brief Read-only iterator to a zone.
+ *
+ * You can get an instance of (descendand of) ZoneIterator from
+ * DataSourceClient::getIterator() method. The actual concrete implementation
+ * will be different depending on the actual data source used. This is the
+ * abstract interface.
+ *
+ * There's no way to start iterating from the beginning again or return.
+ */
+class ZoneIterator : public boost::noncopyable {
+public:
+    /**
+     * \brief Destructor
+     *
+     * Virtual destructor. It is empty, but ensures the right destructor from
+     * descendant is called.
+     */
+    virtual ~ ZoneIterator() { }
+
+    /**
+     * \brief Get next RRset from the zone.
+     *
+     * This returns the next RRset in the zone as a shared pointer. The
+     * shared pointer is used to allow both accessing in-memory data and
+     * automatic memory management.
+     *
+     * Any special order is not guaranteed.
+     *
+     * While this can potentially throw anything (including standard allocation
+     * errors), it should be rare.
+     *
+     * \return Pointer to the next RRset or NULL pointer when the iteration
+     *     gets to the end of the zone.
+     */
+    virtual isc::dns::ConstRRsetPtr getNextRRset() = 0;
+};
+
+}
+}

src/lib/datasrc/memory_datasrc.cc

@@ -25,6 +25,8 @@
 #include <datasrc/memory_datasrc.h>
 #include <datasrc/rbtree.h>
 #include <datasrc/logger.h>
+#include <datasrc/iterator.h>
+#include <datasrc/data_source.h>
 
 using namespace std;
 using namespace isc::dns;
@@ -32,6 +34,27 @@ using namespace isc::dns;
 namespace isc {
 namespace datasrc {
 
+namespace {
+// Some type aliases
+/*
+ * Each domain consists of some RRsets. They will be looked up by the
+ * RRType.
+ *
+ * The use of map is questionable with regard to performance - there'll
+ * be usually only few RRsets in the domain, so the log n benefit isn't
+ * much and a vector/array might be faster due to its simplicity and
+ * continuous memory location. But this is unlikely to be a performance
+ * critical place and map has better interface for the lookups, so we use
+ * that.
+ */
+typedef map<RRType, ConstRRsetPtr> Domain;
+typedef Domain::value_type DomainPair;
+typedef boost::shared_ptr<Domain> DomainPtr;
+// The tree stores domains
+typedef RBTree<Domain> DomainTree;
+typedef RBNode<Domain> DomainNode;
+}
+
 // Private data and hidden methods of InMemoryZoneFinder
 struct InMemoryZoneFinder::InMemoryZoneFinderImpl {
     // Constructor
@@ -44,25 +67,6 @@ struct InMemoryZoneFinder::InMemoryZoneFinderImpl {
         DomainPtr origin_domain(new Domain);
         origin_data_->setData(origin_domain);
     }
-
-    // Some type aliases
-    /*
-     * Each domain consists of some RRsets. They will be looked up by the
-     * RRType.
-     *
-     * The use of map is questionable with regard to performance - there'll
-     * be usually only few RRsets in the domain, so the log n benefit isn't
-     * much and a vector/array might be faster due to its simplicity and
-     * continuous memory location. But this is unlikely to be a performance
-     * critical place and map has better interface for the lookups, so we use
-     * that.
-     */
-    typedef map<RRType, ConstRRsetPtr> Domain;
-    typedef Domain::value_type DomainPair;
-    typedef boost::shared_ptr<Domain> DomainPtr;
-    // The tree stores domains
-    typedef RBTree<Domain> DomainTree;
-    typedef RBNode<Domain> DomainNode;
     static const DomainNode::Flags DOMAINFLAG_WILD = DomainNode::FLAG_USER1;
 
     // Information about the zone
@@ -634,7 +638,7 @@ InMemoryZoneFinder::load(const string& filename) {
     LOG_DEBUG(logger, DBG_TRACE_BASIC, DATASRC_MEM_LOAD).arg(getOrigin()).
         arg(filename);
     // Load it into a temporary tree
-    InMemoryZoneFinderImpl::DomainTree tmp;
+    DomainTree tmp;
     masterLoad(filename.c_str(), getOrigin(), getClass(),
         boost::bind(&InMemoryZoneFinderImpl::addFromLoad, impl_, _1, &tmp));
     // If it went well, put it inside
@@ -700,8 +704,94 @@ InMemoryClient::addZone(ZoneFinderPtr zone_finder) {
 InMemoryClient::FindResult
 InMemoryClient::findZone(const isc::dns::Name& name) const {
     LOG_DEBUG(logger, DBG_TRACE_DATA, DATASRC_MEM_FIND_ZONE).arg(name);
-    return (FindResult(impl_->zone_table.findZone(name).code,
-                       impl_->zone_table.findZone(name).zone));
+    ZoneTable::FindResult result(impl_->zone_table.findZone(name));
+    return (FindResult(result.code, result.zone));
+}
+
+namespace {
+
+class MemoryIterator : public ZoneIterator {
+private:
+    RBTreeNodeChain<Domain> chain_;
+    Domain::const_iterator dom_iterator_;
+    const DomainTree& tree_;
+    const DomainNode* node_;
+    bool ready_;
+public:
+    MemoryIterator(const DomainTree& tree, const Name& origin) :
+        tree_(tree),
+        ready_(true)
+    {
+        // Find the first node (origin) and preserve the node chain for future
+        // searches
+        DomainTree::Result result(tree_.find<void*>(origin, &node_, chain_,
+                                                    NULL, NULL));
+        // It can't happen that the origin is not in there
+        if (result != DomainTree::EXACTMATCH) {
+            isc_throw(Unexpected,
+                      "In-memory zone corrupted, missing origin node");
+        }
+        // Initialize the iterator if there's somewhere to point to
+        if (node_ != NULL && node_->getData() != DomainPtr()) {
+            dom_iterator_ = node_->getData()->begin();
+        }
+    }
+
+    virtual ConstRRsetPtr getNextRRset() {
+        if (!ready_) {
+            isc_throw(Unexpected, "Iterating past the zone end");
+        }
+        /*
+         * This cycle finds the first nonempty node with yet unused RRset.
+         * If it is NULL, we run out of nodes. If it is empty, it doesn't
+         * contain any RRsets. If we are at the end, just get to next one.
+         */
+        while (node_ != NULL && (node_->getData() == DomainPtr() ||
+                                 dom_iterator_ == node_->getData()->end())) {
+            node_ = tree_.nextNode(chain_);
+            // If there's a node, initialize the iterator and check next time
+            // if the map is empty or not
+            if (node_ != NULL && node_->getData() != NULL) {
+                dom_iterator_ = node_->getData()->begin();
+            }
+        }
+        if (node_ == NULL) {
+            // That's all, folks
+            ready_ = false;
+            return (ConstRRsetPtr());
+        }
+        // The iterator points to the next yet unused RRset now
+        ConstRRsetPtr result(dom_iterator_->second);
+        // This one is used, move it to the next time for next call
+        ++dom_iterator_;
+
+        return (result);
+    }
+};
+
+} // End of anonymous namespace
+
+ZoneIteratorPtr
+InMemoryClient::getIterator(const Name& name) const {
+    ZoneTable::FindResult result(impl_->zone_table.findZone(name));
+    if (result.code != result::SUCCESS) {
+        isc_throw(DataSourceError, "No such zone: " + name.toText());
+    }
+
+    const InMemoryZoneFinder*
+        zone(dynamic_cast<const InMemoryZoneFinder*>(result.zone.get()));
+    if (zone == NULL) {
+        /*
+         * TODO: This can happen only during some of the tests and only as
+         * a temporary solution. This should be fixed by #1159 and then
+         * this cast and check shouldn't be necessary. We don't have
+         * test for handling a "can not happen" condition.
+         */
+        isc_throw(Unexpected, "The zone at " + name.toText() +
+                  " is not InMemoryZoneFinder");
+    }
+    return (ZoneIteratorPtr(new MemoryIterator(zone->impl_->domains_, name)));
 }
+
 } // end of namespace datasrc
 } // end of namespace dns

src/lib/datasrc/memory_datasrc.h

@@ -182,6 +182,11 @@ private:
     struct InMemoryZoneFinderImpl;
     InMemoryZoneFinderImpl* impl_;
     //@}
+    // The friend here is for InMemoryClient::getIterator. The iterator
+    // needs to access the data inside the zone, so the InMemoryClient
+    // extracts the pointer to data and puts it into the iterator.
+    // The access is read only.
+    friend class InMemoryClient;
 };
 
 /// \brief A data source client that holds all necessary data in memory.
@@ -258,6 +263,9 @@ public:
     /// For other details see \c DataSourceClient::findZone().
     virtual FindResult findZone(const isc::dns::Name& name) const;
 
+    /// \brief Implementation of the getIterator method
+    virtual ZoneIteratorPtr getIterator(const isc::dns::Name& name) const;
+
 private:
     // TODO: Do we still need the PImpl if nobody should manipulate this class
     // directly any more (it should be handled through DataSourceClient)?

src/lib/datasrc/sqlite3_accessor.cc

@@ -19,6 +19,8 @@
 #include <datasrc/data_source.h>
 #include <util/filename.h>
 
+#include <boost/lexical_cast.hpp>
+
 namespace isc {
 namespace datasrc {
 
@@ -136,9 +138,13 @@ const char* const SCHEMA_LIST[] = {
 
 const char* const q_zone_str = "SELECT id FROM zones WHERE name=?1 AND rdclass = ?2";
 
-const char* const q_any_str = "SELECT rdtype, ttl, sigtype, rdata "
+const char* const q_any_str = "SELECT rdtype, ttl, sigtype, rdata, name "
     "FROM records WHERE zone_id=?1 AND name=?2";
 
+const char* const q_iterate_str = "SELECT rdtype, ttl, sigtype, rdata, name FROM records "
+                                  "WHERE zone_id = ?1 "
+                                  "ORDER BY name, rdtype";
+
 /* TODO: Prune the statements, not everything will be needed maybe?
 const char* const q_record_str = "SELECT rdtype, ttl, sigtype, rdata "
     "FROM records WHERE zone_id=?1 AND name=?2 AND "
@@ -313,30 +319,18 @@ SQLite3Database::getZone(const isc::dns::Name& name) const {
         result = std::pair<bool, int>(true,
                                       sqlite3_column_int(dbparameters_->
                                                          q_zone_, 0));
-    } else {
+        return (result);
+    } else if (rc == SQLITE_DONE) {
         result = std::pair<bool, int>(false, 0);
+        // Free resources
+        sqlite3_reset(dbparameters_->q_zone_);
+        return (result);
     }
-    // Free resources
-    sqlite3_reset(dbparameters_->q_zone_);
-
-    return (result);
-}
 
-void
-SQLite3Database::searchForRecords(int zone_id, const std::string& name) {
-    resetSearch();
-    if (sqlite3_bind_int(dbparameters_->q_any_, 1, zone_id) != SQLITE_OK) {
-        isc_throw(DataSourceError,
-                  "Error in sqlite3_bind_int() for zone_id " <<
-                  zone_id << ": " << sqlite3_errmsg(dbparameters_->db_));
-    }
-    // use transient since name is a ref and may disappear
-    if (sqlite3_bind_text(dbparameters_->q_any_, 2, name.c_str(), -1,
-                               SQLITE_TRANSIENT) != SQLITE_OK) {
-        isc_throw(DataSourceError,
-                  "Error in sqlite3_bind_text() for name " <<
-                  name << ": " << sqlite3_errmsg(dbparameters_->db_));
-    }
+    isc_throw(DataSourceError, "Unexpected failure in sqlite3_step: " <<
+                               sqlite3_errmsg(dbparameters_->db_));
+    // Compilers might not realize isc_throw always throws
+    return (std::pair<bool, int>(false, 0));
 }
 
 namespace {
@@ -368,6 +362,76 @@ convertToPlainChar(const unsigned char* ucp,
 }
 }
 
+// TODO: Once we want to have iterator returned from searchForRecords, this
+// class can be reused. It should be modified to take the sqlite3 statement
+// instead of creating it in constructor, it doesn't have to care which one
+// it is, just provide data from it.
+class SQLite3Database::Context : public DatabaseAccessor::IteratorContext {
+public:
+    Context(const boost::shared_ptr<const SQLite3Database>& database, int id) :
+        database_(database),
+        statement(NULL)
+    {
+        // We create the statement now and then just keep getting data from it
+        statement = prepare(database->dbparameters_->db_, q_iterate_str);
+        if (sqlite3_bind_int(statement, 1, id) != SQLITE_OK) {
+            isc_throw(SQLite3Error, "Could not bind " << id <<
+                      " to SQL statement (iterate)");
+        }
+    }
+    bool getNext(std::string data[], size_t size) {
+        if (size != COLUMN_COUNT) {
+            isc_throw(DataSourceError, "getNext received size of " << size <<
+                      ", not " << COLUMN_COUNT);
+        }
+        // If there's another row, get it
+        int rc(sqlite3_step(statement));
+        if (rc == SQLITE_ROW) {
+            for (size_t i(0); i < size; ++ i) {
+                data[i] = convertToPlainChar(sqlite3_column_text(statement, i),
+                                             database_->dbparameters_);
+            }
+            return (true);
+        } else if (rc != SQLITE_DONE) {
+            isc_throw(DataSourceError,
+                      "Unexpected failure in sqlite3_step: " <<
+                      sqlite3_errmsg(database_->dbparameters_->db_));
+        }
+        return (false);
+    }
+    virtual ~Context() {
+        if (statement) {
+            sqlite3_finalize(statement);
+        }
+    }
+
+private:
+    boost::shared_ptr<const SQLite3Database> database_;
+    sqlite3_stmt *statement;
+};
+
+DatabaseAccessor::IteratorContextPtr
+SQLite3Database::getAllRecords(const isc::dns::Name&, int id) const {
+    return (IteratorContextPtr(new Context(shared_from_this(), id)));
+}
+
+void
+SQLite3Database::searchForRecords(int zone_id, const std::string& name) {
+    resetSearch();
+    if (sqlite3_bind_int(dbparameters_->q_any_, 1, zone_id) != SQLITE_OK) {
+        isc_throw(DataSourceError,
+                  "Error in sqlite3_bind_int() for zone_id " <<
+                  zone_id << ": " << sqlite3_errmsg(dbparameters_->db_));
+    }
+    // use transient since name is a ref and may disappear
+    if (sqlite3_bind_text(dbparameters_->q_any_, 2, name.c_str(), -1,
+                               SQLITE_TRANSIENT) != SQLITE_OK) {
+        isc_throw(DataSourceError,
+                  "Error in sqlite3_bind_text() for name " <<
+                  name << ": " << sqlite3_errmsg(dbparameters_->db_));
+    }
+}
+
 bool
 SQLite3Database::getNextRecord(std::string columns[], size_t column_count) {
     if (column_count != COLUMN_COUNT) {

src/lib/datasrc/sqlite3_accessor.h

@@ -20,6 +20,7 @@
 
 #include <exceptions/exceptions.h>
 
+#include <boost/enable_shared_from_this.hpp>
 #include <string>
 
 namespace isc {
@@ -51,7 +52,8 @@ struct SQLite3Parameters;
  * According to the design, it doesn't interpret the data in any way, it just
  * provides unified access to the DB.
  */
-class SQLite3Database : public DatabaseAccessor {
+class SQLite3Database : public DatabaseAccessor,
+    public boost::enable_shared_from_this<SQLite3Database> {
 public:
     /**
      * \brief Constructor
@@ -89,6 +91,9 @@ public:
      */
     virtual std::pair<bool, int> getZone(const isc::dns::Name& name) const;
 
+    /// \brief Implementation of DatabaseAbstraction::getAllRecords
+    virtual IteratorContextPtr getAllRecords(const isc::dns::Name&,
+                                                  int id) const;
     /**
      * \brief Start a new search for the given name in the given zone.
      *
@@ -151,6 +156,9 @@ private:
     void open(const std::string& filename);
     /// \brief Closes the database
     void close();
+    /// \brief SQLite3 implementation of IteratorContext
+    class Context;
+    friend class Context;
     const std::string database_name_;
 };
 

src/lib/datasrc/tests/Makefile.am

@@ -29,6 +29,7 @@ run_unittests_SOURCES += zonetable_unittest.cc
 run_unittests_SOURCES += memory_datasrc_unittest.cc
 run_unittests_SOURCES += logger_unittest.cc
 run_unittests_SOURCES += database_unittest.cc
+run_unittests_SOURCES += client_unittest.cc
 run_unittests_SOURCES += sqlite3_accessor_unittest.cc
 
 run_unittests_CPPFLAGS = $(AM_CPPFLAGS) $(GTEST_INCLUDES)

src/lib/datasrc/tests/cache_unittest.cc

@@ -202,15 +202,15 @@ TEST_F(CacheTest, retrieveFail) {
 }
 
 TEST_F(CacheTest, expire) {
-    // Insert "foo" with a duration of 2 seconds; sleep 3.  The
+    // Insert "foo" with a duration of 1 seconds; sleep 2.  The
     // record should not be returned from the cache even though it's
     // at the top of the cache.
     RRsetPtr aaaa(new RRset(Name("foo"), RRClass::IN(), RRType::AAAA(),
                             RRTTL(0)));
     aaaa->addRdata(in::AAAA("2001:db8:3:bb::5"));
-    cache.addPositive(aaaa, 0, 2);
+    cache.addPositive(aaaa, 0, 1);
 
-    sleep(3);
+    sleep(2);
 
     RRsetPtr r;
     uint32_t f;

src/lib/datasrc/tests/client_unittest.cc

@@ -0,0 +1,47 @@
+// Copyright (C) 2011  Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <datasrc/client.h>
+
+#include <dns/name.h>
+
+#include <gtest/gtest.h>
+
+using namespace isc::datasrc;
+using isc::dns::Name;
+
+namespace {
+
+/*
+ * The DataSourceClient can't be created as it has pure virtual methods.
+ * So we implement them as NOPs and test the other methods.
+ */
+class NopClient : public DataSourceClient {
+public:
+    virtual FindResult findZone(const isc::dns::Name&) const {
+        return (FindResult(result::NOTFOUND, ZoneFinderPtr()));
+    }
+};
+
+class ClientTest : public ::testing::Test {
+public:
+    NopClient client_;
+};
+
+// The default implementation is NotImplemented
+TEST_F(ClientTest, defaultIterator) {
+    EXPECT_THROW(client_.getIterator(Name(".")), isc::NotImplemented);
+}
+
+}

src/lib/datasrc/tests/database_unittest.cc

@@ -22,6 +22,7 @@
 #include <datasrc/database.h>
 #include <datasrc/zone.h>
 #include <datasrc/data_source.h>
+#include <datasrc/iterator.h>
 
 #include <testutils/dnsmessage_test.h>
 
@@ -30,30 +31,169 @@
 using namespace isc::datasrc;
 using namespace std;
 using namespace boost;
-using isc::dns::Name;
+using namespace isc::dns;
 
 namespace {
 
 /*
- * A virtual database database that pretends it contains single zone --
- * example.org.
+ * An accessor with minimum implementation, keeping the original
+ * "NotImplemented" methods.
  */
-class MockAccessor : public DatabaseAccessor {
+class NopAccessor : public DatabaseAccessor {
 public:
-    MockAccessor() : search_running_(false),
-                       database_name_("mock_database")
-    {
-        fillData();
-    }
+    NopAccessor() : database_name_("mock_database")
+    { }
 
     virtual std::pair<bool, int> getZone(const Name& name) const {
         if (name == Name("example.org")) {
             return (std::pair<bool, int>(true, 42));
+        } else if (name == Name("null.example.org")) {
+            return (std::pair<bool, int>(true, 13));
+        } else if (name == Name("empty.example.org")) {
+            return (std::pair<bool, int>(true, 0));
+        } else if (name == Name("bad.example.org")) {
+            return (std::pair<bool, int>(true, -1));
         } else {
             return (std::pair<bool, int>(false, 0));
         }
     }
 
+    virtual const std::string& getDBName() const {
+        return (database_name_);
+    }
+
+    // These are just to compile, they won't be called
+    virtual void searchForRecords(int, const std::string&) { }
+    virtual bool getNextRecord(string*, size_t) {
+        return (false);
+    }
+    virtual void resetSearch() { }
+private:
+    const std::string database_name_;
+
+};
+
+/*
+ * A virtual database connection that pretends it contains single zone --
+ * example.org.
+ *
+ * It has the same getZone method as NopConnection, but it provides
+ * implementation of the optional functionality.
+ */
+class MockAccessor : public NopAccessor {
+public:
+    MockAccessor() : search_running_(false)
+    {
+        fillData();
+    }
+private:
+    class MockIteratorContext : public IteratorContext {
+    private:
+        int step;
+    public:
+        MockIteratorContext() :
+            step(0)
+        { }
+        virtual bool getNext(string data[], size_t size) {
+            if (size != DatabaseAccessor::COLUMN_COUNT) {
+                isc_throw(DataSourceError, "Wrong column count in getNextRecord");
+            }
+            switch (step ++) {
+                case 0:
+                    data[DatabaseAccessor::NAME_COLUMN] = "example.org";
+                    data[DatabaseAccessor::TYPE_COLUMN] = "SOA";
+                    data[DatabaseAccessor::TTL_COLUMN] = "300";
+                    data[DatabaseAccessor::RDATA_COLUMN] = "ns1.example.org. admin.example.org. "
+                        "1234 3600 1800 2419200 7200";
+                    return (true);
+                case 1:
+                    data[DatabaseAccessor::NAME_COLUMN] = "x.example.org";
+                    data[DatabaseAccessor::TYPE_COLUMN] = "A";
+                    data[DatabaseAccessor::TTL_COLUMN] = "300";
+                    data[DatabaseAccessor::RDATA_COLUMN] = "192.0.2.1";
+                    return (true);
+                case 2:
+                    data[DatabaseAccessor::NAME_COLUMN] = "x.example.org";
+                    data[DatabaseAccessor::TYPE_COLUMN] = "A";
+                    data[DatabaseAccessor::TTL_COLUMN] = "300";
+                    data[DatabaseAccessor::RDATA_COLUMN] = "192.0.2.2";
+                    return (true);
+                case 3:
+                    data[DatabaseAccessor::NAME_COLUMN] = "x.example.org";
+                    data[DatabaseAccessor::TYPE_COLUMN] = "AAAA";
+                    data[DatabaseAccessor::TTL_COLUMN] = "300";
+                    data[DatabaseAccessor::RDATA_COLUMN] = "2001:db8::1";
+                    return (true);
+                case 4:
+                    data[DatabaseAccessor::NAME_COLUMN] = "x.example.org";
+                    data[DatabaseAccessor::TYPE_COLUMN] = "AAAA";
+                    data[DatabaseAccessor::TTL_COLUMN] = "300";
+                    data[DatabaseAccessor::RDATA_COLUMN] = "2001:db8::2";
+                    return (true);
+                default:
+                    ADD_FAILURE() <<
+                        "Request past the end of iterator context";
+                case 5:
+                    return (false);
+            }
+        }
+    };
+    class EmptyIteratorContext : public IteratorContext {
+    public:
+        virtual bool getNext(string[], size_t size) {
+            if (size != DatabaseAccessor::COLUMN_COUNT) {
+                isc_throw(DataSourceError, "Wrong column count in getNextRecord");
+            }
+            return (false);
+        }
+    };
+    class BadIteratorContext : public IteratorContext {
+    private:
+        int step;
+    public:
+        BadIteratorContext() :
+            step(0)
+        { }
+        virtual bool getNext(string data[], size_t size) {
+            if (size != DatabaseAccessor::COLUMN_COUNT) {
+                isc_throw(DataSourceError, "Wrong column count in getNextRecord");
+            }
+            switch (step ++) {
+                case 0:
+                    data[DatabaseAccessor::NAME_COLUMN] = "x.example.org";
+                    data[DatabaseAccessor::TYPE_COLUMN] = "A";
+                    data[DatabaseAccessor::TTL_COLUMN] = "300";
+                    data[DatabaseAccessor::RDATA_COLUMN] = "192.0.2.1";
+                    return (true);
+                case 1:
+                    data[DatabaseAccessor::NAME_COLUMN] = "x.example.org";
+                    data[DatabaseAccessor::TYPE_COLUMN] = "A";
+                    data[DatabaseAccessor::TTL_COLUMN] = "301";
+                    data[DatabaseAccessor::RDATA_COLUMN] = "192.0.2.2";
+                    return (true);
+                default:
+                    ADD_FAILURE() <<
+                        "Request past the end of iterator context";
+                case 2:
+                    return (false);
+            }
+        }
+    };
+public:
+    virtual IteratorContextPtr getAllRecords(const Name&, int id) const {
+        if (id == 42) {
+            return (IteratorContextPtr(new MockIteratorContext()));
+        } else if (id == 13) {
+            return (IteratorContextPtr());
+        } else if (id == 0) {
+            return (IteratorContextPtr(new EmptyIteratorContext()));
+        } else if (id == -1) {
+            return (IteratorContextPtr(new BadIteratorContext()));
+        } else {
+            isc_throw(isc::Unexpected, "Unknown zone ID");
+        }
+    }
+
     virtual void searchForRecords(int zone_id, const std::string& name) {
         search_running_ = true;
 
@@ -114,10 +254,6 @@ public:
     bool searchRunning() const {
         return (search_running_);
     }
-
-    virtual const std::string& getDBName() const {
-        return (database_name_);
-    }
 private:
     std::map<std::string, std::vector< std::vector<std::string> > > records;
     // used as internal index for getNextRecord()
@@ -135,8 +271,6 @@ private:
     // hardcode some exceptions into getNextRecord
     std::string searched_name_;
 
-    const std::string database_name_;
-
     // Adds one record to the current name in the database
     // The actual data will not be added to 'records' until
     // addCurName() is called
@@ -157,6 +291,11 @@ private:
     // so we can immediately start adding new records.
     void addCurName(const std::string& name) {
         ASSERT_EQ(0, records.count(name));
+        // Append the name to all of them
+        for (std::vector<std::vector<std::string> >::iterator
+             i(cur_name.begin()); i != cur_name.end(); ++ i) {
+            i->push_back(name);
+        }
         records[name] = cur_name;
         cur_name.clear();
     }
@@ -306,6 +445,13 @@ private:
     }
 };
 
+// This tests the default getAllRecords behaviour, throwing NotImplemented
+TEST(DatabaseConnectionTest, getAllRecords) {
+    // The parameters don't matter
+    EXPECT_THROW(NopAccessor().getAllRecords(Name("."), 1),
+                 isc::NotImplemented);
+}
+
 class DatabaseClientTest : public ::testing::Test {
 public:
     DatabaseClientTest() {
@@ -380,7 +526,94 @@ TEST_F(DatabaseClientTest, noAccessorException) {
                  isc::InvalidParameter);
 }
 
-namespace {
+// If the zone doesn't exist, exception is thrown
+TEST_F(DatabaseClientTest, noZoneIterator) {
+    EXPECT_THROW(client_->getIterator(Name("example.com")), DataSourceError);
+}
+
+// If the zone doesn't exist and iteration is not implemented, it still throws
+// the exception it doesn't exist
+TEST_F(DatabaseClientTest, noZoneNotImplementedIterator) {
+    EXPECT_THROW(DatabaseClient(boost::shared_ptr<DatabaseAccessor>(
+        new NopAccessor())).getIterator(Name("example.com")),
+                 DataSourceError);
+}
+
+TEST_F(DatabaseClientTest, notImplementedIterator) {
+    EXPECT_THROW(DatabaseClient(shared_ptr<DatabaseAccessor>(
+        new NopAccessor())).getIterator(Name("example.org")),
+                 isc::NotImplemented);
+}
+
+// Pretend a bug in the connection and pass NULL as the context
+// Should not crash, but gracefully throw
+TEST_F(DatabaseClientTest, nullIteratorContext) {
+    EXPECT_THROW(client_->getIterator(Name("null.example.org")),
+                 isc::Unexpected);
+}
+
+// It doesn't crash or anything if the zone is completely empty
+TEST_F(DatabaseClientTest, emptyIterator) {
+    ZoneIteratorPtr it(client_->getIterator(Name("empty.example.org")));
+    EXPECT_EQ(ConstRRsetPtr(), it->getNextRRset());
+    // This is past the end, it should throw
+    EXPECT_THROW(it->getNextRRset(), isc::Unexpected);
+}
+
+// Iterate trough a zone
+TEST_F(DatabaseClientTest, iterator) {
+    ZoneIteratorPtr it(client_->getIterator(Name("example.org")));
+    ConstRRsetPtr rrset(it->getNextRRset());
+    ASSERT_NE(ConstRRsetPtr(), rrset);
+    EXPECT_EQ(Name("example.org"), rrset->getName());
+    EXPECT_EQ(RRClass::IN(), rrset->getClass());
+    EXPECT_EQ(RRType::SOA(), rrset->getType());
+    EXPECT_EQ(RRTTL(300), rrset->getTTL());
+    RdataIteratorPtr rit(rrset->getRdataIterator());
+    ASSERT_FALSE(rit->isLast());
+    rit->next();
+    EXPECT_TRUE(rit->isLast());
+
+    rrset = it->getNextRRset();
+    ASSERT_NE(ConstRRsetPtr(), rrset);
+    EXPECT_EQ(Name("x.example.org"), rrset->getName());
+    EXPECT_EQ(RRClass::IN(), rrset->getClass());
+    EXPECT_EQ(RRType::A(), rrset->getType());
+    EXPECT_EQ(RRTTL(300), rrset->getTTL());
+    rit = rrset->getRdataIterator();
+    ASSERT_FALSE(rit->isLast());
+    EXPECT_EQ("192.0.2.1", rit->getCurrent().toText());
+    rit->next();
+    ASSERT_FALSE(rit->isLast());
+    EXPECT_EQ("192.0.2.2", rit->getCurrent().toText());
+    rit->next();
+    EXPECT_TRUE(rit->isLast());
+
+    rrset = it->getNextRRset();
+    ASSERT_NE(ConstRRsetPtr(), rrset);
+    EXPECT_EQ(Name("x.example.org"), rrset->getName());
+    EXPECT_EQ(RRClass::IN(), rrset->getClass());
+    EXPECT_EQ(RRType::AAAA(), rrset->getType());
+    EXPECT_EQ(RRTTL(300), rrset->getTTL());
+    EXPECT_EQ(ConstRRsetPtr(), it->getNextRRset());
+    rit = rrset->getRdataIterator();
+    ASSERT_FALSE(rit->isLast());
+    EXPECT_EQ("2001:db8::1", rit->getCurrent().toText());
+    rit->next();
+    ASSERT_FALSE(rit->isLast());
+    EXPECT_EQ("2001:db8::2", rit->getCurrent().toText());
+    rit->next();
+    EXPECT_TRUE(rit->isLast());
+}
+
+// This has inconsistent TTL in the set (the rest, like nonsense in
+// the data is handled in rdata itself).
+TEST_F(DatabaseClientTest, badIterator) {
+    // It should not throw, but get the lowest one of them
+    ZoneIteratorPtr it(client_->getIterator(Name("bad.example.org")));
+    EXPECT_EQ(it->getNextRRset()->getTTL(), isc::dns::RRTTL(300));
+}
+
 // checks if the given rrset matches the
 // given name, class, type and rdatas
 void
@@ -433,7 +666,6 @@ doFindTest(shared_ptr<DatabaseClient::Finder> finder,
         EXPECT_EQ(isc::dns::RRsetPtr(), result.rrset);
     }
 }
-} // end anonymous namespace
 
 TEST_F(DatabaseClientTest, find) {
     shared_ptr<DatabaseClient::Finder> finder(getFinder());

src/lib/datasrc/tests/memory_datasrc_unittest.cc

@@ -29,6 +29,8 @@
 #include <dns/masterload.h>
 
 #include <datasrc/memory_datasrc.h>
+#include <datasrc/data_source.h>
+#include <datasrc/iterator.h>
 
 #include <gtest/gtest.h>
 
@@ -138,6 +140,43 @@ TEST_F(InMemoryClientTest, add_find_Zone) {
                   getOrigin());
 }
 
+TEST_F(InMemoryClientTest, iterator) {
+    // Just some preparations of data
+    boost::shared_ptr<InMemoryZoneFinder>
+        zone(new InMemoryZoneFinder(RRClass::IN(), Name("a")));
+    RRsetPtr aRRsetA(new RRset(Name("a"), RRClass::IN(), RRType::A(),
+                                  RRTTL(300)));
+    aRRsetA->addRdata(rdata::in::A("192.0.2.1"));
+    RRsetPtr aRRsetAAAA(new RRset(Name("a"), RRClass::IN(), RRType::AAAA(),
+                                  RRTTL(300)));
+    aRRsetAAAA->addRdata(rdata::in::AAAA("2001:db8::1"));
+    aRRsetAAAA->addRdata(rdata::in::AAAA("2001:db8::2"));
+    RRsetPtr subRRsetA(new RRset(Name("sub.x.a"), RRClass::IN(), RRType::A(),
+                                  RRTTL(300)));
+    subRRsetA->addRdata(rdata::in::A("192.0.2.2"));
+    EXPECT_EQ(result::SUCCESS, memory_client.addZone(zone));
+    // First, the zone is not there, so it should throw
+    EXPECT_THROW(memory_client.getIterator(Name("b")), DataSourceError);
+    // This zone is not there either, even when there's a zone containing this
+    EXPECT_THROW(memory_client.getIterator(Name("x.a")), DataSourceError);
+    // Now, an empty zone
+    ZoneIteratorPtr iterator(memory_client.getIterator(Name("a")));
+    EXPECT_EQ(ConstRRsetPtr(), iterator->getNextRRset());
+    // It throws Unexpected when we are past the end
+    EXPECT_THROW(iterator->getNextRRset(), isc::Unexpected);
+    EXPECT_EQ(result::SUCCESS, zone->add(aRRsetA));
+    EXPECT_EQ(result::SUCCESS, zone->add(aRRsetAAAA));
+    EXPECT_EQ(result::SUCCESS, zone->add(subRRsetA));
+    // Check it with full zone, one by one.
+    // It should be in ascending order in case of InMemory data source
+    // (isn't guaranteed in general)
+    iterator = memory_client.getIterator(Name("a"));
+    EXPECT_EQ(aRRsetA, iterator->getNextRRset());
+    EXPECT_EQ(aRRsetAAAA, iterator->getNextRRset());
+    EXPECT_EQ(subRRsetA, iterator->getNextRRset());
+    EXPECT_EQ(ConstRRsetPtr(), iterator->getNextRRset());
+}
+
 TEST_F(InMemoryClientTest, getZoneCount) {
     EXPECT_EQ(0, memory_client.getZoneCount());
     memory_client.addZone(

src/lib/datasrc/tests/sqlite3_accessor_unittest.cc

@@ -76,8 +76,8 @@ public:
     void initAccessor(const std::string& filename, const RRClass& rrclass) {
         db.reset(new SQLite3Database(filename, rrclass));
     }
-    // The tested dbection
-    boost::scoped_ptr<SQLite3Database> db;
+    // The tested db
+    boost::shared_ptr<SQLite3Database> db;
 };
 
 // This zone exists in the data, so it should be found
@@ -103,6 +103,47 @@ TEST_F(SQLite3Access, noClass) {
     EXPECT_FALSE(db->getZone(Name("example.com")).first);
 }
 
+// This tests the iterator context
+TEST_F(SQLite3Access, iterator) {
+    // Our test zone is conveniently small, but not empty
+    initAccessor(SQLITE_DBFILE_EXAMPLE2, RRClass::IN());
+
+    // Get the iterator context
+    DatabaseAccessor::IteratorContextPtr
+        context(db->getAllRecords(Name("example2.com"), 1));
+    ASSERT_NE(DatabaseAccessor::IteratorContextPtr(),
+              context);
+
+    const size_t size(5);
+    std::string data[size];
+    // Get and check the first and only record
+    EXPECT_TRUE(context->getNext(data, size));
+    EXPECT_EQ("example2.com.", data[4]);
+    EXPECT_EQ("SOA", data[0]);
+    EXPECT_EQ("master.example2.com. admin.example2.com. "
+              "1234 3600 1800 2419200 7200", data[3]);
+    EXPECT_EQ("3600", data[1]);
+    // Check there's no other
+    EXPECT_FALSE(context->getNext(data, size));
+}
+
+TEST_F(SQLite3Access, iteratorColumnCount) {
+    // Our test zone is conveniently small, but not empty
+    initAccessor(SQLITE_DBFILE_EXAMPLE2, RRClass::IN());
+
+    // Get the iterator context
+    DatabaseAccessor::IteratorContextPtr
+        context(db->getAllRecords(Name("example2.com"), 1));
+    ASSERT_NE(DatabaseAccessor::IteratorContextPtr(),
+              context);
+
+    EXPECT_THROW(context->getNext(NULL, 0), DataSourceError);
+    std::string data[6];
+    EXPECT_THROW(context->getNext(data, 4), DataSourceError);
+    EXPECT_THROW(context->getNext(data, 6), DataSourceError);
+    EXPECT_NO_THROW(context->getNext(data, 5));
+}
+
 TEST(SQLite3Open, getDBNameExample2) {
     SQLite3Database db(SQLITE_DBFILE_EXAMPLE2, RRClass::IN());
     EXPECT_EQ(SQLITE_DBNAME_EXAMPLE2, db.getDBName());
@@ -120,12 +161,14 @@ checkRecordRow(const std::string columns[],
                const std::string& field0,
                const std::string& field1,
                const std::string& field2,
-               const std::string& field3)
+               const std::string& field3,
+               const std::string& field4)
 {
     EXPECT_EQ(field0, columns[0]);
     EXPECT_EQ(field1, columns[1]);
     EXPECT_EQ(field2, columns[2]);
     EXPECT_EQ(field3, columns[3]);
+    EXPECT_EQ(field4, columns[4]);
 }
 
 TEST_F(SQLite3Access, getRecords) {
@@ -140,83 +183,89 @@ TEST_F(SQLite3Access, getRecords) {
 
     // without search, getNext() should return false
     EXPECT_FALSE(db->getNextRecord(columns, column_count));
-    checkRecordRow(columns, "", "", "", "");
+    checkRecordRow(columns, "", "", "", "", "");
 
     db->searchForRecords(zone_id, "foo.bar.");
     EXPECT_FALSE(db->getNextRecord(columns, column_count));
-    checkRecordRow(columns, "", "", "", "");
+    checkRecordRow(columns, "", "", "", "", "");
 
     db->searchForRecords(zone_id, "");
     EXPECT_FALSE(db->getNextRecord(columns, column_count));
-    checkRecordRow(columns, "", "", "", "");
+    checkRecordRow(columns, "", "", "", "", "");
 
     // Should error on a bad number of columns
-    EXPECT_THROW(db->getNextRecord(columns, 3), DataSourceError);
-    EXPECT_THROW(db->getNextRecord(columns, 5), DataSourceError);
+    EXPECT_THROW(db->getNextRecord(columns, 4), DataSourceError);
+    EXPECT_THROW(db->getNextRecord(columns, 6), DataSourceError);
 
     // now try some real searches
     db->searchForRecords(zone_id, "foo.example.com.");
     ASSERT_TRUE(db->getNextRecord(columns, column_count));
     checkRecordRow(columns, "CNAME", "3600", "",
-                   "cnametest.example.org.");
+                   "cnametest.example.org.", "foo.example.com.");
     ASSERT_TRUE(db->getNextRecord(columns, column_count));
     checkRecordRow(columns, "RRSIG", "3600", "CNAME",
                    "CNAME 5 3 3600 20100322084538 20100220084538 33495 "
-                   "example.com. FAKEFAKEFAKEFAKE");
+                   "example.com. FAKEFAKEFAKEFAKE", "foo.example.com.");
     ASSERT_TRUE(db->getNextRecord(columns, column_count));
     checkRecordRow(columns, "NSEC", "7200", "",
-                   "mail.example.com. CNAME RRSIG NSEC");
+                   "mail.example.com. CNAME RRSIG NSEC", "foo.example.com.");
     ASSERT_TRUE(db->getNextRecord(columns, column_count));
     checkRecordRow(columns, "RRSIG", "7200", "NSEC",
                    "NSEC 5 3 7200 20100322084538 20100220084538 33495 "
-                   "example.com. FAKEFAKEFAKEFAKE");
+                   "example.com. FAKEFAKEFAKEFAKE", "foo.example.com.");
     EXPECT_FALSE(db->getNextRecord(columns, column_count));
     // with no more records, the array should not have been modified
     checkRecordRow(columns, "RRSIG", "7200", "NSEC",
                    "NSEC 5 3 7200 20100322084538 20100220084538 33495 "
-                   "example.com. FAKEFAKEFAKEFAKE");
+                   "example.com. FAKEFAKEFAKEFAKE", "foo.example.com.");
 
     db->searchForRecords(zone_id, "example.com.");
     ASSERT_TRUE(db->getNextRecord(columns, column_count));
     checkRecordRow(columns, "SOA", "3600", "",
                    "master.example.com. admin.example.com. "
-                   "1234 3600 1800 2419200 7200");
+                   "1234 3600 1800 2419200 7200", "example.com.");
     ASSERT_TRUE(db->getNextRecord(columns, column_count));
     checkRecordRow(columns, "RRSIG", "3600", "SOA",
                    "SOA 5 2 3600 20100322084538 20100220084538 "
-                   "33495 example.com. FAKEFAKEFAKEFAKE");
+                   "33495 example.com. FAKEFAKEFAKEFAKE", "example.com.");
     ASSERT_TRUE(db->getNextRecord(columns, column_count));
-    checkRecordRow(columns, "NS", "1200", "", "dns01.example.com.");
+    checkRecordRow(columns, "NS", "1200", "", "dns01.example.com.",
+                   "example.com.");
     ASSERT_TRUE(db->getNextRecord(columns, column_count));
-    checkRecordRow(columns, "NS", "3600", "", "dns02.example.com.");
+    checkRecordRow(columns, "NS", "3600", "", "dns02.example.com.",
+                   "example.com.");
     ASSERT_TRUE(db->getNextRecord(columns, column_count));
-    checkRecordRow(columns, "NS", "1800", "", "dns03.example.com.");
+    checkRecordRow(columns, "NS", "1800", "", "dns03.example.com.",
+                   "example.com.");
     ASSERT_TRUE(db->getNextRecord(columns, column_count));
     checkRecordRow(columns, "RRSIG", "3600", "NS",
                    "NS 5 2 3600 20100322084538 20100220084538 "
-                   "33495 example.com. FAKEFAKEFAKEFAKE");
+                   "33495 example.com. FAKEFAKEFAKEFAKE",
+                   "example.com.");
     ASSERT_TRUE(db->getNextRecord(columns, column_count));
-    checkRecordRow(columns, "MX", "3600", "", "10 mail.example.com.");
+    checkRecordRow(columns, "MX", "3600", "", "10 mail.example.com.",
+                   "example.com.");
     ASSERT_TRUE(db->getNextRecord(columns, column_count));
     checkRecordRow(columns, "MX", "3600", "",
-                   "20 mail.subzone.example.com.");
+                   "20 mail.subzone.example.com.", "example.com.");
     ASSERT_TRUE(db->getNextRecord(columns, column_count));
     checkRecordRow(columns, "RRSIG", "3600", "MX",
                    "MX 5 2 3600 20100322084538 20100220084538 "
-                   "33495 example.com. FAKEFAKEFAKEFAKE");
+                   "33495 example.com. FAKEFAKEFAKEFAKE", "example.com.");
     ASSERT_TRUE(db->getNextRecord(columns, column_count));
     checkRecordRow(columns, "NSEC", "7200", "",
-                   "cname-ext.example.com. NS SOA MX RRSIG NSEC DNSKEY");
+                   "cname-ext.example.com. NS SOA MX RRSIG NSEC DNSKEY",
+                   "example.com.");
     ASSERT_TRUE(db->getNextRecord(columns, column_count));
     checkRecordRow(columns, "RRSIG", "7200", "NSEC",
                    "NSEC 5 2 7200 20100322084538 20100220084538 "
-                   "33495 example.com. FAKEFAKEFAKEFAKE");
+                   "33495 example.com. FAKEFAKEFAKEFAKE", "example.com.");
     ASSERT_TRUE(db->getNextRecord(columns, column_count));
     checkRecordRow(columns, "DNSKEY", "3600", "",
                    "256 3 5 AwEAAcOUBllYc1hf7ND9uDy+Yz1BF3sI0m4q NGV7W"
                    "cTD0WEiuV7IjXgHE36fCmS9QsUxSSOV o1I/FMxI2PJVqTYHkX"
                    "FBS7AzLGsQYMU7UjBZ SotBJ6Imt5pXMu+lEDNy8TOUzG3xm7g"
-                   "0qcbW YF6qCEfvZoBtAqi5Rk7Mlrqs8agxYyMx");
+                   "0qcbW YF6qCEfvZoBtAqi5Rk7Mlrqs8agxYyMx", "example.com.");
     ASSERT_TRUE(db->getNextRecord(columns, column_count));
     checkRecordRow(columns, "DNSKEY", "3600", "",
                    "257 3 5 AwEAAe5WFbxdCPq2jZrZhlMj7oJdff3W7syJ tbvzg"
@@ -226,20 +275,20 @@ TEST_F(SQLite3Access, getRecords) {
                    "qiODyNZYQ+ZrLmF0KIJ2yPN3iO6Zq 23TaOrVTjB7d1a/h31OD"
                    "fiHAxFHrkY3t3D5J R9Nsl/7fdRmSznwtcSDgLXBoFEYmw6p86"
                    "Acv RyoYNcL1SXjaKVLG5jyU3UR+LcGZT5t/0xGf oIK/aKwEN"
-                   "rsjcKZZj660b1M=");
+                   "rsjcKZZj660b1M=", "example.com.");
     ASSERT_TRUE(db->getNextRecord(columns, column_count));
     checkRecordRow(columns, "RRSIG", "3600", "DNSKEY",
                    "DNSKEY 5 2 3600 20100322084538 20100220084538 "
-                   "4456 example.com. FAKEFAKEFAKEFAKE");
+                   "4456 example.com. FAKEFAKEFAKEFAKE", "example.com.");
     ASSERT_TRUE(db->getNextRecord(columns, column_count));
     checkRecordRow(columns, "RRSIG", "3600", "DNSKEY",
                    "DNSKEY 5 2 3600 20100322084538 20100220084538 "
-                   "33495 example.com. FAKEFAKEFAKEFAKE");
+                   "33495 example.com. FAKEFAKEFAKEFAKE", "example.com.");
     EXPECT_FALSE(db->getNextRecord(columns, column_count));
     // getnextrecord returning false should mean array is not altered
     checkRecordRow(columns, "RRSIG", "3600", "DNSKEY",
                    "DNSKEY 5 2 3600 20100322084538 20100220084538 "
-                   "33495 example.com. FAKEFAKEFAKEFAKE");
+                   "33495 example.com. FAKEFAKEFAKEFAKE", "example.com.");
 }
 
 } // end anonymous namespace

src/lib/exceptions/exceptions.h

@@ -137,6 +137,18 @@ public:
 };
 
 ///
+/// \brief A generic exception that is thrown when a function is
+/// not implemented.
+///
+/// This may be due to unfinished implementation or in case the
+/// function isn't even planned to be provided for that situation.
+class NotImplemented : public Exception {
+public:
+    NotImplemented(const char* file, size_t line, const char* what) :
+        isc::Exception(file, line, what) {}
+};
+
+///
 /// A shortcut macro to insert known values into exception arguments.
 ///
 /// It allows the \c stream argument to be part of a statement using an