[1261] added a test using a workable sqlite3 DB file and its data source

client to see it really works for an operational environment (and indeed
I found a bug in the Diff class).  To make it possible, also updated
how to set the data source client for XfrinConnection.

src/bin/xfrin/tests/Makefile.am

@@ -20,5 +20,7 @@ endif
 	echo Running test: $$pytest ; \
 	$(LIBRARY_PATH_PLACEHOLDER) \
 	PYTHONPATH=$(abs_top_builddir)/src/lib/dns/python/.libs:$(abs_top_builddir)/src/bin/xfrin:$(COMMON_PYTHON_PATH) \
+	TESTDATASRCDIR=$(abs_top_srcdir)/src/bin/xfrin/tests/testdata/ \
+	TESTDATAOBJDIR=$(abs_top_objdir)/src/bin/xfrin/tests/testdata/ \
 	$(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \
 	done

src/bin/xfrin/tests/testdata/example.com

@@ -0,0 +1,17 @@
+;; This is a simplest form of zone file for 'example.com', which is the
+;; source of the corresponding sqlite3 DB file.  This file is provided
+;; for reference purposes only; it's not actually used anywhere.
+
+example.com.		3600	IN SOA	master.example.com. admin.example.com. (
+					1230       ; serial
+					3600       ; refresh (1 hour)
+					1800       ; retry (30 minutes)
+					2419200    ; expire (4 weeks)
+					7200       ; minimum (2 hours)
+					)
+			3600	NS	dns01.example.com.
+			3600	NS	dns02.example.com.
+			3600	NS	dns03.example.com.
+dns01.example.com.	3600	IN A	192.0.2.1
+dns02.example.com.	3600	IN A	192.0.2.2
+dns03.example.com.	3600	IN A	192.0.2.3

src/bin/xfrin/tests/xfrin_test.py

@@ -14,6 +14,7 @@
 # WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
 import unittest
+import shutil
 import socket
 import io
 from isc.testutils.tsigctx_mock import MockTSIGContext
@@ -37,6 +38,9 @@ TEST_MASTER_IPV6_ADDRESS = '::1'
 TEST_MASTER_IPV6_ADDRINFO = (socket.AF_INET6, socket.SOCK_STREAM,
                              socket.IPPROTO_TCP, '',
                              (TEST_MASTER_IPV6_ADDRESS, 53))
+
+TESTDATA_SRCDIR = os.getenv("TESTDATASRCDIR")
+TESTDATA_OBJDIR = os.getenv("TESTDATAOBJDIR")
 # XXX: This should be a non priviledge port that is unlikely to be used.
 # If some other process uses this port test will fail.
 TEST_MASTER_PORT = '53535'
@@ -127,7 +131,7 @@ class MockDataSourceClient():
     def add_rrset(self, rrset):
         self.diffs.append(('add', rrset))
 
-    def remove_rrset(self, rrset):
+    def delete_rrset(self, rrset):
         self.diffs.append(('remove', rrset))
 
     def commit(self):
@@ -161,15 +165,15 @@ class MockXfrin(Xfrin):
         # method in the superclass
         self.xfrin_started_master_addr = master_addrinfo[2][0]
         self.xfrin_started_master_port = master_addrinfo[2][1]
-        return Xfrin.xfrin_start(self, zone_name, rrclass, db_file,
+        return Xfrin.xfrin_start(self, zone_name, rrclass, None,
                                  master_addrinfo, tsig_key,
                                  check_soa)
 
 class MockXfrinConnection(XfrinConnection):
     def __init__(self, sock_map, zone_name, rrclass, db_file, shutdown_event,
                  master_addr):
-        super().__init__(sock_map, zone_name, rrclass, db_file, shutdown_event,
-                         master_addr)
+        super().__init__(sock_map, zone_name, rrclass, MockDataSourceClient(),
+                         db_file, shutdown_event, master_addr)
         self.query_data = b''
         self.reply_data = b''
         self.force_time_out = False
@@ -178,11 +182,6 @@ class MockXfrinConnection(XfrinConnection):
         self.qid = None
         self.response_generator = None
 
-    # The following three implement a simplified mock of DataSourceClient
-    # and ZoneFinder classes for testing purposes.
-    def _get_datasrc_client(self, rrclass):
-        return MockDataSourceClient()
-
     def _asyncore_loop(self):
         if self.force_close:
             self.handle_close()
@@ -634,8 +633,8 @@ class TestAXFR(TestXfrinConnection):
         c.close()
 
     def test_init_chclass(self):
-        c = XfrinConnection({}, TEST_ZONE_NAME, RRClass.CH(), TEST_DB_FILE,
-                            threading.Event(), TEST_MASTER_IPV4_ADDRINFO)
+        c = MockXfrinConnection({}, TEST_ZONE_NAME, RRClass.CH(), TEST_DB_FILE,
+                                threading.Event(), TEST_MASTER_IPV4_ADDRINFO)
         axfrmsg = c._create_query(RRType.AXFR())
         self.assertEqual(axfrmsg.get_question()[0].get_class(),
                          RRClass.CH())
@@ -1200,6 +1199,49 @@ class TestIXFRSession(TestXfrinConnection):
         self._create_broken_response_data()
         self.assertEqual(XFRIN_FAIL, self.conn.do_xfrin(False, True))
 
+class TestIXFRSessionWithSQLite3(TestXfrinConnection):
+    '''Tests for IXFR sessions using an SQLite3 DB.
+
+    These are provided mainly to confirm the implementation actually works
+    in an environment closer to actual operational environments.  So we
+    only check a few common cases; other details are tested using mock
+    data sources.
+
+    '''
+    def setUp(self):
+        self.sqlite3db_src = TESTDATA_SRCDIR + '/example.com.sqlite3'
+        self.sqlite3db_obj = TESTDATA_SRCDIR + '/example.com.sqlite3.copy'
+        super().setUp()
+        if os.path.exists(self.sqlite3db_obj):
+            os.unlink(self.sqlite3db_obj)
+        shutil.copyfile(self.sqlite3db_src, self.sqlite3db_obj)
+        self.conn._datasrc_client = DataSourceClient(self.sqlite3db_obj)
+
+    def tearDown(self):
+        if os.path.exists(self.sqlite3db_obj):
+            os.unlink(self.sqlite3db_obj)
+
+    def get_zone_serial(self):
+        result, finder = self.conn._datasrc_client.find_zone(TEST_ZONE_NAME)
+        self.assertEqual(DataSourceClient.SUCCESS, result)
+        result, soa = finder.find(TEST_ZONE_NAME, RRType.SOA(),
+                                  None, ZoneFinder.FIND_DEFAULT)
+        self.assertEqual(ZoneFinder.SUCCESS, result)
+        self.assertEqual(1, soa.get_rdata_count())
+        return get_soa_serial(soa.get_rdata()[0])
+
+    def test_do_xfrin_sqlite3(self):
+        def create_ixfr_response():
+            self.conn.reply_data = self.conn.create_response_data(
+                questions=[Question(TEST_ZONE_NAME, TEST_RRCLASS,
+                                    RRType.IXFR())],
+                answers=[soa_rrset, begin_soa_rrset, soa_rrset, soa_rrset])
+        self.conn.response_generator = create_ixfr_response
+
+        self.assertEqual(1230, self.get_zone_serial())
+        self.assertEqual(XFRIN_OK, self.conn.do_xfrin(False, True))
+        self.assertEqual(1234, self.get_zone_serial())
+
 class TestXfrinRecorder(unittest.TestCase):
     def setUp(self):
         self.recorder = XfrinRecorder()

src/bin/xfrin/xfrin.py.in

@@ -217,7 +217,7 @@ class XfrinIXFRDeleteSOA(XfrinState):
         # This is the beginning state of one difference sequence (changes
         # for one SOA update).  We need to create a new Diff object now.
         conn._diff = Diff(conn._datasrc_client, conn._zone_name)
-        conn._diff.remove_data(rr)
+        conn._diff.delete_data(rr)
         self.set_xfrstate(conn, XfrinIXFRDelete())
         return True
 
@@ -228,7 +228,7 @@ class XfrinIXFRDelete(XfrinState):
             conn._current_serial = get_soa_serial(rr.get_rdata()[0])
             self.set_xfrstate(conn, XfrinIXFRAddSOA())
             return False
-        conn._diff.remove_data(rr)
+        conn._diff.delete_data(rr)
         return True
 
 class XfrinIXFRAddSOA(XfrinState):
@@ -285,12 +285,16 @@ class XfrinConnection(asyncore.dispatcher):
     '''Do xfrin in this class. '''
 
     def __init__(self,
-                 sock_map, zone_name, rrclass, db_file, shutdown_event,
-                 master_addrinfo, tsig_key = None, verbose = False,
-                 idle_timeout = 60):
-        ''' idle_timeout: max idle time for read data from socket.
-            db_file: specify the data source file.
-            check_soa: when it's true, check soa first before sending xfr query
+                 sock_map, zone_name, rrclass, datasrc_client, db_file,
+                 shutdown_event, master_addrinfo, tsig_key = None,
+                 verbose=False, idle_timeout=60):
+        '''Constructor of the XfirnConnection class.
+
+        idle_timeout: max idle time for read data from socket.
+        datasrc_client: the data source client object used for the XFR session.
+                        This will eventually replace db_file completely.
+        db_file: specify the data source file (should soon be deprecated).
+
         '''
 
         asyncore.dispatcher.__init__(self, map=sock_map)
@@ -306,7 +310,7 @@ class XfrinConnection(asyncore.dispatcher):
 
         # Data source handlers
         self._db_file = db_file # temporary for sqlite3 specific code
-        self._datasrc_client = self._get_datasrc_client(rrclass)
+        self._datasrc_client = datasrc_client
 
         self.create_socket(master_addrinfo[0], master_addrinfo[1])
         self._sock_map = sock_map
@@ -325,10 +329,6 @@ class XfrinConnection(asyncore.dispatcher):
     def __create_tsig_ctx(self, key):
         return TSIGContext(key)
 
-    def _get_datasrc_client(self, rrclass):
-        # TODO: create a DataSourceClient assuming the sqlite3 backend for now
-        return None
-
     def __set_xfrstate(self, new_state):
         self.__state = new_state
 
@@ -658,9 +658,21 @@ def process_xfrin(server, xfrin_recorder, zone_name, rrclass, db_file,
                   shutdown_event, master_addrinfo, check_soa, verbose,
                   tsig_key):
     xfrin_recorder.increment(zone_name)
+
+    # Create a data source client used in this XFR session.  Right now we
+    # still assume an sqlite3-based data source, and use both the old and new
+    # data source APIs.  We also need to use a mock client for tests.
+    # For a temporary workaround to deal with these situations, we skip the
+    # creation when the given file is none (the test case).  Eventually
+    # this code will be much cleaner.
+    datasrc_client = None
+    if db_file is not None:
+        datasrc_client = DataSourceClient(db_file)
+
+    # Create a TCP connection for the XFR session and perform the operation.
     sock_map = {}
-    conn = XfrinConnection(sock_map, zone_name, rrclass, db_file,
-                           shutdown_event, master_addrinfo,
+    conn = XfrinConnection(sock_map, zone_name, rrclass, datasrc_client,
+                           db_file, shutdown_event, master_addrinfo,
                            tsig_key, verbose)
     ret = XFRIN_FAIL
     if conn.connect_to_master():
@@ -1045,8 +1057,8 @@ class Xfrin:
         while not self._shutdown_event.is_set():
             self._cc_check_command()
 
-    def xfrin_start(self, zone_name, rrclass, db_file, master_addrinfo, tsig_key,
-                    check_soa = True):
+    def xfrin_start(self, zone_name, rrclass, db_file, master_addrinfo,
+                    tsig_key, check_soa=True):
         if "pydnspp" not in sys.modules:
             return (1, "xfrin failed, can't load dns message python library: 'pydnspp'")