[master]Merge branch 'master' of ssh://git.bind10.isc.org/var/bind10/git/bind10

ChangeLog

@@ -1,3 +1,26 @@
+579.	[bug]		jinmei
+	libdatasrc/b10-auth: corrected some corner cases in query handling
+	of in-memory data source that led to the following invalid/odd
+	responses from b10-auth:
+	- duplicate RRs in answer and additional for type ANY query
+	- incorrect NSEC for no error, no data (NXRRSET) response that
+	  matches a wildcard
+	(Trac #2585, git abe78fae4ba3aca5eb01806dd4e05607b1241745)
+
+578.	[bug]		jinmei
+	b10-auth now returns closest encloser NSEC3 proof to queries for
+	an empty non terminal derived from an Opt-Out NSEC RR, as clarified
+	in errata 3441 for RFC5155.  Previously it regarded such case as
+	broken zone and returned SERVFAIL.
+	(Trac #2659, git 24c235cb1b379c6472772d340e21577c3460b742)
+
+577.	[func]		muks
+	Added an SQLite3 index on records(rname, rdtype). This decreases
+	insert performance by ~28% and adds about ~20% to the file size,
+	but increases zone iteration performance. As it introduces a new
+	index, a database upgrade would be required.
+	(Trac #1756, git 9b3c959af13111af1fa248c5010aa33ee7e307ee)
+
 576.	[bug]		tmark, tomek
 	b10-dhcp6: Fixed bug when the server aborts operation when
 	receiving renew and there are no IPv6 subnets configured.

src/bin/auth/query.cc

@@ -298,14 +298,18 @@ Query::addNXRRsetProof(ZoneFinder& finder,
             addWildcardNXRRSETProof(finder, db_context.rrset);
         }
     } else if (db_context.isNSEC3Signed() && !db_context.isWildcard()) {
-        if (*qtype_ == RRType::DS()) {
-            // RFC 5155, Section 7.2.4.  Add either NSEC3 for the qname or
-            // closest (provable) encloser proof in case of optout.
-            addClosestEncloserProof(finder, *qname_, true);
-        } else {
-            // RFC 5155, Section 7.2.3.  Just add NSEC3 for the qname.
-            addNSEC3ForName(finder, *qname_, true);
-        }
+        // Section 7.2.3 and 7.2.4 of RFC 5155 with clarification by errata
+        // http://www.rfc-editor.org/errata_search.php?rfc=5155&eid=3441
+        // In the end, these two cases are basically the same: if the qname is
+        // equal to or derived from insecure delegation covered by an Opt-Out
+        // NSEC3 RR, include the closest provable encloser proof; otherwise we
+        // have a matching NSEC3, so we include it.
+        //
+        // Note: This implementation does not check in the former case whether
+        // the NSEC3 for the next closer has Opt-Out bit on; this must be the
+        // case as long as the zone is correctly signed, and if it's broken
+        // we'd just return what we are given and have the validator detect it.
+        addClosestEncloserProof(finder, *qname_, true);
     } else if (db_context.isNSEC3Signed() && db_context.isWildcard()) {
         // Case for RFC 5155 Section 7.2.5: add closest encloser proof for the
         // qname, construct the matched wildcard name and add NSEC3 for it.

src/bin/auth/tests/query_unittest.cc

@@ -217,6 +217,13 @@ public:
             "t644ebqk9bibcna874givr6joj62mlhv";
         hash_map_[Name("www1.uwild.example.com")] =
             "q04jkcevqvmu85r014c7dkba38o0ji6r"; // a bit larger than H(www)
+
+        // For empty-non-terminal derived from insecure delegation (we don't
+        // need a hash for the delegation point itself for that test).  the
+        // hash for empty name is the same as that for unsigned-delegation
+        // above, as the case is similar to that.
+        hash_map_[Name("empty.example.com")] =
+            "q81r598950igr1eqvc60aedlq66425b5"; // a bit larger than H(www)
     }
     virtual string calculate(const Name& name) const {
         const NSEC3HashMap::const_iterator found = hash_map_.find(name);
@@ -262,8 +269,6 @@ public:
 // to child zones are identified by the existence of non origin NS records.
 // Another special name is "dname.example.com".  Query names under this name
 // will result in DNAME.
-// This mock zone doesn't handle empty non terminal nodes (if we need to test
-// such cases find() should have specialized code for it).
 class MockZoneFinder : public ZoneFinder {
 public:
     MockZoneFinder() :
@@ -1162,12 +1167,6 @@ TEST_P(QueryTest, apexNSMatch) {
 
 // test type any query logic
 TEST_P(QueryTest, exactAnyMatch) {
-    // This is an in-memory specific bug (#2585), until it's fixed we
-    // tentatively skip the test for in-memory
-    if (GetParam() == INMEMORY) {
-        return;
-    }
-
     // find match rrset, omit additional data which has already been provided
     // in the answer section from the additional.
     EXPECT_NO_THROW(query.process(*list_, Name("noglue.example.com"),
@@ -1373,17 +1372,11 @@ TEST_P(QueryTest, nxdomainWithNSEC) {
 }
 
 TEST_P(QueryTest, nxdomainWithNSEC2) {
-    // there seems to be a bug in the SQLite3 (or database in general) data
-    // source and this doesn't work (Trac #2586).
-    if (GetParam() == SQLITE3) {
-        return;
-    }
-
     // See comments about no_txt.  In this case the best possible wildcard
     // is derived from the next domain of the NSEC that proves NXDOMAIN, and
     // the NSEC to provide the non existence of wildcard is different from
     // the first NSEC.
-    query.process(*list_, Name("(.no.example.com"), qtype, response,
+    query.process(*list_, Name("!.no.example.com"), qtype, response,
                   true);
     responseCheck(response, Rcode::NXDOMAIN(), AA_FLAG, 0, 6, 0,
                   NULL, (string(soa_minttl_txt) +
@@ -1393,19 +1386,12 @@ TEST_P(QueryTest, nxdomainWithNSEC2) {
                          string("mx.example.com. 3600 IN RRSIG ") +
                          getCommonRRSIGText("NSEC") + "\n" +
                          string(nsec_no_txt) + "\n" +
-                         string(").no.example.com. 3600 IN RRSIG ") +
+                         string("&.no.example.com. 3600 IN RRSIG ") +
                          getCommonRRSIGText("NSEC")).c_str(),
                   NULL, mock_finder->getOrigin());
 }
 
 TEST_P(QueryTest, nxdomainWithNSECDuplicate) {
-    // there seems to be a bug in the SQLite3 (or database in general) data
-    // source and this doesn't work.  This is probably the same type of bug
-    // as nxdomainWithNSEC2 (Trac #2586).
-    if (GetParam() == SQLITE3) {
-        return;
-    }
-
     // See comments about nz_txt.  In this case we only need one NSEC,
     // which proves both NXDOMAIN and the non existence of wildcard.
     query.process(*list_, Name("nx.no.example.com"), qtype, response,
@@ -1415,7 +1401,7 @@ TEST_P(QueryTest, nxdomainWithNSECDuplicate) {
                          string("example.com. 0 IN RRSIG ") +
                          getCommonRRSIGText("SOA") + "\n" +
                          string(nsec_no_txt) + "\n" +
-                         string(").no.example.com. 3600 IN RRSIG ") +
+                         string("&.no.example.com. 3600 IN RRSIG ") +
                          getCommonRRSIGText("NSEC")).c_str(),
                   NULL, mock_finder->getOrigin());
 }
@@ -1529,7 +1515,7 @@ TEST_P(QueryTest, nxrrsetWithNSEC) {
 TEST_P(QueryTest, emptyNameWithNSEC) {
     // Empty non terminal with DNSSEC proof.  This is one of the cases of
     // Section 3.1.3.2 of RFC4035.
-    // mx.example.com. NSEC ).no.example.com. proves no.example.com. is a
+    // mx.example.com. NSEC &.no.example.com. proves no.example.com. is a
     // non empty terminal node.  Note that it also implicitly proves there
     // should be no closer wildcard match (because the empty name is an
     // exact match), so we only need one NSEC.
@@ -1700,12 +1686,6 @@ TEST_F(QueryTestForMockOnly, badWildcardProof3) {
 }
 
 TEST_P(QueryTest, wildcardNxrrsetWithDuplicateNSEC) {
-    // This is an in-memory specific bug (#2585), until it's fixed we
-    // tentatively skip the test for in-memory
-    if (GetParam() == INMEMORY) {
-        return;
-    }
-
     // NXRRSET on WILDCARD with DNSSEC proof.  We should have SOA, NSEC that
     // proves the NXRRSET and their RRSIGs. In this case we only need one NSEC,
     // which proves both NXDOMAIN and the non existence RRSETs of wildcard.
@@ -1723,12 +1703,6 @@ TEST_P(QueryTest, wildcardNxrrsetWithDuplicateNSEC) {
 }
 
 TEST_P(QueryTest, wildcardNxrrsetWithNSEC) {
-    // This is an in-memory specific bug (#2585), until it's fixed we
-    // tentatively skip the test for in-memory
-    if (GetParam() == INMEMORY) {
-        return;
-    }
-
     // WILDCARD + NXRRSET with DNSSEC proof.  We should have SOA, NSEC that
     // proves the NXRRSET and their RRSIGs. In this case we need two NSEC RRs,
     // one proves NXDOMAIN and the other proves non existence RRSETs of
@@ -2468,21 +2442,32 @@ TEST_P(QueryTest, nxrrsetWithNSEC3) {
                   NULL, mock_finder->getOrigin());
 }
 
-// Check the exception is correctly raised when the NSEC3 thing isn't in the
-// zone
-TEST_F(QueryTestForMockOnly, nxrrsetMissingNSEC3) {
-    // This is a broken data source scenario; works only with mock.
-
-    mock_finder->setNSEC3Flag(true);
-    // We just need it to return false for "matched". This indicates
-    // there's no exact match for NSEC3 on www.example.com.
-    ZoneFinder::FindNSEC3Result nsec3(false, 0, ConstRRsetPtr(),
-                                      ConstRRsetPtr());
-    mock_finder->setNSEC3Result(&nsec3);
+TEST_P(QueryTest, nxrrsetDerivedFromOptOutNSEC3) {
+    // In this test we emulate the situation where an empty non-terminal name
+    // is derived from insecure delegation and covered by an opt-out NSEC3.
+    // In the actual test data the covering NSEC3 really has the opt-out
+    // bit set, although the implementation doesn't check it anyway.
+    enableNSEC3(rrsets_to_add_);
+    query.process(*list_, Name("empty.example.com"), RRType::TXT(), response,
+                  true);
 
-    EXPECT_THROW(query.process(*list_, Name("www.example.com"),
-                               RRType::TXT(), response, true),
-                 Query::BadNSEC3);
+    // The closest provable encloser is the origin name (example.com.), and
+    // the next closer is the empty name itself, which is expected to be
+    // covered by an opt-out NSEC3 RR.  The response should contain these 2
+    // NSEC3s.
+    responseCheck(response, Rcode::NOERROR(), AA_FLAG, 0, 6, 0, NULL,
+                  (string(soa_minttl_txt) +
+                   string("example.com. 0 IN RRSIG ") +
+                   getCommonRRSIGText("SOA") + "\n" +
+                   string(nsec3_apex_txt) + "\n" +
+                   nsec3_hash_.calculate(Name("example.com.")) +
+                   ".example.com. 3600 IN RRSIG " +
+                   getCommonRRSIGText("NSEC3") + "\n" +
+                   string(nsec3_www_txt) + "\n" +
+                   nsec3_hash_.calculate(Name("www.example.com.")) +
+                   ".example.com. 3600 IN RRSIG " +
+                   getCommonRRSIGText("NSEC3") + "\n").c_str(),
+                  NULL, mock_finder->getOrigin());
 }
 
 TEST_P(QueryTest, nxrrsetWithNSEC3_ds_exact) {

src/bin/auth/tests/testdata/example-base-inc.zone

@@ -150,32 +150,32 @@ t.example.com. 3600 IN RRSIG NSEC 5 3 3600 20000101000000 20000201000000 12345 e
 ;; the best possible wildcard is below the "next domain" of the NSEC RR that
 ;; proves the NXDOMAIN, i.e.,
 ;; mx.example.com. (exist)
-;; (.no.example.com. (qname, NXDOMAIN)
-;; ).no.example.com. (exist)
+;; !.no.example.com. (qname, NXDOMAIN)
+;; &.no.example.com. (exist)
 ;; *.no.example.com. (best possible wildcard, not exist)
 ;var=no_txt
-\).no.example.com. 3600 IN AAAA 2001:db8::53
+&.no.example.com. 3600 IN AAAA 2001:db8::53
 ;; NSEC records.
 ;var=nsec_apex_txt
 example.com. 3600 IN NSEC cname.example.com. NS SOA NSEC RRSIG
 ;var=
 example.com. 3600 IN RRSIG NSEC 5 3 3600 20000101000000 20000201000000 12345 example.com. FAKEFAKEFAKE
 ;var=nsec_mx_txt
-mx.example.com. 3600 IN NSEC \).no.example.com. MX NSEC RRSIG
+mx.example.com. 3600 IN NSEC &.no.example.com. MX NSEC RRSIG
 
 ;var=
 mx.example.com. 3600 IN RRSIG NSEC 5 3 3600 20000101000000 20000201000000 12345 example.com. FAKEFAKEFAKE
 
 ;var=nsec_no_txt
-\).no.example.com. 3600 IN NSEC nz.no.example.com. AAAA NSEC RRSIG
+&.no.example.com. 3600 IN NSEC nz.no.example.com. AAAA NSEC RRSIG
 
 ;var=
-\).no.example.com. 3600 IN RRSIG NSEC 5 3 3600 20000101000000 20000201000000 12345 example.com. FAKEFAKEFAKE
+&.no.example.com. 3600 IN RRSIG NSEC 5 3 3600 20000101000000 20000201000000 12345 example.com. FAKEFAKEFAKE
 
 ;; We'll also test the case where a single NSEC proves both NXDOMAIN and the
 ;; non existence of wildcard.  The following records will be used for that
 ;; test.
-;; ).no.example.com. (exist, whose NSEC proves everything)
+;; &.no.example.com. (exist, whose NSEC proves everything)
 ;; *.no.example.com. (best possible wildcard, not exist)
 ;; nx.no.example.com. (NXDOMAIN)
 ;; nz.no.example.com. (exist)
@@ -234,3 +234,8 @@ bad-delegation.example.com. 3600 IN NS ns.example.net.
 ;; or NSEC3 that proves it.
 ;var=nosec_delegation_txt
 nosec-delegation.example.com. 3600 IN NS ns.nosec.example.net.
+
+;; Setup for emulating insecure delegation that contain an empty name.
+;; the delegation itself isn't expected to be used directly in tests.
+;var=
+delegation.empty.example.com. 3600 IN NS ns.delegation.empty.example

src/bin/auth/tests/testdata/example-nsec3-inc.zone

@@ -1,4 +1,4 @@
-;; See query_testzone_data.txt for general notes.
+;; See example-base-inc.zone for general notes.
 
 ;; NSEC3PARAM.  This is needed for database-based data source to
 ;; signal the zone is NSEC3-signed

src/bin/stats/tests/b10-stats-httpd_test.py

@@ -34,6 +34,7 @@ import http.client
 import xml.etree.ElementTree
 import random
 import urllib.parse
+import sys
 # load this module for xml validation with xsd. For this test, an
 # installation of lxml is required in advance. See http://lxml.de/.
 try:
@@ -250,6 +251,7 @@ class TestHttpHandler(unittest.TestCase):
         # reset the signal handler
         self.sig_handler.reset()
 
+    @unittest.skipIf(sys.version_info >= (3, 3), "Unsupported in Python 3.3 or higher")
     @unittest.skipUnless(xml_parser, "skipping the test using XMLParser")
     def test_do_GET(self):
         self.assertTrue(type(self.stats_httpd.httpd) is list)

src/bin/stats/tests/b10-stats_test.py

@@ -27,6 +27,7 @@ import threading
 import io
 import time
 import imp
+import sys
 
 import stats
 import isc.log
@@ -605,6 +606,7 @@ class TestStats(unittest.TestCase):
         self.assertEqual(self.stats.update_statistics_data(
                 'Foo', 'foo1', _test_exp6), ['unknown module name: Foo'])
 
+    @unittest.skipIf(sys.version_info >= (3, 3), "Unsupported in Python 3.3 or higher")
     def test_update_statistics_data_withmid(self):
         self.stats = stats.Stats()
         self.stats.do_polling()
@@ -736,6 +738,7 @@ class TestStats(unittest.TestCase):
                          isc.config.create_answer(0))
         self.assertFalse(self.stats.running)
 
+    @unittest.skipIf(sys.version_info >= (3, 3), "Unsupported in Python 3.3 or higher")
     def test_command_show(self):
         # two auth instances invoked
         list_auth = [ self.base.auth.server,
@@ -1143,6 +1146,7 @@ class TestStats(unittest.TestCase):
                          isc.config.create_answer(
                 1, "module name is not specified"))
 
+    @unittest.skipIf(sys.version_info >= (3, 3), "Unsupported in Python 3.3 or higher")
     def test_polling(self):
         stats_server = ThreadingServerManager(MyStats)
         stat = stats_server.server

src/lib/datasrc/memory/zone_finder.cc

@@ -866,7 +866,8 @@ InMemoryZoneFinder::findInternal(const isc::dns::Name& name,
         const RdataSet* cur_rds = node->getData();
         while (cur_rds != NULL) {
             target->push_back(createTreeNodeRRset(node, cur_rds, rrclass_,
-                                                  options, &name));
+                                                  options,
+                                                  wild ? &name : NULL));
             cur_rds = cur_rds->getNext();
         }
         LOG_DEBUG(logger, DBG_TRACE_DATA, DATASRC_MEM_ANY_SUCCESS).
@@ -893,9 +894,13 @@ InMemoryZoneFinder::findInternal(const isc::dns::Name& name,
         }
     }
     // No exact match or CNAME.  Get NSEC if necessary and return NXRRSET.
+    // Note that we don't have to provide the "real name" even if this is
+    // a wildcard; if NSEC is needed its owner name shouldn't be subject to
+    // wildcard substitution; if NSEC isn't needed the "real name" doesn't
+    // matter anyway.
     return (createFindResult(rrclass_, zone_data_, NXRRSET, node,
                              getNSECForNXRRSET(zone_data_, options, node),
-                             options, wild, &name));
+                             options, wild));
 }
 
 isc::datasrc::ZoneFinder::FindNSEC3Result

src/lib/datasrc/tests/memory/zone_finder_unittest.cc

@@ -442,14 +442,23 @@ protected:
         }
         EXPECT_EQ((expected_flags & ZoneFinder::RESULT_WILDCARD) != 0,
                   find_result->isWildcard());
-        EXPECT_EQ((expected_flags & ZoneFinder::RESULT_NSEC_SIGNED)
-                  != 0, find_result->isNSECSigned());
-        EXPECT_EQ((expected_flags & ZoneFinder::RESULT_NSEC3_SIGNED)
-                  != 0, find_result->isNSEC3Signed());
-        // Convert all rrsets to 'full' ones before checking
+        EXPECT_EQ((expected_flags & ZoneFinder::RESULT_NSEC_SIGNED) != 0,
+                  find_result->isNSECSigned());
+        EXPECT_EQ((expected_flags & ZoneFinder::RESULT_NSEC3_SIGNED) != 0,
+                  find_result->isNSEC3Signed());
+        // Convert all rrsets to 'full' ones before checking.  Also, confirm
+        // each RRset of the vector is of the "same kind" as one would be
+        // found by the find() method.
         std::vector<ConstRRsetPtr> converted_rrsets;
         BOOST_FOREACH(ConstRRsetPtr cur_rrset, target) {
             converted_rrsets.push_back(convertRRset(cur_rrset));
+
+            // As we know findAll() succeeded, this find() should also
+            // succeed, and the two sets should be "identical".
+            const ZoneFinderContextPtr result =
+                finder->find(name, cur_rrset->getType());
+            ASSERT_TRUE(result->rrset);
+            EXPECT_TRUE(result->rrset->isSameKind(*cur_rrset));
         }
         rrsetsCheck(expected_rrsets.begin(), expected_rrsets.end(),
                     converted_rrsets.begin(), converted_rrsets.end());
@@ -1133,21 +1142,42 @@ InMemoryZoneFinderTest::wildcardCheck(
     }
 }
 
+// We have combinations of these cases (6 in total)
+// expected_flags: NSEC, NSEC3, RESULT_DEFAULT
+// options: NO_WILDCARD, FIND_DEFAULT
+
+// 1. Normal case: expected = DEFAULT, options = DEFAULT
 TEST_F(InMemoryZoneFinderTest, wildcard) {
-    // Normal case
     wildcardCheck();
 }
 
+// 2. options: expected = DEFAULT, options = NO_WILDCARD
+TEST_F(InMemoryZoneFinderTest, wildcardDisabled) {
+    // Similar to the previous once, but check the behavior for a non signed
+    // zone just in case.
+    wildcardCheck(ZoneFinder::RESULT_DEFAULT, ZoneFinder::NO_WILDCARD);
+}
+
+// 3. options: expected = NSEC_SIGNED, options = DEFAULT
+TEST_F(InMemoryZoneFinderTest, wildcardWithNSEC) {
+    wildcardCheck(ZoneFinder::RESULT_NSEC_SIGNED, ZoneFinder::FIND_DEFAULT);
+}
+
+// 4. options: expected = NSEC_SIGNED, options = NO_WILDCARD
 TEST_F(InMemoryZoneFinderTest, wildcardDisabledWithNSEC) {
     // Wildcard is disabled.  In practice, this is used as part of query
     // processing for an NSEC-signed zone, so we test that case specifically.
     wildcardCheck(ZoneFinder::RESULT_NSEC_SIGNED, ZoneFinder::NO_WILDCARD);
 }
 
-TEST_F(InMemoryZoneFinderTest, wildcardDisabledWithoutNSEC) {
-    // Similar to the previous once, but check the behavior for a non signed
-    // zone just in case.
-    wildcardCheck(ZoneFinder::RESULT_DEFAULT, ZoneFinder::NO_WILDCARD);
+// 5. options: expected = NSEC3_SIGNED, options = DEFAULT
+TEST_F(InMemoryZoneFinderTest, wildcardWithNSEC3) {
+    wildcardCheck(ZoneFinder::RESULT_NSEC3_SIGNED, ZoneFinder::FIND_DEFAULT);
+}
+
+// 6. options: expected = NSEC3_SIGNED, options = DEFAULT
+TEST_F(InMemoryZoneFinderTest, wildcardDisabledWithNSEC3) {
+    wildcardCheck(ZoneFinder::RESULT_NSEC3_SIGNED, ZoneFinder::NO_WILDCARD);
 }
 
 /*

src/lib/util/io/socketsession.cc

@@ -14,6 +14,8 @@
 
 #include <config.h>
 
+#include <unistd.h>
+
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/uio.h>