Browse Source

Added verbose options to exactly what is happening with loadzone. Added loadzone test suite of different file formats to load.

git-svn-id: svn://bind10.isc.org/svn/bind10/trunk@2340 e5f2f494-b856-4b98-b285-d166d9295462
tingting shen 15 years ago
parent
commit
31ec8c6c65
55 changed files with 812 additions and 89 deletions
  1. 5 0
      ChangeLog
  2. 6 0
      configure.ac
  3. 26 20
      src/bin/loadzone/Makefile.am
  4. 16 6
      src/bin/loadzone/b10-loadzone.py.in
  5. 24 0
      src/bin/loadzone/tests/correct/Makefile.am
  6. 72 0
      src/bin/loadzone/tests/correct/correct_test.sh.in
  7. 14 0
      src/bin/loadzone/tests/correct/example.db
  8. 8 0
      src/bin/loadzone/tests/correct/get_zonedatas.py
  9. 4 0
      src/bin/loadzone/tests/correct/inclsub.db
  10. 23 0
      src/bin/loadzone/tests/correct/include.db
  11. 79 0
      src/bin/loadzone/tests/correct/known.test.out
  12. 21 0
      src/bin/loadzone/tests/correct/mix1.db
  13. 3 0
      src/bin/loadzone/tests/correct/mix1sub1.db
  14. 3 0
      src/bin/loadzone/tests/correct/mix1sub2.db
  15. 21 0
      src/bin/loadzone/tests/correct/mix2.db
  16. 3 0
      src/bin/loadzone/tests/correct/mix2sub1.txt
  17. 3 0
      src/bin/loadzone/tests/correct/mix2sub2.txt
  18. 17 0
      src/bin/loadzone/tests/correct/ttl1.db
  19. 17 0
      src/bin/loadzone/tests/correct/ttl2.db
  20. 18 0
      src/bin/loadzone/tests/correct/ttlext.db
  21. 25 0
      src/bin/loadzone/tests/error/Makefile.am
  22. 11 0
      src/bin/loadzone/tests/error/error.known
  23. 75 0
      src/bin/loadzone/tests/error/error_test.sh.in
  24. 13 0
      src/bin/loadzone/tests/error/formerr1.db
  25. 12 0
      src/bin/loadzone/tests/error/formerr2.db
  26. 12 0
      src/bin/loadzone/tests/error/formerr3.db
  27. 12 0
      src/bin/loadzone/tests/error/formerr4.db
  28. 13 0
      src/bin/loadzone/tests/error/formerr5.db
  29. 1 0
      src/bin/loadzone/tests/error/include.txt
  30. 12 0
      src/bin/loadzone/tests/error/keyerror1.db
  31. 12 0
      src/bin/loadzone/tests/error/keyerror2.db
  32. 13 0
      src/bin/loadzone/tests/error/keyerror3.db
  33. 11 0
      src/bin/loadzone/tests/error/originerr1.db
  34. 12 0
      src/bin/loadzone/tests/error/originerr2.db
  35. 0 0
      src/bin/loadzone/tests/normal/Kexample.com.+005+04456.key
  36. 0 0
      src/bin/loadzone/tests/normal/Kexample.com.+005+04456.private
  37. 0 0
      src/bin/loadzone/tests/normal/Kexample.com.+005+33495.key
  38. 0 0
      src/bin/loadzone/tests/normal/Kexample.com.+005+33495.private
  39. 0 0
      src/bin/loadzone/tests/normal/Ksql1.example.com.+005+12447.key
  40. 0 0
      src/bin/loadzone/tests/normal/Ksql1.example.com.+005+12447.private
  41. 0 0
      src/bin/loadzone/tests/normal/Ksql1.example.com.+005+33313.key
  42. 0 0
      src/bin/loadzone/tests/normal/Ksql1.example.com.+005+33313.private
  43. 0 0
      src/bin/loadzone/tests/normal/Ksql2.example.com.+005+38482.key
  44. 0 0
      src/bin/loadzone/tests/normal/Ksql2.example.com.+005+38482.private
  45. 0 0
      src/bin/loadzone/tests/normal/Ksql2.example.com.+005+63192.key
  46. 0 0
      src/bin/loadzone/tests/normal/Ksql2.example.com.+005+63192.private
  47. 0 0
      src/bin/loadzone/tests/normal/README
  48. 0 0
      src/bin/loadzone/tests/normal/dsset-subzone.example.com
  49. 0 0
      src/bin/loadzone/tests/normal/example.com
  50. 0 0
      src/bin/loadzone/tests/normal/example.com.signed
  51. 0 0
      src/bin/loadzone/tests/normal/sql1.example.com
  52. 0 0
      src/bin/loadzone/tests/normal/sql1.example.com.signed
  53. 0 0
      src/bin/loadzone/tests/normal/sql2.example.com
  54. 0 0
      src/bin/loadzone/tests/normal/sql2.example.com.signed
  55. 195 63
      src/lib/python/isc/datasrc/master.py

+ 5 - 0
ChangeLog

@@ -1,3 +1,8 @@
+  65.  [func]       shentingting
+    Added verbose options to exactly what is happening with loadzone.
+	Added loadzone test suite of different file formats to load.
+	(tingting-loadzone, svn r2339)
+
   64.  [func]       jerry
   64.  [func]       jerry
     Added python logging framework. It is for testing and experimenting
     Added python logging framework. It is for testing and experimenting
 	with logging ideas. Currently, it supports three channels(file, 
 	with logging ideas. Currently, it supports three channels(file, 

+ 6 - 0
configure.ac

@@ -415,6 +415,8 @@ AC_CONFIG_FILES([Makefile
                  src/bin/cfgmgr/tests/Makefile
                  src/bin/cfgmgr/tests/Makefile
                  src/bin/host/Makefile
                  src/bin/host/Makefile
                  src/bin/loadzone/Makefile
                  src/bin/loadzone/Makefile
+                 src/bin/loadzone/tests/correct/Makefile
+                 src/bin/loadzone/tests/error/Makefile
                  src/bin/msgq/Makefile
                  src/bin/msgq/Makefile
                  src/bin/msgq/tests/Makefile
                  src/bin/msgq/tests/Makefile
                  src/bin/auth/Makefile
                  src/bin/auth/Makefile
@@ -464,6 +466,8 @@ AC_OUTPUT([src/bin/cfgmgr/b10-cfgmgr.py
            src/bin/bindctl/bindctl-source.py
            src/bin/bindctl/bindctl-source.py
            src/bin/bindctl/tests/bindctl_test
            src/bin/bindctl/tests/bindctl_test
            src/bin/loadzone/run_loadzone.sh
            src/bin/loadzone/run_loadzone.sh
+           src/bin/loadzone/tests/correct/correct_test.sh
+           src/bin/loadzone/tests/error/error_test.sh
            src/bin/loadzone/b10-loadzone.py
            src/bin/loadzone/b10-loadzone.py
            src/bin/usermgr/run_b10-cmdctl-usermgr.sh
            src/bin/usermgr/run_b10-cmdctl-usermgr.sh
            src/bin/usermgr/b10-cmdctl-usermgr.py
            src/bin/usermgr/b10-cmdctl-usermgr.py
@@ -491,6 +495,8 @@ AC_OUTPUT([src/bin/cfgmgr/b10-cfgmgr.py
            chmod +x src/bin/bindctl/tests/bindctl_test
            chmod +x src/bin/bindctl/tests/bindctl_test
            chmod +x src/bin/bindctl/run_bindctl.sh
            chmod +x src/bin/bindctl/run_bindctl.sh
            chmod +x src/bin/loadzone/run_loadzone.sh
            chmod +x src/bin/loadzone/run_loadzone.sh
+           chmod +x src/bin/loadzone/tests/correct/correct_test.sh
+           chmod +x src/bin/loadzone/tests/error/error_test.sh
            chmod +x src/bin/usermgr/run_b10-cmdctl-usermgr.sh
            chmod +x src/bin/usermgr/run_b10-cmdctl-usermgr.sh
            chmod +x src/bin/msgq/run_msgq.sh
            chmod +x src/bin/msgq/run_msgq.sh
            chmod +x src/bin/msgq/tests/msgq_test
            chmod +x src/bin/msgq/tests/msgq_test

+ 26 - 20
src/bin/loadzone/Makefile.am

@@ -1,3 +1,5 @@
+SUBDIRS = tests/correct
+SUBDIRS += tests/error
 bin_SCRIPTS = b10-loadzone
 bin_SCRIPTS = b10-loadzone
 
 
 CLEANFILES = b10-loadzone
 CLEANFILES = b10-loadzone
@@ -22,23 +24,27 @@ install-data-local:
 	$(mkinstalldirs) $(DESTDIR)/@localstatedir@/@PACKAGE@
 	$(mkinstalldirs) $(DESTDIR)/@localstatedir@/@PACKAGE@
 # TODO: permissions handled later
 # TODO: permissions handled later
 
 
-EXTRA_DIST += testdata/README
-EXTRA_DIST += testdata/dsset-subzone.example.com.
-EXTRA_DIST += testdata/example.com
-EXTRA_DIST += testdata/example.com.signed
-EXTRA_DIST += testdata/Kexample.com.+005+04456.key
-EXTRA_DIST += testdata/Kexample.com.+005+04456.private
-EXTRA_DIST += testdata/Kexample.com.+005+33495.key
-EXTRA_DIST += testdata/Kexample.com.+005+33495.private
-EXTRA_DIST += testdata/Ksql1.example.com.+005+12447.key
-EXTRA_DIST += testdata/Ksql1.example.com.+005+12447.private
-EXTRA_DIST += testdata/Ksql1.example.com.+005+33313.key
-EXTRA_DIST += testdata/Ksql1.example.com.+005+33313.private
-EXTRA_DIST += testdata/Ksql2.example.com.+005+38482.key
-EXTRA_DIST += testdata/Ksql2.example.com.+005+38482.private
-EXTRA_DIST += testdata/Ksql2.example.com.+005+63192.key
-EXTRA_DIST += testdata/Ksql2.example.com.+005+63192.private
-EXTRA_DIST += testdata/sql1.example.com
-EXTRA_DIST += testdata/sql1.example.com.signed
-EXTRA_DIST += testdata/sql2.example.com
-EXTRA_DIST += testdata/sql2.example.com.signed
+EXTRA_DIST += tests/normal/README
+EXTRA_DIST += tests/normal/dsset-subzone.example.com.
+EXTRA_DIST += tests/normal/example.com
+EXTRA_DIST += tests/normal/example.com.signed
+EXTRA_DIST += tests/normal/Kexample.com.+005+04456.key
+EXTRA_DIST += tests/normal/Kexample.com.+005+04456.private
+EXTRA_DIST += tests/normal/Kexample.com.+005+33495.key
+EXTRA_DIST += tests/normal/Kexample.com.+005+33495.private
+EXTRA_DIST += tests/normal/Ksql1.example.com.+005+12447.key
+EXTRA_DIST += tests/normal/Ksql1.example.com.+005+12447.private
+EXTRA_DIST += tests/normal/Ksql1.example.com.+005+33313.key
+EXTRA_DIST += tests/normal/Ksql1.example.com.+005+33313.private
+EXTRA_DIST += tests/normal/Ksql2.example.com.+005+38482.key
+EXTRA_DIST += tests/normal/Ksql2.example.com.+005+38482.private
+EXTRA_DIST += tests/normal/Ksql2.example.com.+005+63192.key
+EXTRA_DIST += tests/normal/Ksql2.example.com.+005+63192.private
+EXTRA_DIST += tests/normal/sql1.example.com
+EXTRA_DIST += tests/normal/sql1.example.com.signed
+EXTRA_DIST += tests/normal/sql2.example.com
+EXTRA_DIST += tests/normal/sql2.example.com.signed
+
+pytest:
+	$(SHELL) tests/correct/correct_test.sh
+	$(SHELL) tests/error/error_test.sh

+ 16 - 6
src/bin/loadzone/b10-loadzone.py.in

@@ -19,7 +19,8 @@ import sys; sys.path.append ('@@PYTHONPATH@@')
 import re, getopt
 import re, getopt
 import isc.datasrc
 import isc.datasrc
 from isc.datasrc.master import MasterFile
 from isc.datasrc.master import MasterFile
-
+import time
+import os
 #########################################################################
 #########################################################################
 # usage: print usage note and exit
 # usage: print usage note and exit
 #########################################################################
 #########################################################################
@@ -57,23 +58,32 @@ def main():
     if len(args) != 1:
     if len(args) != 1:
         usage()
         usage()
     zonefile = args[0]
     zonefile = args[0]
-
+    verbose = os.isatty(sys.stdout.fileno())
     try:
     try:
-        master = MasterFile(zonefile, initial_origin)
+        master = MasterFile(zonefile, initial_origin, verbose)
     except Exception as e:
     except Exception as e:
-        print("Error reading zone file: " + str(e))
+        sys.stderr.write("Error reading zone file: %s\n" % str(e))
         exit(1)
         exit(1)
 
 
     try:
     try:
         zone = master.zonename()
         zone = master.zonename()
+        if verbose:
+            sys.stdout.write("Using SQLite3 database file %s\n" % dbfile)
+            sys.stdout.write("Zone name is %s\n" % zone)
+            sys.stdout.write("Loading file \"%s\"\n" % zonefile)
     except Exception as e:
     except Exception as e:
-        print("Error reading zone file: " + str(e))
+        sys.stdout.write("\n")
+        sys.stderr.write("Error reading zone file: %s\n" % str(e))
         exit(1)
         exit(1)
 
 
     try:
     try:
         isc.datasrc.sqlite3_ds.load(dbfile, zone, master.zonedata)
         isc.datasrc.sqlite3_ds.load(dbfile, zone, master.zonedata)
+        if verbose:
+            master.closeverbose()
+            sys.stdout.write("\nDone.\n")
     except Exception as e:
     except Exception as e:
-        print("Error loading database: " + str(e))
+        sys.stdout.write("\n")
+        sys.stderr.write("Error loading database: %s\n"% str(e))
         exit(1)
         exit(1)
 
 
 if __name__ == "__main__":
 if __name__ == "__main__":

+ 24 - 0
src/bin/loadzone/tests/correct/Makefile.am

@@ -0,0 +1,24 @@
+PYTESTS = correct_test.sh
+EXTRA_DIST = get_zonedatas.py
+EXTRA_DIST += include.db
+EXTRA_DIST += inclsub.db
+EXTRA_DIST += known.test.out
+EXTRA_DIST += mix1.db
+EXTRA_DIST += mix1sub1.db
+EXTRA_DIST += mix1sub2.db
+EXTRA_DIST += mix2.db
+EXTRA_DIST += mix2sub1.txt
+EXTRA_DIST += mix2sub2.txt
+EXTRA_DIST += ttl1.db
+EXTRA_DIST += ttl2.db
+EXTRA_DIST += ttlext.db
+
+# later will have configure option to choose this, like: coverage run --branch
+PYCOVERAGE = $(PYTHON)
+# test using command-line arguments, so use check-local target instead of TESTS
+check-local:
+	for pytest in $(PYTESTS) ; do \
+	echo Running test: $$pytest ; \
+	env PYTHONPATH=$(abs_top_srcdir)/src/lib/python:$(abs_top_builddir)/src/bin/loadzone \
+	$(SHELL) $(abs_srcdir)/$$pytest ; \
+	done

+ 72 - 0
src/bin/loadzone/tests/correct/correct_test.sh.in

@@ -0,0 +1,72 @@
+#! /bin/sh
+
+# Copyright (C) 2010  Internet Systems Consortium.
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
+# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
+# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+PYTHON_EXEC=${PYTHON_EXEC:-@PYTHON@}
+export PYTHON_EXEC
+
+PYTHONPATH=@abs_top_builddir@/src/lib/python
+export PYTHONPATH
+
+LOADZONE_PATH=@abs_top_srcdir@/src/bin/loadzone
+status=0
+echo "Loadzone include. from include.db file"
+cd ${LOADZONE_PATH}/tests/correct
+${LOADZONE_PATH}/b10-loadzone -d zone.sqlite3 include.db >> /dev/null
+
+echo "loadzone  ttl1. from ttl1.db file"
+${LOADZONE_PATH}/b10-loadzone -d zone.sqlite3 ttl1.db >> /dev/null
+
+echo "loadzone ttl2. from ttl2.db file"
+${LOADZONE_PATH}/b10-loadzone -d zone.sqlite3 ttl2.db >> /dev/null
+
+echo "loadzone mix1. from mix1.db"
+${LOADZONE_PATH}/b10-loadzone -d zone.sqlite3 mix1.db >> /dev/null
+
+echo "loadzone mix2. from mix2.db"
+${LOADZONE_PATH}/b10-loadzone -d zone.sqlite3 mix2.db >> /dev/null
+
+echo "loadzone ttlext. from ttlext.db"
+${LOADZONE_PATH}/b10-loadzone -d zone.sqlite3 ttlext.db >> /dev/null
+
+echo "loadzone example.com. from example.db"
+${LOADZONE_PATH}/b10-loadzone -d zone.sqlite3 example.db >> /dev/null
+
+echo "I:test master file \$INCLUDE semantics"
+echo "I:test master file BIND 8 compatibility TTL and \$TTL semantics"
+echo "I:test master file RFC1035 TTL and \$TTL semantics"
+echo "I:test master file BIND8 compatibility and mixed \$INCLUDE with \$TTL semantics"
+echo "I:test master file RFC1035 TTL and mixed \$INCLUDE with \$TTL semantics"
+echo "I:test master file BIND9 extenstion of TTL"
+echo "I:test master file RFC1035 missing CLASS, TTL, NAME semantics"
+
+${PYTHON_EXEC} get_zonedatas.py > test.out
+echo "Compare test results."
+diff test.out known.test.out || status=1
+
+echo "Clean tmp files."
+rm -f zone.sqlite3
+rm -f test.out
+echo "I:exit status: $status"
+echo "------------------------------------------------------------------------------"
+echo "Ran 7 test files"
+echo ""
+if [ "$status" -eq 1 ] ;then
+    echo "ERROR"
+else
+    echo "OK"
+fi
+exit $status

+ 14 - 0
src/bin/loadzone/tests/correct/example.db

@@ -0,0 +1,14 @@
+;This file includes all kinds of rr form. missing name, ttl and class or not.
+$ORIGIN example.com.
+$TTL 60
+@    IN SOA   ns1.example.com. hostmaster.example.com. (1 43200 900 1814400 7200)
+     IN     20      NS  ns1
+                    NS  ns2
+ns1  IN     30      A   192.168.1.102
+            70      NS  ns3
+     IN             NS  ns4
+     10     IN      MX  10  mail.example.com.
+ns2         80      A   1.1.1.1
+ns3  IN             A   2.2.2.2
+ns4                 A   3.3.3.3
+ns5  90     IN      A   4.4.4.4

+ 8 - 0
src/bin/loadzone/tests/correct/get_zonedatas.py

@@ -0,0 +1,8 @@
+from isc.datasrc import sqlite3_ds
+import sys
+ZONE_FILE = "zone.sqlite3"
+zonename_set = ["include.", "ttl1.", "ttl2.", "mix1.", "mix2.", "ttlext.", "example.com."]
+for zone_name in zonename_set:
+    for rr_data in sqlite3_ds.get_zone_datas(zone_name, ZONE_FILE):
+        data_len = len(rr_data[2])
+        sys.stdout.write(rr_data[2] + '\t\t' + str(rr_data[4]) + '\tIN\t' + rr_data[5] + '\t' + rr_data[7] + '\n')

+ 4 - 0
src/bin/loadzone/tests/correct/inclsub.db

@@ -0,0 +1,4 @@
+a    300		A	10.0.1.1
+$ORIGIN foo
+b	 300     	A	10.0.2.2
+

+ 23 - 0
src/bin/loadzone/tests/correct/include.db

@@ -0,0 +1,23 @@
+$ORIGIN include.   ; initialize origin
+$TTL 300
+@			IN SOA	ns hostmaster (
+				1        ; serial
+				3600
+				1800
+				1814400
+				3600
+				)
+			NS	ns
+
+ns			A	127.0.0.1
+
+a			A	10.0.0.1
+$INCLUDE inclsub.db sub   ;  a.include. is the relative domain name origin for the included file
+; use the current domain name
+            A	99.99.99.99
+b			A	10.0.0.2
+$ORIGIN b
+$INCLUDE inclsub.db
+; use the current domain name
+			A	10.0.0.99
+c			A	10.0.0.3

+ 79 - 0
src/bin/loadzone/tests/correct/known.test.out

@@ -0,0 +1,79 @@
+include.		300	IN	SOA	ns.include. hostmaster.include. 1 3600 1800 1814400 3600
+include.		300	IN	NS	ns.include.
+ns.include.		300	IN	A	127.0.0.1
+a.include.		300	IN	A	10.0.0.1
+a.sub.include.		300	IN	A	10.0.1.1
+b.foo.sub.include.		300	IN	A	10.0.2.2
+a.include.		300	IN	A	99.99.99.99
+b.include.		300	IN	A	10.0.0.2
+a.b.include.		300	IN	A	10.0.1.1
+b.foo.b.include.		300	IN	A	10.0.2.2
+b.include.		300	IN	A	10.0.0.99
+c.b.include.		300	IN	A	10.0.0.3
+ttl1.		3	IN	SOA	ns.ttl1. hostmaster.ttl1. 1 3600 1800 1814400 3
+ttl1.		3	IN	NS	ns.ttl1.
+ns.ttl1.		3	IN	A	10.53.0.1
+a.ttl1.		3	IN	TXT	"soa minttl 3"
+b.ttl1.		2	IN	TXT	"explicit ttl 2"
+c.ttl1.		3	IN	TXT	"soa minttl 3"
+d.ttl1.		1	IN	TXT	"default ttl 1"
+e.ttl1.		4	IN	TXT	"explicit ttl 4"
+f.ttl1.		1	IN	TXT	"default ttl 1"
+ttl2.		1	IN	SOA	ns.ttl2. hostmaster.ttl2. 1 3600 1800 1814400 3
+ttl2.		1	IN	NS	ns.ttl2.
+ns.ttl2.		1	IN	A	10.53.0.1
+a.ttl2.		1	IN	TXT	"inherited ttl 1"
+b.ttl2.		2	IN	TXT	"explicit ttl 2"
+c.ttl2.		2	IN	TXT	"inherited ttl 2"
+d.ttl2.		3	IN	TXT	"default ttl 3"
+e.ttl2.		2	IN	TXT	"explicit ttl 2"
+f.ttl2.		3	IN	TXT	"default ttl 3"
+mix1.		3	IN	SOA	ns.mix1. hostmaster.mix1. 1 3600 1800 1814400 3
+mix1.		3	IN	NS	ns.mix1.
+ns.mix1.		3	IN	A	10.53.0.1
+a.mix1.		3	IN	TXT	"soa minttl 3"
+b.mix1.		2	IN	TXT	"explicit ttl 2"
+a.mix1.		3	IN	TXT	"soa minttl 3"
+b.foo.mix1.		3	IN	TXT	"soa minttl 3"
+c.mix1.		3	IN	TXT	"soa minttl 3"
+d.mix1.		1	IN	TXT	"default ttl 1"
+e.mix1.		4	IN	TXT	"explicit ttl 4"
+f.mix1.		1	IN	TXT	"default ttl 1"
+i.mix1.		1	IN	TXT	"default ttl 1"
+g.mix1.		5	IN	TXT	"default ttl 5"
+h.mix1.		5	IN	TXT	"the include ttl 5"
+mix2.		1	IN	SOA	ns.mix2. hostmaster.mix2. 1 3600 1800 1814400 3
+mix2.		1	IN	NS	ns.mix2.
+ns.mix2.		1	IN	A	10.53.0.1
+a.mix2.		1	IN	TXT	"inherited ttl 1"
+h.mix2.		1	IN	TXT	"inherited ttl 1"
+g.mix2.		6	IN	TXT	"inherited ttl 6"
+b.mix2.		6	IN	TXT	"explicit ttl 6"
+c.mix2.		2	IN	TXT	"inherited ttl 2"
+m.mix2.		6	IN	TXT	"explicit ttl 6"
+d.mix2.		3	IN	TXT	"default ttl 3"
+e.mix2.		2	IN	TXT	"explicit ttl 2"
+n.mix2.		3	IN	TXT	"default ttl 3"
+f.mix2.		3	IN	TXT	"default ttl 3"
+g.mix2.		5	IN	TXT	"default ttl 5"
+f.mix2.		5	IN	TXT	"default ttl 5"
+ttlext.		3	IN	SOA	ns.ttlext. hostmaster.ttlext. 1 3600 1800 1814400 3
+ttlext.		3	IN	NS	ns.ttlext.
+ns.ttlext.		3	IN	A	10.53.0.1
+a.ttlext.		3	IN	TXT	"soa minttl 3"
+b.ttlext.		2	IN	TXT	"explicit ttl 2"
+c.ttlext.		3	IN	TXT	"soa minttl 3"
+d.ttlext.		600	IN	TXT	"default ttl 600"
+e.ttlext.		4	IN	TXT	"explicit ttl 4"
+f.ttlext.		600	IN	TXT	"default ttl 600"
+example.com.		60	IN	SOA	ns1.example.com. hostmaster.example.com. 1 43200 900 1814400 7200
+example.com.		20	IN	NS	ns1.example.com.
+example.com.		60	IN	NS	ns2.example.com.
+ns1.example.com.		30	IN	A	192.168.1.102
+ns1.example.com.		70	IN	NS	ns3.example.com.
+ns1.example.com.		60	IN	NS	ns4.example.com.
+ns1.example.com.		10	IN	MX	10 mail.example.com.
+ns2.example.com.		80	IN	A	1.1.1.1
+ns3.example.com.		60	IN	A	2.2.2.2
+ns4.example.com.		60	IN	A	3.3.3.3
+ns5.example.com.		90	IN	A	4.4.4.4

+ 21 - 0
src/bin/loadzone/tests/correct/mix1.db

@@ -0,0 +1,21 @@
+; $Id: ttl1.db,v 1.6 2007/06/19 23:47:04 tbox Exp $
+$ORIGIN mix1.
+@			IN SOA	ns hostmaster (
+				1        ; serial
+				3600
+				1800
+				1814400
+				3
+				)
+			NS	ns
+ns			A	10.53.0.1
+a			TXT	"soa minttl 3"
+b		2	TXT	"explicit ttl 2"
+$INCLUDE mix1sub1.db
+c			TXT	"soa minttl 3"
+$TTL 1
+d			TXT	"default ttl 1"
+e		4	TXT	"explicit ttl 4"
+f			TXT	"default ttl 1"
+$INCLUDE mix1sub2.db
+h       5   TXT "the include ttl 5"

+ 3 - 0
src/bin/loadzone/tests/correct/mix1sub1.db

@@ -0,0 +1,3 @@
+a                       TXT      "soa minttl 3"
+$ORIGIN foo
+b                       TXT      "soa minttl 3"

+ 3 - 0
src/bin/loadzone/tests/correct/mix1sub2.db

@@ -0,0 +1,3 @@
+i                       TXT      "default ttl 1"
+$TTL   5
+g                       TXT      "default ttl 5"

+ 21 - 0
src/bin/loadzone/tests/correct/mix2.db

@@ -0,0 +1,21 @@
+$ORIGIN mix2.
+@		1	IN SOA	ns hostmaster (
+				1        ; serial
+				3600
+				1800
+				1814400
+				3
+				)
+			NS	ns
+ns			A	10.53.0.1
+a			TXT	"inherited ttl 1"
+$INCLUDE mix2sub1.txt
+b			TXT	"explicit ttl 6"
+c		2	TXT	"inherited ttl 2"
+m           TXT "explicit ttl 6"
+$TTL 3
+d			TXT	"default ttl 3"
+e		2	TXT	"explicit ttl 2"
+n           TXT "default ttl 3"
+$INCLUDE mix2sub2.txt
+f			TXT	"default ttl 5"

+ 3 - 0
src/bin/loadzone/tests/correct/mix2sub1.txt

@@ -0,0 +1,3 @@
+h                       TXT     "inherited ttl 1"
+$TTL 6
+g                       TXT     "inherited ttl 6"

+ 3 - 0
src/bin/loadzone/tests/correct/mix2sub2.txt

@@ -0,0 +1,3 @@
+f                       TXT     "default  ttl 3"
+$TTL 5
+g                       TXT     "default  ttl 5"

+ 17 - 0
src/bin/loadzone/tests/correct/ttl1.db

@@ -0,0 +1,17 @@
+$ORIGIN ttl1.
+@			IN SOA	ns hostmaster (
+				1        ; serial
+				3600
+				1800
+				1814400
+				3
+				)
+			NS	ns
+ns			A	10.53.0.1
+a			TXT	"soa minttl 3"
+b		2	TXT	"explicit ttl 2"
+c			TXT	"soa minttl 3"
+$TTL 1
+d			TXT	"default ttl 1"
+e		4	TXT	"explicit ttl 4"
+f			TXT	"default ttl 1"

+ 17 - 0
src/bin/loadzone/tests/correct/ttl2.db

@@ -0,0 +1,17 @@
+$ORIGIN ttl2.
+@		1	IN SOA	ns hostmaster (
+				1        ; serial
+				3600
+				1800
+				1814400
+				3
+				)
+			NS	ns
+ns			A	10.53.0.1
+a			TXT	"inherited ttl 1"
+b		2	TXT	"explicit ttl 2"
+c			TXT	"inherited ttl 2"
+$TTL 3    ; a new ttl
+d			TXT	"default ttl 3"
+e		2	TXT	"explicit ttl 2"
+f			TXT	"default ttl 3"

+ 18 - 0
src/bin/loadzone/tests/correct/ttlext.db

@@ -0,0 +1,18 @@
+; $Id: ttl1.db,v 1.6 2007/06/19 23:47:04 tbox Exp $
+$ORIGIN ttlext.
+@			IN SOA	ns hostmaster (
+				1        ; serial
+				3600
+				1800
+				1814400
+				3
+				)
+			NS	ns
+ns			A	10.53.0.1
+a			TXT	"soa minttl 3"
+b		2S	TXT	"explicit ttl 2"
+c			TXT	"soa minttl 3"
+$TTL 10M  ; bind9 extention ttl
+d			TXT	"default ttl 600"
+e		4	TXT	"explicit ttl 4"
+f			TXT	"default ttl 600"

+ 25 - 0
src/bin/loadzone/tests/error/Makefile.am

@@ -0,0 +1,25 @@
+PYTESTS = error_test.sh
+
+EXTRA_DIST = error.known
+EXTRA_DIST += formerr1.db 
+EXTRA_DIST += formerr2.db
+EXTRA_DIST += formerr3.db
+EXTRA_DIST += formerr4.db
+EXTRA_DIST += formerr5.db
+EXTRA_DIST += include.txt
+EXTRA_DIST += keyerror1.db
+EXTRA_DIST += keyerror2.db
+EXTRA_DIST += keyerror3.db
+EXTRA_DIST += nofilenane.db
+EXTRA_DIST += originerr1.db
+EXTRA_DIST += originerr2.db
+
+# later will have configure option to choose this, like: coverage run --branch
+PYCOVERAGE = $(PYTHON)
+# test using command-line arguments, so use check-local target instead of TESTS
+check-local:
+	for pytest in $(PYTESTS) ; do \
+	echo Running test: $$pytest ; \
+	env PYTHONPATH=$(abs_top_srcdir)/src/lib/python:$(abs_top_builddir)/src/bin/loadzone \
+	$(SHELL) $(abs_srcdir)/$$pytest ; \
+	done

+ 11 - 0
src/bin/loadzone/tests/error/error.known

@@ -0,0 +1,11 @@
+Error reading zone file: Cannot parse RR, No $ORIGIN: @ IN SOA ns hostmaster 1 3600 1800 1814400 3600
+Error reading zone file: $ORIGIN is not absolute in record:$ORIGIN com
+Error reading zone file: Cannot parse RR: $TL 300
+Error reading zone file: Cannot parse RR: $OIGIN com.
+Error loading database: Error while loading com.: Cannot parse RR: $INLUDE file.txt
+Error loading database: Error while loading com.: Invalid $include format
+Error loading database: Error while loading com.: Cannot parse RR, No $ORIGIN:  include.txt sub
+Error reading zone file: Invalid TTL: ""
+Error reading zone file: Invalid TTL: "M"
+Error loading database: Error while loading com.: Cannot parse RR: b "no type error!"
+Error reading zone file: Could not open bogusfile

+ 75 - 0
src/bin/loadzone/tests/error/error_test.sh.in

@@ -0,0 +1,75 @@
+#! /bin/sh
+
+# Copyright (C) 2010  Internet Systems Consortium.
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
+# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
+# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+PYTHON_EXEC=${PYTHON_EXEC:-@PYTHON@}
+export PYTHON_EXEC
+
+PYTHONPATH=@abs_top_builddir@/src/lib/python
+export PYTHONPATH
+
+LOADZONE_PATH=@abs_top_srcdir@/src/bin/loadzone
+export LOADZONE_PATH
+status=0
+cd ${LOADZONE_PATH}/tests/error
+
+echo "Test no \$ORIGIN error in zone file"
+${LOADZONE_PATH}/b10-loadzone -d zone.sqlite3  originerr1.db 1> /dev/null 2> error.out
+${LOADZONE_PATH}/b10-loadzone -d zone.sqlite3  originerr2.db 1> /dev/null 2>> error.out
+
+echo "Test: key word TTL spell error"
+${LOADZONE_PATH}/b10-loadzone -d zone.sqlite3  keyerror1.db 1> /dev/null 2>> error.out
+
+echo "Test: key word ORIGIN spell error"
+${LOADZONE_PATH}/b10-loadzone -d zone.sqlite3  keyerror2.db 1> /dev/null 2>> error.out
+
+echo "Test: key INCLUDE spell error"
+${LOADZONE_PATH}/b10-loadzone -d zone.sqlite3  keyerror3.db 1> /dev/null 2>> error.out
+
+echo "Test: include formal error, miss filename"
+${LOADZONE_PATH}/b10-loadzone -d zone.sqlite3  formerr1.db 1> /dev/null 2>>error.out
+
+echo "Test: include form error, domain is not absolute"
+${LOADZONE_PATH}/b10-loadzone -d zone.sqlite3  formerr2.db 1> /dev/null 2>> error.out
+
+echo "Test: TTL form error, no ttl value"
+${LOADZONE_PATH}/b10-loadzone -d zone.sqlite3  formerr3.db 1> /dev/null 2>> error.out
+
+echo "Test: TTL form error, ttl value error"
+${LOADZONE_PATH}/b10-loadzone -d zone.sqlite3  formerr4.db 1> /dev/null 2>> error.out
+
+echo "Test: rr form error, no type"
+${LOADZONE_PATH}/b10-loadzone -d zone.sqlite3  formerr5.db 1> /dev/null 2>> error.out
+
+echo "Test: zone file is bogus"
+${LOADZONE_PATH}/b10-loadzone -d zone.sqlite3  bogusfile 1> /dev/null 2>> error.out
+
+diff error.out error.known || status=1
+
+echo "Clean tmp file."
+rm -f error.out
+rm -f zone.sqlite3
+
+echo "I:exit status:$status"
+echo "-----------------------------------------------------------------------------"
+echo "Ran 11 test files"
+echo ""
+if [ "$status" -eq 1 ];then
+    echo "ERROR"
+else 
+    echo "OK"
+fi
+exit $status

+ 13 - 0
src/bin/loadzone/tests/error/formerr1.db

@@ -0,0 +1,13 @@
+$TTL 300
+$ORIGIN com.
+@			IN SOA	ns hostmaster (
+				1        ; serial
+				3600
+				1800
+				1814400
+				3600
+				)
+			NS	ns
+ns			A	127.0.0.1
+$INCLUDE
+a			A	10.0.0.1

+ 12 - 0
src/bin/loadzone/tests/error/formerr2.db

@@ -0,0 +1,12 @@
+$TTL 300
+com.			IN SOA	ns.com. hostmaster.com. (
+				1        ; serial
+				3600
+				1800
+				1814400
+				3600
+				)
+			NS	ns.example.com.
+ns.com.			A	127.0.0.1
+$INCLUDE include.txt sub
+a.com.			A	10.0.0.1

+ 12 - 0
src/bin/loadzone/tests/error/formerr3.db

@@ -0,0 +1,12 @@
+$TTL 
+$ORIGIN com.
+@			IN SOA	ns hostmaster (
+				1        ; serial
+				3600
+				1800
+				1814400
+				3600
+				)
+			NS	ns
+ns			A	127.0.0.1
+a			A	10.0.0.1

+ 12 - 0
src/bin/loadzone/tests/error/formerr4.db

@@ -0,0 +1,12 @@
+$TTL M
+$ORIGIN com.
+@			IN SOA	ns hostmaster (
+				1        ; serial
+				3600
+				1800
+				1814400
+				3600
+				)
+			NS	ns
+ns			A	127.0.0.1
+a			A	10.0.0.1

+ 13 - 0
src/bin/loadzone/tests/error/formerr5.db

@@ -0,0 +1,13 @@
+$TTL 2M
+$ORIGIN com.
+@			IN SOA	ns hostmaster (
+				1        ; serial
+				3600
+				1800
+				1814400
+				3600
+				)
+			NS	ns
+ns			A	127.0.0.1 ; ip value
+b               "no type error!"
+a			A	10.0.0.1

+ 1 - 0
src/bin/loadzone/tests/error/include.txt

@@ -0,0 +1 @@
+a  300 A 127.0.0.1

+ 12 - 0
src/bin/loadzone/tests/error/keyerror1.db

@@ -0,0 +1,12 @@
+$TL 300
+@ORIGIN com.
+@			IN SOA	ns hostmaster (
+				1        ; serial
+				3600
+				1800
+				1814400
+				3600
+				)
+			NS	ns
+ns			A	127.0.0.1
+a			A	10.0.0.1

+ 12 - 0
src/bin/loadzone/tests/error/keyerror2.db

@@ -0,0 +1,12 @@
+$TTL 300
+$OIGIN com.
+@			IN SOA	ns hostmaster (
+				1        ; serial
+				3600
+				1800
+				1814400
+				3600
+				)
+			NS	ns
+ns			A	127.0.0.1
+a			A	10.0.0.1

+ 13 - 0
src/bin/loadzone/tests/error/keyerror3.db

@@ -0,0 +1,13 @@
+$TTL 300
+$ORIGIN com.
+@			IN SOA	ns hostmaster (
+				1        ; serial
+				3600
+				1800
+				1814400
+				3600
+				)
+			NS	ns
+ns			A	127.0.0.1
+$INLUDE file.txt
+a			A	10.0.0.1

+ 11 - 0
src/bin/loadzone/tests/error/originerr1.db

@@ -0,0 +1,11 @@
+$TTL 300
+@			IN SOA	ns hostmaster (
+				1        ; serial
+				3600
+				1800
+				1814400
+				3600
+				)
+			NS	ns
+ns			A	127.0.0.1
+a			A	10.0.0.1

+ 12 - 0
src/bin/loadzone/tests/error/originerr2.db

@@ -0,0 +1,12 @@
+$TTL 300
+$ORIGIN com
+@			IN SOA	ns hostmaster (
+				1        ; serial
+				3600
+				1800
+				1814400
+				3600
+				)
+			NS	ns
+ns			A	127.0.0.1
+a			A	10.0.0.1

src/bin/loadzone/testdata/Kexample.com.+005+04456.key → src/bin/loadzone/tests/normal/Kexample.com.+005+04456.key


src/bin/loadzone/testdata/Kexample.com.+005+04456.private → src/bin/loadzone/tests/normal/Kexample.com.+005+04456.private


src/bin/loadzone/testdata/Kexample.com.+005+33495.key → src/bin/loadzone/tests/normal/Kexample.com.+005+33495.key


src/bin/loadzone/testdata/Kexample.com.+005+33495.private → src/bin/loadzone/tests/normal/Kexample.com.+005+33495.private


src/bin/loadzone/testdata/Ksql1.example.com.+005+12447.key → src/bin/loadzone/tests/normal/Ksql1.example.com.+005+12447.key


src/bin/loadzone/testdata/Ksql1.example.com.+005+12447.private → src/bin/loadzone/tests/normal/Ksql1.example.com.+005+12447.private


src/bin/loadzone/testdata/Ksql1.example.com.+005+33313.key → src/bin/loadzone/tests/normal/Ksql1.example.com.+005+33313.key


src/bin/loadzone/testdata/Ksql1.example.com.+005+33313.private → src/bin/loadzone/tests/normal/Ksql1.example.com.+005+33313.private


src/bin/loadzone/testdata/Ksql2.example.com.+005+38482.key → src/bin/loadzone/tests/normal/Ksql2.example.com.+005+38482.key


src/bin/loadzone/testdata/Ksql2.example.com.+005+38482.private → src/bin/loadzone/tests/normal/Ksql2.example.com.+005+38482.private


src/bin/loadzone/testdata/Ksql2.example.com.+005+63192.key → src/bin/loadzone/tests/normal/Ksql2.example.com.+005+63192.key


src/bin/loadzone/testdata/Ksql2.example.com.+005+63192.private → src/bin/loadzone/tests/normal/Ksql2.example.com.+005+63192.private


src/bin/loadzone/testdata/README → src/bin/loadzone/tests/normal/README


src/bin/loadzone/testdata/dsset-subzone.example.com. → src/bin/loadzone/tests/normal/dsset-subzone.example.com


src/bin/loadzone/testdata/example.com → src/bin/loadzone/tests/normal/example.com


src/bin/loadzone/testdata/example.com.signed → src/bin/loadzone/tests/normal/example.com.signed


src/bin/loadzone/testdata/sql1.example.com → src/bin/loadzone/tests/normal/sql1.example.com


src/bin/loadzone/testdata/sql1.example.com.signed → src/bin/loadzone/tests/normal/sql1.example.com.signed


src/bin/loadzone/testdata/sql2.example.com → src/bin/loadzone/tests/normal/sql2.example.com


src/bin/loadzone/testdata/sql2.example.com.signed → src/bin/loadzone/tests/normal/sql2.example.com.signed


+ 195 - 63
src/lib/python/isc/datasrc/master.py

@@ -16,7 +16,8 @@
 # $Id$
 # $Id$
 
 
 import sys, re, string
 import sys, re, string
-
+import time
+import os
 #########################################################################
 #########################################################################
 # define exceptions
 # define exceptions
 #########################################################################
 #########################################################################
@@ -102,7 +103,7 @@ def isname(s):
 # isttl: check whether a string is a valid TTL specifier.
 # isttl: check whether a string is a valid TTL specifier.
 # returns: boolean
 # returns: boolean
 #########################################################################
 #########################################################################
-ttl_regex = re.compile('[0-9]+[wdhms]?', re.I)
+ttl_regex = re.compile('([0-9]+[wdhms]?)+', re.I)
 def isttl(s):
 def isttl(s):
     global ttl_regex
     global ttl_regex
     if ttl_regex.match(s):
     if ttl_regex.match(s):
@@ -121,19 +122,26 @@ def isttl(s):
 #   MasterFileError
 #   MasterFileError
 #########################################################################
 #########################################################################
 def parse_ttl(s):
 def parse_ttl(s):
-    m = re.match('([0-9]+)(.*)', s)
-    if not m:
+    sum = 0
+    if not isttl(s):
         raise MasterFileError('Invalid TTL: ' + s)
         raise MasterFileError('Invalid TTL: ' + s)
-    ttl, suffix = int(m.group(1)), m.group(2)
-    if suffix.lower() == 'w':
-        ttl *= 604800
-    elif suffix.lower() == 'd':
-        ttl *= 86400
-    elif suffix.lower() == 'h':
-        ttl *= 3600
-    elif suffix.lower() == 'm':
-        ttl *= 60
-    return str(ttl)
+    for ttl_expr in re.findall('\d+[wdhms]?', s, re.I):
+        if ttl_expr.isdigit():
+            ttl = int(ttl_expr)
+            sum += ttl
+            continue
+        ttl = int(ttl_expr[:-1])
+        suffix = ttl_expr[-1].lower()
+        if suffix == 'w':
+            ttl *= 604800
+        elif suffix == 'd':
+            ttl *= 86400
+        elif suffix == 'h':
+            ttl *= 3600
+        elif suffix == 'm':
+            ttl *= 60
+        sum += ttl
+    return str(sum)
 
 
 #########################################################################
 #########################################################################
 # records: generator function to return complete RRs from the zone file,
 # records: generator function to return complete RRs from the zone file,
@@ -147,7 +155,9 @@ def records(input):
     record = []
     record = []
     complete = True
     complete = True
     paren = 0
     paren = 0
+    size = 0
     for line in input:
     for line in input:
+        size += len(line)
         list = cleanup(line).split()
         list = cleanup(line).split()
         for word in list:
         for word in list:
             if paren == 0:
             if paren == 0:
@@ -169,10 +179,12 @@ def records(input):
 
 
         if paren == 1 or not record:
         if paren == 1 or not record:
             continue
             continue
-
+        
         ret = ' '.join(record)
         ret = ' '.join(record)
         record = []
         record = []
-        yield ret
+        oldsize = size
+        size = 0
+        yield ret, oldsize
 
 
 #########################################################################
 #########################################################################
 # define the MasterFile class for reading zone master files
 # define the MasterFile class for reading zone master files
@@ -181,24 +193,62 @@ class MasterFile:
     __rrclass = 'IN'
     __rrclass = 'IN'
     __maxttl = 0x7fffffff
     __maxttl = 0x7fffffff
     __ttl = ''
     __ttl = ''
+    __lastttl = ''
     __zonefile = ''
     __zonefile = ''
     __name = ''
     __name = ''
+    __file_level = 0
+    __file_type = ""
+    __init_time = time.time()
+    __records_num = 0
 
 
-    def __init__(self, filename, initial_origin = ''):
-        if initial_origin == '.':
-            initial_origin = ''
+    def __init__(self, filename, initial_origin = '', verbose = False):
         self.__initial_origin = initial_origin
         self.__initial_origin = initial_origin
         self.__origin = initial_origin
         self.__origin = initial_origin
+        self.__datafile = filename
+
+        try:
+            self.__zonefile = open(filename, 'r')
+        except:
+            raise MasterFileError("Could not open " + filename)
+        self.__filesize = os.fstat(self.__zonefile.fileno()).st_size
+
+        self.__cur = 0
+        self.__numback = 0
+        self.__verbose = verbose
         try:
         try:
             self.__zonefile = open(filename, 'r')
             self.__zonefile = open(filename, 'r')
         except:
         except:
             raise MasterFileError("Could not open " + filename)
             raise MasterFileError("Could not open " + filename)
 
 
+    def __status(self):
+        interval = time.time() - MasterFile.__init_time
+        if self.__filesize == 0:
+            percent = 100
+        else:
+            percent = (self.__cur * 100)/self.__filesize
+
+        sys.stdout.write("\r" + (80 * " "))
+        sys.stdout.write("\r%d RR(s) loaded in %d second(s) (%.2f%% of %s%s)"\
+                % (MasterFile.__records_num, interval, percent, MasterFile.__file_type, self.__datafile))
+
     def __del__(self):
     def __del__(self):
         if self.__zonefile:
         if self.__zonefile:
             self.__zonefile.close()
             self.__zonefile.close()
-
-    #########################################################################
+    ########################################################################
+    # check if the zonename is relative
+    # no then return
+    # yes , sets the relative domain name to the stated name
+    #######################################################################
+    def __statedname(self, name, record):
+        if name[-1] != '.':
+            if not self.__origin:
+                raise MasterFileError("Cannot parse RR, No $ORIGIN: " + record)
+            elif self.__origin == '.':
+                name += '.'
+            else:
+                name += '.' + self.__origin
+        return name
+    #####################################################################
     # handle $ORIGIN, $TTL and $GENERATE directives
     # handle $ORIGIN, $TTL and $GENERATE directives
     # (currently only $ORIGIN and $TTL are implemented)
     # (currently only $ORIGIN and $TTL are implemented)
     # input:
     # input:
@@ -216,20 +266,22 @@ class MasterFile:
                 raise MasterFileError('Invalid $ORIGIN')
                 raise MasterFileError('Invalid $ORIGIN')
             if more:
             if more:
                 raise MasterFileError('Invalid $ORIGIN')
                 raise MasterFileError('Invalid $ORIGIN')
-            if second == '.':
-                self.__origin = ''
-            elif second[-1] == '.':
+            if second[-1] == '.':
                 self.__origin = second
                 self.__origin = second
-            else:
+            elif not self.__origin:
+                raise MasterFileError("$ORIGIN is not absolute in record:%s" % s)
+            elif self.__origin != '.':
                 self.__origin = second + '.' + self.__origin
                 self.__origin = second + '.' + self.__origin
+            else:
+                self.__origin = second + '.'
             return True
             return True
         elif re.match('\$ttl', first, re.I):
         elif re.match('\$ttl', first, re.I):
             if not second or not isttl(second):
             if not second or not isttl(second):
                 raise MasterFileError('Invalid TTL: "' + second + '"')
                 raise MasterFileError('Invalid TTL: "' + second + '"')
             if more:
             if more:
                 raise MasterFileError('Invalid $TTL statement')
                 raise MasterFileError('Invalid $TTL statement')
-            self.__ttl = parse_ttl(second)
-            if int(self.__ttl) > self.__maxttl:
+            MasterFile.__ttl = parse_ttl(second)
+            if int(MasterFile.__ttl) > self.__maxttl:
                 raise MasterFileError('TTL too high: ' + second)
                 raise MasterFileError('TTL too high: ' + second)
             return True
             return True
         elif re.match('\$generate', first, re.I):
         elif re.match('\$generate', first, re.I):
@@ -246,14 +298,28 @@ class MasterFile:
     # throws:
     # throws:
     #   MasterFileError
     #   MasterFileError
     #########################################################################
     #########################################################################
-    __filename = re.compile('[\"\']*([^\'\"]+)[\"\']*')
+    __include_syntax1 = re.compile('\s+(\S+)(?:\s+(\S+))?$', re.I)
+    __include_syntax2 = re.compile('\s+"([^"]+)"(?:\s+(\S+))?$', re.I)
+    __include_syntax3 = re.compile("\s+'([^']+)'(?:\s+(\S+))?$", re.I)
     def __include(self, s):
     def __include(self, s):
-        first, rest = pop(s)
-        if re.match('\$include', first, re.I):
-            m = self.__filename.match(rest)
-            if m:
-                file = m.group(1)
-                return file
+        if not s.lower().startswith('$include'):
+            return "", ""
+        s = s[len('$include'):]
+        m = self.__include_syntax1.match(s)
+        if not m:
+            m = self.__include_syntax2.match(s)
+        if not m:
+            m = self.__include_syntax3.match(s)
+        if not m:
+            raise MasterFileError('Invalid $include format')
+        file = m.group(1)
+        if m.group(2):
+            if not isname(m.group(2)):
+                raise MasterFileError('Invalid $include format (invalid origin)')
+            origin = self.__statedname(m.group(2), s)
+        else:
+            origin = self.__origin
+        return file, origin
 
 
     #########################################################################
     #########################################################################
     # try parsing an RR on the assumption that the type is specified in
     # try parsing an RR on the assumption that the type is specified in
@@ -272,6 +338,14 @@ class MasterFile:
         if istype(list[3]):
         if istype(list[3]):
             if isclass(list[2]) and isttl(list[1]) and isname(list[0]):
             if isclass(list[2]) and isttl(list[1]) and isname(list[0]):
                 name, ttl, rrclass, rrtype = list[0:4]
                 name, ttl, rrclass, rrtype = list[0:4]
+                ttl = parse_ttl(ttl)
+                MasterFile.__lastttl = ttl or MasterFile.__lastttl
+                rdata = ' '.join(list[4:])
+                ret = name, ttl, rrclass, rrtype, rdata
+            elif isclass(list[1]) and isttl(list[2]) and isname(list[0]):
+                name, rrclass, ttl, rrtype = list[0:4]
+                ttl = parse_ttl(ttl)
+                MasterFile.__lastttl = ttl or MasterFile.__lastttl
                 rdata = ' '.join(list[4:])
                 rdata = ' '.join(list[4:])
                 ret = name, ttl, rrclass, rrtype, rdata
                 ret = name, ttl, rrclass, rrtype, rdata
         return ret
         return ret
@@ -284,6 +358,9 @@ class MasterFile:
     # returns:
     # returns:
     #   empty list if parse failed, else name, ttl, class, type, rdata
     #   empty list if parse failed, else name, ttl, class, type, rdata
     #########################################################################
     #########################################################################
+    def __getttl(self):
+        return MasterFile.__ttl or MasterFile.__lastttl
+
     def __three(self, record, curname):
     def __three(self, record, curname):
         ret = ''
         ret = ''
         list = record.split()
         list = record.split()
@@ -292,19 +369,25 @@ class MasterFile:
         if istype(list[2]) and not istype(list[1]):
         if istype(list[2]) and not istype(list[1]):
             if isclass(list[1]) and not isttl(list[0]) and isname(list[0]):
             if isclass(list[1]) and not isttl(list[0]) and isname(list[0]):
                 rrclass = list[1]
                 rrclass = list[1]
-                ttl = self.__ttl
+                ttl = self.__getttl()
                 name = list[0]
                 name = list[0]
-            elif not isclass(list[1]) and isttl(list[1]) and isname(list[0]):
+            elif not isclass(list[1]) and isttl(list[1]) and not isclass(list[0]) and isname(list[0]):
                 rrclass = self.__rrclass
                 rrclass = self.__rrclass
                 ttl = parse_ttl(list[1])
                 ttl = parse_ttl(list[1])
+                MasterFile.__lastttl = ttl or MasterFile.__lastttl
                 name = list[0]
                 name = list[0]
             elif curname and isclass(list[1]) and isttl(list[0]):
             elif curname and isclass(list[1]) and isttl(list[0]):
-                rrclass = self.__rrclass
+                rrclass = list[1]
                 ttl = parse_ttl(list[0])
                 ttl = parse_ttl(list[0])
+                MasterFile.__lastttl = ttl or MasterFile.__lastttl
+                name = curname
+            elif curname and isttl(list[1]) and isclass(list[0]):
+                rrclass = list[0]
+                ttl = parse_ttl(list[1])
+                MasterFile.__lastttl = ttl or MasterFile.__lastttl
                 name = curname
                 name = curname
             else:
             else:
                 return ret
                 return ret
-
             rrtype = list[2]
             rrtype = list[2]
             rdata = ' '.join(list[3:])
             rdata = ' '.join(list[3:])
             ret = name, ttl, rrclass, rrtype, rdata
             ret = name, ttl, rrclass, rrtype, rdata
@@ -325,46 +408,81 @@ class MasterFile:
         list = record.split()
         list = record.split()
         if len(list) <= 2:
         if len(list) <= 2:
             return ret
             return ret
-
         if istype(list[1]):
         if istype(list[1]):
             rrclass = self.__rrclass
             rrclass = self.__rrclass
             rrtype = list[1]
             rrtype = list[1]
             if list[0].lower() == 'rrsig':
             if list[0].lower() == 'rrsig':
                 name = curname
                 name = curname
-                ttl = self.__ttl
+                ttl = self.__getttl()
                 rrtype = list[0]
                 rrtype = list[0]
                 rdata = ' '.join(list[1:])
                 rdata = ' '.join(list[1:])
             elif isttl(list[0]):
             elif isttl(list[0]):
                 ttl = parse_ttl(list[0])
                 ttl = parse_ttl(list[0])
                 name = curname
                 name = curname
                 rdata = ' '.join(list[2:])
                 rdata = ' '.join(list[2:])
+            elif isclass(list[0]):
+                ttl = self.__getttl()
+                name = curname
+                rdata = ' '.join(list[2:])
             elif isname(list[0]):
             elif isname(list[0]):
                 name = list[0]
                 name = list[0]
-                ttl = self.__ttl
+                ttl = self.__getttl()
                 rdata = ' '.join(list[2:])
                 rdata = ' '.join(list[2:])
             else:
             else:
                 raise MasterFileError("Cannot parse RR: " + record)
                 raise MasterFileError("Cannot parse RR: " + record)
 
 
             ret = name, ttl, rrclass, rrtype, rdata
             ret = name, ttl, rrclass, rrtype, rdata
-
         return ret
         return ret
 
 
+    ########################################################################
+    #close verbose
+    ######################################################################
+    def closeverbose(self):
+        self.__status()
+
     #########################################################################
     #########################################################################
     # zonedata: generator function to parse a zone master file and return
     # zonedata: generator function to parse a zone master file and return
     # each RR as a (name, ttl, type, class, rdata) tuple
     # each RR as a (name, ttl, type, class, rdata) tuple
     #########################################################################
     #########################################################################
     def zonedata(self):
     def zonedata(self):
         name = ''
         name = ''
-
-        for record in records(self.__zonefile):
+        last_status = 0.0
+        flag = 1
+
+        for record, size in records(self.__zonefile):
+            if self.__verbose:
+                now = time.time()
+                if flag == 1:
+                    self.__status()
+                    flag = 0
+                if now - last_status >= 1.0:
+                    self.__status()
+                    last_status = now
+
+            self.__cur += size
             if self.__directive(record):
             if self.__directive(record):
                 continue
                 continue
 
 
-            incl = self.__include(record)
+            incl, suborigin = self.__include(record)
             if incl:
             if incl:
-                sub = MasterFile(incl, self.__origin)
-                for name, ttl, rrclass, rrtype, rdata in sub.zonedata():
-                    yield (name, ttl, rrclass, rrtype, rdata)
+                if self.__filesize == 0:
+                    percent = 100
+                else:
+                    percent = (self.__cur * 100)/self.__filesize
+                if self.__verbose:
+                    sys.stdout.write("\r" + (80 * " "))
+                    sys.stdout.write("\rIncluding \"%s\" from \"%s\"\n" % (incl, self.__datafile))
+                MasterFile.__file_level += 1
+                MasterFile.__file_type = "included "
+                sub = MasterFile(incl, suborigin, self.__verbose)
+
+                for rrname, ttl, rrclass, rrtype, rdata in sub.zonedata():
+                    yield (rrname, ttl, rrclass, rrtype, rdata)
+                if self.__verbose:
+                    sub.closeverbose()
+                MasterFile.__file_level -= 1
+                if MasterFile.__file_level == 0:
+                    MasterFile.__file_type = ""
                 del sub
                 del sub
                 continue
                 continue
 
 
@@ -373,7 +491,7 @@ class MasterFile:
             if rl[0] == '@':
             if rl[0] == '@':
                 rl[0] = self.__origin
                 rl[0] = self.__origin
                 if not self.__origin:
                 if not self.__origin:
-                    rl[0] = '.'
+                    raise MasterFileError("Cannot parse RR, No $ORIGIN: " + record)
                 record = ' '.join(rl)
                 record = ' '.join(rl)
 
 
             result = self.__four(record, name)
             result = self.__four(record, name)
@@ -387,36 +505,43 @@ class MasterFile:
             if not result:
             if not result:
                 first, rdata = pop(record)
                 first, rdata = pop(record)
                 if istype(first):
                 if istype(first):
-                    result = name, self.__ttl, self.__rrclass, first, rdata
+                    result = name, self.__getttl(), self.__rrclass, first, rdata
 
 
             if not result:
             if not result:
                 raise MasterFileError("Cannot parse RR: " + record)
                 raise MasterFileError("Cannot parse RR: " + record)
 
 
             name, ttl, rrclass, rrtype, rdata = result
             name, ttl, rrclass, rrtype, rdata = result
-            if name[-1] != '.':
-                name += '.' + self.__origin
+            name = self.__statedname(name, record)
 
 
             if rrclass.lower() != 'in':
             if rrclass.lower() != 'in':
                 raise MasterFileError("CH and HS zones not supported")
                 raise MasterFileError("CH and HS zones not supported")
 
 
-            if not ttl:
-                raise MasterFileError("No TTL specified; zone rejected")
-
             # add origin to rdata containing names, if necessary
             # add origin to rdata containing names, if necessary
             if rrtype.lower() in ('cname', 'dname', 'ns', 'ptr'):
             if rrtype.lower() in ('cname', 'dname', 'ns', 'ptr'):
                 if not isname(rdata):
                 if not isname(rdata):
                     raise MasterFileError("Invalid " + rrtype + ": " + rdata)
                     raise MasterFileError("Invalid " + rrtype + ": " + rdata)
-                if rdata[-1] != '.':
-                    rdata += '.' + self.__origin
+                rdata = self.__statedname(rdata, record)
+
             if rrtype.lower() == 'soa':
             if rrtype.lower() == 'soa':
                 soa = rdata.split()
                 soa = rdata.split()
                 if len(soa) < 2 or not isname(soa[0]) or not isname(soa[1]):
                 if len(soa) < 2 or not isname(soa[0]) or not isname(soa[1]):
                     raise MasterFileError("Invalid " + rrtype + ": " + rdata)
                     raise MasterFileError("Invalid " + rrtype + ": " + rdata)
-                if soa[0][-1] != '.':
-                    soa[0] += '.' + self.__origin
-                if soa[1][-1] != '.':
-                    soa[1] += '.' + self.__origin
+                soa[0] = self.__statedname(soa[0], record)
+                soa[1] = self.__statedname(soa[1], record)
+                if not MasterFile.__ttl and not ttl:
+                    MasterFile.__ttl = MasterFile.__ttl or parse_ttl(soa[-1])
+                    ttl = MasterFile.__ttl
+
+                for index in range(3, len(soa)):
+                    if isttl(soa[index]):
+                        soa[index] = parse_ttl(soa[index])
+                    else :
+                        raise MasterFileError("No TTL specified; in soa record!")
                 rdata = ' '.join(soa)
                 rdata = ' '.join(soa)
+
+            if not ttl:
+                raise MasterFileError("No TTL specified; zone rejected")
+
             if rrtype.lower() == 'mx':
             if rrtype.lower() == 'mx':
                 mx = rdata.split()
                 mx = rdata.split()
                 if len(mx) != 2 or not isname(mx[1]):
                 if len(mx) != 2 or not isname(mx[1]):
@@ -424,7 +549,7 @@ class MasterFile:
                 if mx[1][-1] != '.':
                 if mx[1][-1] != '.':
                     mx[1] += '.' + self.__origin
                     mx[1] += '.' + self.__origin
                     rdata = ' '.join(mx)
                     rdata = ' '.join(mx)
-
+            MasterFile.__records_num += 1
             yield (name, ttl, rrclass, rrtype, rdata)
             yield (name, ttl, rrclass, rrtype, rdata)
 
 
     #########################################################################
     #########################################################################
@@ -436,16 +561,22 @@ class MasterFile:
             return self.__name
             return self.__name
         old_origin = self.__origin
         old_origin = self.__origin
         self.__origin = self.__initial_origin
         self.__origin = self.__initial_origin
+        cur_value = self.__cur
         old_location = self.__zonefile.tell()
         old_location = self.__zonefile.tell()
+        old_verbose = self.__verbose
+        self.__verbose = False
         self.__zonefile.seek(0)
         self.__zonefile.seek(0)
+
         for name, ttl, rrclass, rrtype, rdata in self.zonedata():
         for name, ttl, rrclass, rrtype, rdata in self.zonedata():
             if rrtype.lower() == 'soa':
             if rrtype.lower() == 'soa':
                 break
                 break
         self.__zonefile.seek(old_location)
         self.__zonefile.seek(old_location)
         self.__origin = old_origin
         self.__origin = old_origin
+        self.__cur = cur_value
         if rrtype.lower() != 'soa':
         if rrtype.lower() != 'soa':
             raise MasterFileError("No SOA found")
             raise MasterFileError("No SOA found")
         self.__name = name
         self.__name = name
+        self.__verbose = old_verbose
         return name
         return name
 
 
     #########################################################################
     #########################################################################
@@ -454,7 +585,8 @@ class MasterFile:
     def reset(self):
     def reset(self):
         self.__zonefile.seek(0)
         self.__zonefile.seek(0)
         self.__origin = self.__initial_origin
         self.__origin = self.__initial_origin
-        self.__ttl = ''
+        MasterFile.__ttl = ''
+        MasterFile.__lastttl = ''
 
 
 #########################################################################
 #########################################################################
 # main: used for testing; parse a zone file and print out each record
 # main: used for testing; parse a zone file and print out each record