Rename my pycoverage variable, since need it without the arguments too.

For loadzone tests, don't use "python" variable names.

In configure be able to set path of pycoverage too.
And report it at end of configure.

Add make targets:

 clean-cpp-coverage (was clean-coverage)
 clean-python-coverage
 report-cpp-coverage (was report-coverage)
 report-python-coverage (this creates ./py-coverage-html directory)

Now generic make targets for these do both.


git-svn-id: svn://bind10.isc.org/svn/bind10/branches/trac362@3510 e5f2f494-b856-4b98-b285-d166d9295462

Makefile.am

@@ -8,7 +8,7 @@ DISTCLEANFILES = config.report
 # When running distcheck target, do not install the configurations
 DISTCHECK_CONFIGURE_FLAGS = --disable-install-configurations
 
-clean-coverage:
+clean-cpp-coverage:
 	@if [ $(USE_LCOV) = yes ] ; then \
 		$(LCOV) --directory . --zerocounters; \
 		rm -rf coverage/; \
@@ -17,9 +17,12 @@ clean-coverage:
 		exit 1; \
 	fi
 
+clean-python-coverage:
+	rm -f $(abs_top_srcdir)/.coverage $(abs_top_srcdir)/py-coverage-html
+
 perform-coverage: check
 
-report-coverage:
+report-cpp-coverage:
 	$(LCOV) --capture --directory . --output-file all.info
 	$(LCOV) --remove all.info \
 			c++/4.4\*/\* \
@@ -39,8 +42,18 @@ report-coverage:
 		--output report.info
 	$(GENHTML) -o coverage report.info 
 
+report-python-coverage:
+	@if [ $(USE_PYCOVERAGE) = yes ] ; then \
+		$(PYCOVERAGE) html -d py-coverage-html --omit=src/lib/python/isc/config/tests,src/bin/bind10/tests,src/bin/bindctl/tests/,src/bin/cfgmgr/tests/,src/bin/cmdctl/tests/,src/bin/msgq/tests/,src/bin/tests/,src/bin/xfrin/tests/,src/bin/xfrout/tests/,src/bin/zonemgr/tests/,src/lib/dns/python/tests/,src/lib/python/isc/cc/tests/,src/lib/python/isc/log/tests/,src/lib/python/isc/utils/tests/,src/lib/python/isc/notify/tests ; \
+	fi
+
+# for python and c++ test coverage
 coverage: clean-coverage perform-coverage report-coverage
 
+clean-coverage: clean-cpp-coverage clean-python-coverage
+
+report-coverage: report-cpp-coverage report-python-coverage
+
 #### include external sources in the distributed tarball:
 EXTRA_DIST = ext/asio/README
 EXTRA_DIST += ext/asio/asio/local/stream_protocol.hpp

configure.ac

@@ -256,13 +256,22 @@ AC_TRY_COMPILE([
 
 AC_ARG_WITH(pycoverage,
 [  --with-pycoverage[=PROGRAM]         enable python code coverage using the specified coverage], pycoverage="$withval", pycoverage="no")
-if test "$pycoverage" != "no" ; then
-	PYCOVERAGE="coverage run --branch --append"
+if test "$pycoverage" = "no" ; then
+	# just run the tests normally with python
+	PYCOVERAGE_RUN="${PYTHON}"
+	USE_PYCOVERAGE="no"
+elif test "$pycoverage" = "yes" ; then
+	PYCOVERAGE="coverage"
+	PYCOVERAGE_RUN="${PYCOVERAGE} run --branch --append"
+	USE_PYCOVERAGE="yes"
 else
-	PYCOVERAGE="${PYTHON}"
+	PYCOVERAGE="$pycoverage"
+	PYCOVERAGE_RUN="${PYCOVERAGE} run --branch --append"
+	USE_PYCOVERAGE="yes"
 fi
-AM_CONDITIONAL(ENABLE_PYTHON_COVERAGE, test x$pycoverage != xno)
+AM_CONDITIONAL(ENABLE_PYTHON_COVERAGE, test x$USE_PYCOVERAGE != xno)
 AC_SUBST(PYCOVERAGE)
+AC_SUBST(USE_PYCOVERAGE)
 
 AC_ARG_WITH(lcov,
 [  --with-lcov[=PROGRAM]         enable gtest and coverage target using the specified lcov], lcov="$withval", lcov="no")
@@ -616,6 +625,7 @@ Features:
 Developer:
   Google Tests:  $gtest_path
   C++ Code Coverage: $USE_LCOV
+  Python Code Coverage: $USE_PYCOVERAGE
   Generate Manuals:  $enable_man
 
 END

src/bin/bind10/tests/Makefile.am

@@ -1,4 +1,4 @@
-PYCOVERAGE = @PYCOVERAGE@
+PYCOVERAGE_RUN = @PYCOVERAGE_RUN@
 #PYTESTS = args_test.py bind10_test.py
 PYTESTS = bind10_test.py
 EXTRA_DIST = $(PYTESTS)
@@ -13,5 +13,5 @@ endif
 	for pytest in $(PYTESTS) ; do \
 	echo Running test: $$pytest ; \
 	env PYTHONPATH=$(abs_top_srcdir)/src/lib/python:$(abs_top_builddir)/src/lib/python:$(abs_top_builddir)/src/bin/bind10 \
-	$(PYCOVERAGE) $(abs_srcdir)/$$pytest || exit ; \
+	$(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \
 	done

src/bin/bindctl/tests/Makefile.am

@@ -1,4 +1,4 @@
-PYCOVERAGE = @PYCOVERAGE@
+PYCOVERAGE_RUN = @PYCOVERAGE_RUN@
 PYTESTS = bindctl_test.py
 EXTRA_DIST = $(PYTESTS)
 
@@ -12,5 +12,5 @@ endif
 	for pytest in $(PYTESTS) ; do \
 	echo Running test: $$pytest ; \
 	env PYTHONPATH=$(abs_top_srcdir)/src/lib/python:$(abs_top_builddir)/src/lib/python:$(abs_top_srcdir)/src/bin \
-	$(PYCOVERAGE) $(abs_srcdir)/$$pytest || exit ; \
+	$(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \
 	done

src/bin/cfgmgr/tests/Makefile.am

@@ -1,4 +1,4 @@
-PYCOVERAGE = @PYCOVERAGE@
+PYCOVERAGE_RUN = @PYCOVERAGE_RUN@
 PYTESTS = b10-cfgmgr_test.py
 
 EXTRA_DIST = $(PYTESTS)
@@ -13,5 +13,5 @@ endif
 	for pytest in $(PYTESTS) ; do \
 	echo Running test: $$pytest ; \
 	env PYTHONPATH=$(abs_top_srcdir)/src/lib/python:$(abs_top_builddir)/src/lib/python:$(abs_top_builddir)/src/bin/cfgmgr \
-	$(PYCOVERAGE) $(abs_builddir)/$$pytest || exit ; \
+	$(PYCOVERAGE_RUN) $(abs_builddir)/$$pytest || exit ; \
 	done

src/bin/cmdctl/tests/Makefile.am

@@ -1,4 +1,4 @@
-PYCOVERAGE=@PYCOVERAGE@
+PYCOVERAGE_RUN=@PYCOVERAGE_RUN@
 PYTESTS = cmdctl_test.py
 EXTRA_DIST = $(PYTESTS)
 
@@ -14,5 +14,5 @@ endif
 	env PYTHONPATH=$(abs_top_srcdir)/src/lib/python:$(abs_top_builddir)/src/lib/python:$(abs_top_builddir)/src/bin/cmdctl \
 	CMDCTL_SPEC_PATH=$(abs_top_builddir)/src/bin/cmdctl \
 	CMDCTL_SRC_PATH=$(abs_top_srcdir)/src/bin/cmdctl \
-	$(PYCOVERAGE) $(abs_srcdir)/$$pytest || exit ; \
+	$(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \
 	done

src/bin/loadzone/tests/correct/Makefile.am

@@ -1,5 +1,3 @@
-PYCOVERAGE = @PYCOVERAGE@
-PYTESTS = correct_test.sh
 EXTRA_DIST = get_zonedatas.py
 EXTRA_DIST += include.db
 EXTRA_DIST += inclsub.db
@@ -15,15 +13,8 @@ EXTRA_DIST += ttl2.db
 EXTRA_DIST += ttlext.db
 EXTRA_DIST += example.db
 
+# TODO: maybe use TESTS?
 # test using command-line arguments, so use check-local target instead of TESTS
 check-local:
-if ENABLE_PYTHON_COVERAGE
-	touch $(abs_top_srcdir)/.coverage 
-	rm -f .coverage
-	${LN_S} $(abs_top_srcdir)/.coverage .coverage
-endif
-	for pytest in $(PYTESTS) ; do \
-	echo Running test: $$pytest ; \
-	env PYTHONPATH=$(abs_top_srcdir)/src/lib/python:$(abs_top_builddir)/src/lib/python:$(abs_top_builddir)/src/bin/loadzone \
-	$(SHELL) $(abs_builddir)/$$pytest || exit ; \
-	done
+	echo Running test: correct_test.sh 
+	$(SHELL) $(abs_builddir)/correct_test.sh

src/bin/loadzone/tests/error/Makefile.am

@@ -1,6 +1,3 @@
-PYCOVERAGE = @PYCOVERAGE@
-PYTESTS = error_test.sh
-
 EXTRA_DIST = error.known
 EXTRA_DIST += formerr1.db 
 EXTRA_DIST += formerr2.db
@@ -15,15 +12,8 @@ EXTRA_DIST += keyerror3.db
 EXTRA_DIST += originerr1.db
 EXTRA_DIST += originerr2.db
 
+# TODO: use TESTS ?
 # test using command-line arguments, so use check-local target instead of TESTS
 check-local:
-if ENABLE_PYTHON_COVERAGE
-	touch $(abs_top_srcdir)/.coverage 
-	rm -f .coverage
-	${LN_S} $(abs_top_srcdir)/.coverage .coverage
-endif
-	for pytest in $(PYTESTS) ; do \
-	echo Running test: $$pytest ; \
-	env PYTHONPATH=$(abs_top_srcdir)/src/lib/python:$(abs_top_builddir)/src/bin/loadzone \
-	$(SHELL) $(abs_builddir)/$$pytest || exit ; \
-	done
+	echo Running test: error_test.sh
+	$(SHELL) $(abs_builddir)/error_test.sh

src/bin/msgq/tests/Makefile.am

@@ -1,4 +1,4 @@
-PYCOVERAGE = @PYCOVERAGE@          
+PYCOVERAGE_RUN = @PYCOVERAGE_RUN@          
 PYTESTS = msgq_test.py
 EXTRA_DIST = $(PYTESTS)
 
@@ -12,6 +12,6 @@ endif
 	for pytest in $(PYTESTS) ; do \
 	echo Running test: $$pytest ; \
 	env PYTHONPATH=$(abs_top_builddir)/src/bin/msgq:$(abs_top_srcdir)/src/lib/python:$(abs_top_builddir)/src/lib/python \
-	$(PYCOVERAGE) $(abs_srcdir)/$$pytest || exit ; \
+	$(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \
 	done
 

src/bin/tests/Makefile.am

@@ -1,4 +1,4 @@
-PYCOVERAGE = @PYCOVERAGE@
+PYCOVERAGE_RUN = @PYCOVERAGE_RUN@
 PYTESTS = process_rename_test.py
 # .py will be generated by configure, so we don't have to include it
 # in EXTRA_DIST.
@@ -13,5 +13,5 @@ endif
 	for pytest in $(PYTESTS) ; do \
 	echo Running test: $$pytest ; \
 	env PYTHONPATH=$(abs_top_srcdir)/src/lib/python:$(abs_top_builddir)/src/lib/python:$(abs_top_builddir)/src/lib/dns/python/.libs \
-	$(PYCOVERAGE) $(abs_builddir)/$$pytest || exit ; \
+	$(PYCOVERAGE_RUN) $(abs_builddir)/$$pytest || exit ; \
 	done

src/bin/xfrin/tests/Makefile.am

@@ -1,4 +1,4 @@
-PYCOVERAGE=@PYCOVERAGE@
+PYCOVERAGE_RUN=@PYCOVERAGE_RUN@
 PYTESTS = xfrin_test.py
 EXTRA_DIST = $(PYTESTS)
 
@@ -20,5 +20,5 @@ endif
 	echo Running test: $$pytest ; \
 	env PYTHONPATH=$(abs_top_builddir)/src/lib/dns/python/.libs:$(abs_top_builddir)/src/bin/xfrin:$(abs_top_srcdir)/src/lib/python:$(abs_top_builddir)/src/lib/python \
 	$(LIBRARY_PATH_PLACEHOLDER) \
-	$(PYCOVERAGE) $(abs_srcdir)/$$pytest || exit ; \
+	$(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \
 	done

src/bin/xfrout/tests/Makefile.am

@@ -1,4 +1,4 @@
-PYCOVERAGE=@PYCOVERAGE@
+PYCOVERAGE_RUN=@PYCOVERAGE_RUN@
 PYTESTS = xfrout_test.py
 EXTRA_DIST = $(PYTESTS)
 
@@ -20,5 +20,5 @@ endif
 	echo Running test: $$pytest ; \
 	env PYTHONPATH=$(abs_top_builddir)/src/bin/xfrout:$(abs_top_srcdir)/src/lib/python:$(abs_top_builddir)/src/lib/python:$(abs_top_builddir)/src/lib/dns/python/.libs:$(abs_top_builddir)/src/lib/xfr/.libs \
 	$(LIBRARY_PATH_PLACEHOLDER) \
-	$(PYCOVERAGE) $(abs_srcdir)/$$pytest || exit ; \
+	$(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \
 	done

src/bin/zonemgr/tests/Makefile.am

@@ -1,4 +1,4 @@
-PYCOVERAGE=@PYCOVERAGE@
+PYCOVERAGE_RUN=@PYCOVERAGE_RUN@
 PYTESTS = zonemgr_test.py
 EXTRA_DIST = $(PYTESTS)
 
@@ -12,5 +12,5 @@ endif
 	for pytest in $(PYTESTS) ; do \
 	echo Running test: $$pytest ; \
 	env PYTHONPATH=$(abs_top_builddir)/src/bin/zonemgr:$(abs_top_srcdir)/src/lib/python:$(abs_top_builddir)/src/lib/python:$(abs_top_builddir)/src/lib/dns/.libs:$(abs_top_builddir)/src/lib/dns/python/.libs:$(abs_top_builddir)/src/lib/xfr/.libs \
-	$(PYCOVERAGE) $(abs_srcdir)/$$pytest || exit ; \
+	$(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \
 	done

src/lib/dns/python/tests/Makefile.am

@@ -1,4 +1,4 @@
-PYCOVERAGE = @PYCOVERAGE@
+PYCOVERAGE_RUN = @PYCOVERAGE_RUN@
 PYTESTS = edns_python_test.py
 PYTESTS += message_python_test.py
 PYTESTS += messagerenderer_python_test.py
@@ -34,5 +34,5 @@ endif
 	env PYTHONPATH=$(abs_top_srcdir)/src/lib/dns/.libs:$(abs_top_srcdir)/src/lib/python:$(abs_top_builddir)/src/lib/python:$(abs_top_builddir)/src/lib/dns/python/.libs \
 	TESTDATA_PATH=$(abs_top_srcdir)/src/lib/dns/tests/testdata:$(abs_top_builddir)/src/lib/dns/tests/testdata \
 	$(LIBRARY_PATH_PLACEHOLDER) \
-	$(PYCOVERAGE) $(abs_srcdir)/$$pytest || exit ; \
+	$(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \
 	done

src/lib/python/isc/cc/tests/Makefile.am

@@ -1,4 +1,4 @@
-PYCOVERAGE = @PYCOVERAGE@          
+PYCOVERAGE_RUN = @PYCOVERAGE_RUN@          
 
 PYTESTS = message_test.py data_test.py session_test.py
 # NOTE: test_session.py is to be run manually, so not automated.
@@ -17,5 +17,5 @@ endif
 	echo Running test: $$pytest ; \
 	env PYTHONPATH=$(abs_top_srcdir)/src/lib/python:$(abs_top_builddir)/src/lib/python \
 	BIND10_TEST_SOCKET_FILE=$(builddir)/test_socket.sock \
-	$(PYCOVERAGE) $(abs_srcdir)/$$pytest || exit ; \
+	$(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \
 	done

src/lib/python/isc/config/tests/Makefile.am

@@ -1,4 +1,4 @@
-PYCOVERAGE=@PYCOVERAGE@
+PYCOVERAGE_RUN=@PYCOVERAGE_RUN@
 PYTESTS = ccsession_test.py cfgmgr_test.py config_data_test.py
 PYTESTS += module_spec_test.py
 EXTRA_DIST = $(PYTESTS)
@@ -16,5 +16,5 @@ endif
 	env PYTHONPATH=$(abs_top_srcdir)/src/lib/python:$(abs_top_builddir)/src/lib/python \
 	CONFIG_TESTDATA_PATH=$(abs_top_srcdir)/src/lib/config/tests/testdata \
 	CONFIG_WR_TESTDATA_PATH=$(abs_top_builddir)/src/lib/config/tests/testdata \
-	$(PYCOVERAGE) $(abs_srcdir)/$$pytest || exit ; \
+	$(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \
 	done

src/lib/python/isc/log/tests/Makefile.am

@@ -1,4 +1,4 @@
-PYCOVERAGE = @PYCOVERAGE@
+PYCOVERAGE_RUN = @PYCOVERAGE_RUN@
 PYTESTS = log_test.py
 EXTRA_DIST = $(PYTESTS)
 
@@ -12,5 +12,5 @@ endif
 	for pytest in $(PYTESTS) ; do \
 	echo Running test: $$pytest ; \
 	env PYTHONPATH=$(abs_top_srcdir)/src/lib/python:$(abs_top_builddir)/src/lib/python:$(abs_top_builddir)/src/lib/python/isc/log \
-	$(PYCOVERAGE) $(abs_srcdir)/$$pytest || exit ; \
+	$(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \
 	done

src/lib/python/isc/notify/tests/Makefile.am

@@ -1,4 +1,4 @@
-PYCOVERAGE=@PYCOVERAGE@
+PYCOVERAGE_RUN=@PYCOVERAGE_RUN@
 PYTESTS = notify_out_test.py
 EXTRA_DIST = $(PYTESTS)
 
@@ -20,5 +20,5 @@ endif
 	echo Running test: $$pytest ; \
 	env PYTHONPATH=$(abs_top_srcdir)/src/lib/python:$(abs_top_builddir)/src/lib/python:$(abs_top_builddir)/src/lib/dns/python/.libs \
 	$(LIBRARY_PATH_PLACEHOLDER) \
-	$(PYCOVERAGE) $(abs_srcdir)/$$pytest || exit ; \
+	$(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \
 	done

src/lib/python/isc/utils/tests/Makefile.am

@@ -1,4 +1,4 @@
-PYCOVERAGE = @PYCOVERAGE@
+PYCOVERAGE_RUN = @PYCOVERAGE_RUN@
 PYTESTS = process_test.py
 EXTRA_DIST = $(PYTESTS)
 
@@ -12,5 +12,5 @@ endif
 	for pytest in $(PYTESTS) ; do \
 	echo Running test: $$pytest ; \
 	env PYTHONPATH=$(abs_top_srcdir)/src/lib/python:$(abs_top_builddir)/src/lib/python:$(abs_top_builddir)/src/lib/dns/python/.libs \
-	$(PYCOVERAGE) $(abs_srcdir)/$$pytest || exit ; \
+	$(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \
 	done