Merge trac364 to trunk.

git-svn-id: svn://bind10.isc.org/svn/bind10/trunk@3382 e5f2f494-b856-4b98-b285-d166d9295462

ChangeLog

@@ -1,3 +1,10 @@
+  113.	[func]*		zhanglikun
+	Folder name 'utils'(the folder in /src/lib/python/isc/) has been 
+	renamed	to 'util'. Programs that used 'import isc.utils.process'
+	now need to use 'import isc.util.process'. The folder
+	/src/lib/python/isc/Util is removed since it isn't used by any
+	program. (Trac #364, rTBD)
+
   112.	[func]		zhang likun
 	Add one mixin class to override the naive serve_forever() provided
 	in python library socketserver. Instead of polling for shutdwon

configure.ac

@@ -476,7 +476,7 @@ AC_CONFIG_FILES([Makefile
                  src/bin/stats/tests/isc/Makefile
                  src/bin/stats/tests/isc/cc/Makefile
                  src/bin/stats/tests/isc/config/Makefile
-                 src/bin/stats/tests/isc/utils/Makefile
+                 src/bin/stats/tests/isc/util/Makefile
                  src/bin/stats/tests/testdata/Makefile
                  src/bin/usermgr/Makefile
                  src/bin/tests/Makefile
@@ -488,8 +488,8 @@ AC_CONFIG_FILES([Makefile
                  src/lib/cc/tests/Makefile
                  src/lib/python/Makefile
                  src/lib/python/isc/Makefile
-                 src/lib/python/isc/utils/Makefile
-                 src/lib/python/isc/utils/tests/Makefile
+                 src/lib/python/isc/util/Makefile
+                 src/lib/python/isc/util/tests/Makefile
                  src/lib/python/isc/datasrc/Makefile
                  src/lib/python/isc/cc/Makefile
                  src/lib/python/isc/cc/tests/Makefile

src/bin/bind10/bind10.py.in

@@ -63,11 +63,11 @@ import pwd
 import posix
 
 import isc.cc
+import isc.util.process
 import isc.net.parse
-import isc.utils.process
 
 # Assign this process some longer name
-isc.utils.process.rename(sys.argv[0])
+isc.util.process.rename(sys.argv[0])
 
 # This is the version that gets displayed to the user.
 # The VERSION string consists of the module name, the module version
@@ -629,7 +629,7 @@ def check_addr(option, opt_str, value, parser):
 
 def process_rename(option, opt_str, value, parser):
     """Function that renames the process if it is requested by a option."""
-    isc.utils.process.rename(value)
+    isc.util.process.rename(value)
 
 def main():
     global options

src/bin/bindctl/bindctl-source.py.in

@@ -24,9 +24,9 @@ from bindctl.moduleinfo import *
 from bindctl.bindcmd import *
 import pprint
 from optparse import OptionParser, OptionValueError
-import isc.utils.process
+import isc.util.process
 
-isc.utils.process.rename()
+isc.util.process.rename()
 
 __version__ = 'Bindctl'
 

src/bin/cfgmgr/b10-cfgmgr.py.in

@@ -21,11 +21,11 @@ import sys; sys.path.append ('@@PYTHONPATH@@')
 
 from isc.config.cfgmgr import ConfigManager, ConfigManagerDataReadError
 from isc.cc import SessionError
-import isc.utils.process
+import isc.util.process
 import signal
 import os
 
-isc.utils.process.rename()
+isc.util.process.rename()
 
 # If B10_FROM_SOURCE is set in the environment, we use data files
 # from a directory relative to that, otherwise we use the ones

src/bin/cmdctl/cmdctl.py.in

@@ -43,18 +43,18 @@ import random
 import time
 import signal
 from isc.config import ccsession
+import isc.util.process
 import isc.net.parse
-import isc.utils.process
 from optparse import OptionParser, OptionValueError
 from hashlib import sha1
-from isc.utils import socketserver_mixin
+from isc.util import socketserver_mixin
 
 try:
     import threading
 except ImportError:
     import dummy_threading as threading
 
-isc.utils.process.rename()
+isc.util.process.rename()
 
 __version__ = 'BIND10'
 URL_PATTERN = re.compile('/([\w]+)(?:/([\w]+))?/?')

src/bin/loadzone/b10-loadzone.py.in

@@ -18,12 +18,12 @@
 import sys; sys.path.append ('@@PYTHONPATH@@')
 import re, getopt
 import isc.datasrc
-import isc.utils.process
+import isc.util.process
 from isc.datasrc.master import MasterFile
 import time
 import os
 
-isc.utils.process.rename()
+isc.util.process.rename()
 
 #########################################################################
 # usage: print usage note and exit

src/bin/msgq/msgq.py.in

@@ -31,11 +31,11 @@ import select
 import pprint
 import random
 from optparse import OptionParser, OptionValueError
-import isc.utils.process
+import isc.util.process
 
 import isc.cc
 
-isc.utils.process.rename()
+isc.util.process.rename()
 
 # This is the version that gets displayed to the user.
 __version__ = "v20091030 (Paving the DNS Parking Lot)"

src/bin/stats/stats.py.in

@@ -35,8 +35,8 @@ if __name__ == 'stats':					#@@REMOVED@@
         pass						#@@REMOVED@@
 
 # for setproctitle
-import isc.utils.process
-isc.utils.process.rename()
+import isc.util.process
+isc.util.process.rename()
 
 # If B10_FROM_BUILD is set in the environment, we use data files
 # from a directory relative to that, otherwise we use the ones

src/bin/stats/stats_stub.py.in

@@ -27,8 +27,8 @@ from isc.cc import Session, SessionError
 from stats import get_datetime
 
 # for setproctitle
-import isc.utils.process
-isc.utils.process.rename()
+import isc.util.process
+isc.util.process.rename()
 
 # If B10_FROM_BUILD is set in the environment, we use data files
 # from a directory relative to that, otherwise we use the ones

src/bin/stats/tests/isc/Makefile.am

@@ -1,3 +1,3 @@
-SUBDIRS = cc config utils
+SUBDIRS = cc config util
 EXTRA_DIST = __init__.py
 CLEANFILES = __init__.pyc

src/bin/stats/tests/isc/util/Makefile.am

@@ -0,0 +1,2 @@
+EXTRA_DIST = __init__.py process.py
+CLEANFILES = __init__.pyc process.pyc

src/bin/stats/tests/isc/util/process.py

@@ -0,0 +1,20 @@
+# Copyright (C) 2010  Internet Systems Consortium.
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
+# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
+# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+# $Id$
+
+# A dummy function of isc.util.process.rename()
+def rename(name=None):
+    pass

src/bin/tests/process_rename_test.py.in

@@ -17,7 +17,7 @@
 import unittest
 import os
 import os.path
-import isc.utils.process
+import isc.util.process
 import re
 
 class TestRename(unittest.TestCase):
@@ -28,7 +28,7 @@ class TestRename(unittest.TestCase):
         data = ''.join(open(filename).readlines())
         prettyname = 'src' + filename[filename.rfind('../') + 2:]
         self.assertTrue(fun.search(data),
-            "Didn't find a call to isc.utils.process.rename in " + prettyname)
+            "Didn't find a call to isc.util.process.rename in " + prettyname)
 
     def test_calls(self):
         """
@@ -45,7 +45,7 @@ class TestRename(unittest.TestCase):
         # Script name regular expression
         scripts = re.compile(r'((\w|[-.0-9])+)')
         # Line with the call
-        fun = re.compile(r'^\s*isc\.utils\.process\.rename\s*\(.*\)\s*(|#.*)$',
+        fun = re.compile(r'^\s*isc\.util\.process\.rename\s*\(.*\)\s*(|#.*)$',
             re.MULTILINE)
 
         # Find all Makefile and extract names of scripts

src/bin/usermgr/b10-cmdctl-usermgr.py.in

@@ -25,9 +25,9 @@ import csv
 import getpass
 import getopt
 import sys
-import isc.utils.process
+import isc.util.process
 
-isc.utils.process.rename()
+isc.util.process.rename()
 
 VERSION_NUMBER = 'bind10'
 DEFAULT_FILE = 'cmdctl-accounts.csv'

src/bin/xfrin/xfrin.py.in

@@ -30,8 +30,8 @@ import random
 from optparse import OptionParser, OptionValueError
 from isc.config.ccsession import *
 from isc.notify import notify_out
+import isc.util.process
 import isc.net.parse
-import isc.utils.process
 try:
     from pydnspp import *
 except ImportError as e:
@@ -39,7 +39,7 @@ except ImportError as e:
     # must keep running, so we warn about it and move forward.
     sys.stderr.write('[b10-xfrin] failed to import DNS module: %s\n' % str(e))
 
-isc.utils.process.rename()
+isc.util.process.rename()
 
 # If B10_FROM_BUILD is set in the environment, we use data files
 # from a directory relative to that, otherwise we use the ones

src/bin/xfrout/xfrout.py.in

@@ -30,12 +30,12 @@ from isc.config.ccsession import *
 from isc.log.log import *
 from isc.cc import SessionError, SessionTimeout
 from isc.notify import notify_out
-import isc.utils.process
+import isc.util.process
 import socket
 import select
 import errno
 from optparse import OptionParser, OptionValueError
-from isc.utils import socketserver_mixin
+from isc.util import socketserver_mixin
 
 try:
     from libxfr_python import *
@@ -45,7 +45,7 @@ except ImportError as e:
     # must keep running, so we warn about it and move forward.
     sys.stderr.write('[b10-xfrout] failed to import DNS or XFR module: %s\n' % str(e))
 
-isc.utils.process.rename()
+isc.util.process.rename()
 
 if "B10_FROM_BUILD" in os.environ:
     SPECFILE_PATH = os.environ["B10_FROM_BUILD"] + "/src/bin/xfrout"

src/bin/zonemgr/zonemgr.py.in

@@ -37,9 +37,9 @@ import errno
 from isc.datasrc import sqlite3_ds
 from optparse import OptionParser, OptionValueError
 from isc.config.ccsession import *
-import isc.utils.process
+import isc.util.process
 
-isc.utils.process.rename()
+isc.util.process.rename()
 
 # If B10_FROM_BUILD is set in the environment, we use data files
 # from a directory relative to that, otherwise we use the ones

src/lib/python/isc/Makefile.am

@@ -1,4 +1,4 @@
-SUBDIRS = datasrc cc config log net notify utils # Util
+SUBDIRS = datasrc cc config log net notify util 
 
 python_PYTHON = __init__.py
 

src/lib/python/isc/Util/Makefile.am

@@ -1,3 +0,0 @@
-python_PYTHON = __init__.py hexdump.py
-
-pythondir = $(pyexecdir)/isc/Util

src/lib/python/isc/Util/__init__.py

@@ -1 +0,0 @@
-from isc.Util.hexdump import *

src/lib/python/isc/Util/hexdump.py

@@ -1,13 +0,0 @@
-import sys
-
-_FILTER=''.join([(len(repr(chr(x)))==3) and chr(x) or '.' for x in range(256)])
-
-def hexdump(src, length=16):
-    result=[]
-    for i in range(0, len(src), length):
-        s = src[i:i+length]
-        hexa = ' '.join(["%02X" % x for x in s])
-        printable = s.decode().translate(_FILTER)
-        item = "%08X   %-*s   %s\n" % (i, length * 3, hexa, printable)
-        result.append(item[0:34] + ' ' + item[34:])
-    print(''.join(result))

src/lib/python/isc/util/Makefile.am

@@ -0,0 +1,5 @@
+SUBDIRS = . tests
+
+python_PYTHON = __init__.py process.py socketserver_mixin.py
+
+pythondir = $(pyexecdir)/isc/util

src/lib/python/isc/util/__init__.py

@@ -0,0 +1 @@
+from isc.util.socketserver_mixin import *

src/lib/python/isc/util/process.py

@@ -0,0 +1,37 @@
+# Copyright (C) 2010  CZ NIC
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
+# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
+# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+"""
+Module to manipulate the python processes.
+
+It contains only function to rename the process, which is currently
+wrapper around setproctitle library. Does not fail if the setproctitle
+module is missing, but does nothing in that case.
+"""
+try:
+    from setproctitle import setproctitle
+except ImportError:
+    def setproctitle(_): pass
+import sys
+import os.path
+
+"""
+Rename the current process to given name (so it can be found in ps).
+If name is None, use zero'th command line argument.
+"""
+def rename(name=None):
+    if name is None:
+        name = os.path.basename(sys.argv[0])
+    setproctitle(name)

src/lib/python/isc/util/socketserver_mixin.py

@@ -0,0 +1,92 @@
+# Copyright (C) 2010  Internet Systems Consortium.
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
+# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
+# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+import threading
+import socket
+import select
+
+SOCK_DATA = b'somedata'
+class NoPollMixIn:
+    '''This is a mix-in class to override the function serve_forever()
+    and shutdown() in class socketserver.BaseServer.
+
+    As commented in the module source code,  serve_forever() in
+    socketserver.BaseServer uses polling for a shutdown request, which
+    "reduces the responsiveness to a shutdown request and wastes cpu at
+    all other times."
+
+    This class fixes this problem by introducing internal message
+    passing via a separate socket. Note, however, that according to
+    the module documentation serve_forever() and shutdown() are not
+    categorized as functions that can be overridden via mix-ins.  So
+    this mix-in class may not be compatible with future versions of
+    socketserver.  It should be considered a short term workaround
+    until the original implementation is fixed.
+
+    The NoPollMixIn class should be used together with
+    socketserver.BaseServer or some derived classes of it, and it must
+    be placed before the corresponding socketserver class.  In
+    addition, the constructor of this mix-in must be called
+    explicitely in the derived class.  For example, a basic TCP server
+    without the problem of polling is created as follows:
+
+       class MyServer(NoPollMixIn, socketserver.TCPServer):
+           def __init__(...):
+               ...
+               NoPollMixIn.__init__(self)
+               ...
+
+    To shutdown the server correctly, the serve_forever() method must
+    be run in a separate thread, and shutdown() must be called from
+    some other thread.
+    '''
+    def __init__(self):
+        self.__read_sock, self.__write_sock = socket.socketpair()
+        self._is_shut_down = threading.Event()
+
+    def serve_forever(self, poll_interval=None):
+        ''' Overrides the serve_forever([poll_interval]) in class
+        socketserver.BaseServer.
+
+        It uses a socketpair to wake up the select when shutdown() is
+        called in anther thread.  Note, parameter 'poll_interval' is
+        just used for interface compatibility; it's never used in this
+        function.
+        '''        
+        while True:
+            # block until the self.socket or self.__read_sock is readable
+            try:
+                r, w, e = select.select([self, self.__read_sock], [], [])
+            except select.error as err:
+                if err.args[0] == EINTR:
+                    continue
+                else:
+                    break
+            
+            if self.__read_sock in r:
+                break
+            else:
+                self._handle_request_noblock()
+
+        self._is_shut_down.set()
+
+    def shutdown(self):
+        '''Stops the serve_forever loop.
+
+        Blocks until the loop has finished, the function should be called
+        in another thread when serve_forever is running, or it will block.
+        '''
+        self.__write_sock.send(SOCK_DATA) # make self.__read_sock readable.
+        self._is_shut_down.wait()  # wait until the serve thread terminate

src/lib/python/isc/util/tests/Makefile.am

@@ -0,0 +1,12 @@
+PYTESTS = process_test.py socketserver_mixin_test.py
+EXTRA_DIST = $(PYTESTS)
+
+# later will have configure option to choose this, like: coverage run --branch
+PYCOVERAGE = $(PYTHON)
+# test using command-line arguments, so use check-local target instead of TESTS
+check-local:
+	for pytest in $(PYTESTS) ; do \
+	echo Running test: $$pytest ; \
+	env PYTHONPATH=$(abs_top_srcdir)/src/lib/python:$(abs_top_builddir)/src/lib/python:$(abs_top_builddir)/src/lib/dns/python/.libs \
+	$(PYCOVERAGE) $(abs_srcdir)/$$pytest || exit ; \
+	done

src/lib/python/isc/util/tests/process_test.py

@@ -0,0 +1,39 @@
+# Copyright (C) 2010  CZ NIC
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
+# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
+# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+"""Tests for isc.util.process."""
+import unittest
+import isc.util.process
+run_tests = True
+try:
+    import setproctitle
+except ImportError:
+    run_tests = False
+
+class TestRename(unittest.TestCase):
+    """Testcase for isc.process.rename."""
+    def __get_self_name(self):
+        return setproctitle.getproctitle()
+
+    @unittest.skipIf(not run_tests, "Setproctitle not installed, not testing")
+    def test_rename(self):
+        """Test if the renaming function works."""
+        isc.util.process.rename("rename-test")
+        self.assertEqual("rename-test", self.__get_self_name())
+        isc.util.process.rename()
+        self.assertEqual("process_test.py", self.__get_self_name())
+
+if __name__ == "__main__":
+    unittest.main()

src/lib/python/isc/util/tests/socketserver_mixin_test.py

@@ -0,0 +1,63 @@
+# Copyright (C) 2010  Internet Systems Consortium.
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
+# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
+# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+import unittest
+from isc.util.socketserver_mixin import NoPollMixIn
+import socketserver
+import threading
+import socket
+import time
+
+class MyHandler(socketserver.BaseRequestHandler):
+    def handle(self):
+        data = self.request.recv(20)
+        self.request.send(data)
+
+class MyServer(NoPollMixIn, 
+               socketserver.ThreadingMixIn,
+               socketserver.TCPServer):
+
+    def __init__(self, server_addr, handler_class):
+        NoPollMixIn.__init__(self)
+        socketserver.TCPServer.__init__(self, server_addr, handler_class)
+
+def send_and_get_reply(ip, port, msg):
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.connect((ip, port))
+    sock.send(msg)
+    response = sock.recv(20)
+    sock.close()
+    return response
+
+class TestNoPollMixIn(unittest.TestCase):
+    def test_serve_forever(self):
+        # use port 0 to select an arbitrary unused port.
+        server = MyServer(('127.0.0.1', 0), MyHandler)
+        ip, port = server.server_address
+        server_thread = threading.Thread(target=server.serve_forever)
+        server_thread.start()
+
+        msg = b'senddata'
+        self.assertEqual(msg, send_and_get_reply(ip, port, msg))
+        self.assertTrue(server_thread.is_alive())
+
+        self.assertFalse(server._is_shut_down.is_set())
+        server.shutdown() # Now shutdown the server
+        self.assertTrue(server._is_shut_down.is_set())
+
+if __name__== "__main__":
+    unittest.main()
+
+