== Release 0.1.0 ==

git-svn-id: http://proj.badc.rl.ac.uk/svn/ndg-security/trunk/ndg_httpsclient@7989 051b1e3e-aa0c-0410-b6c2-bfbade6052be

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2011, Science & Technology Facilities Council (STFC)
+Copyright (c) 2012, Science & Technology Facilities Council (STFC)
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without 

README

@@ -1,12 +1,13 @@
-================
+===============
-urllib2pyopenssl
+ndg_httpsclient
-================
+===============
 
 Description
 ===========
-This is a library to enable urllib2 to be used with SSL sockets from pyOpenSSL instead of the built in ssl library. A script is provided to exercise it:
+This is a library to enable urllib2 to be used with SSL sockets from pyOpenSSL 
+instead of the built in ssl library. A script is provided to exercise it:
 
-urllib2pyopenssl_get::
+ndg_httpclient::
  - Utility to fetch data using HTTP or HTTPS GET from a specified URL.
 
 Prerequisites
@@ -18,10 +19,10 @@ pyOpenSSL
 Installation
 ============
 Installation can be performed using easy_install, e.g.::
-  easy_install urllib2pyopenssl-0.1.0-py2.6.egg
+  easy_install ndg_httpsclient-0.1.0-py2.6.egg
 
-Running urllib2pyopenssl_get
+Running ndg_httpclient
-============================
+======================
 Parameter::
     url                   The URL of the resource to be fetched
 
@@ -33,7 +34,8 @@ Options::
                           Private key file - defaults to the certificate file
     -t DIR, --ca-certificate-dir=DIR
                           Trusted CA certificate file directory.
-    -d, --debug           Print debug information - this may be useful in solving problems with HTTP
+    -d, --debug           Print debug information - this may be useful in 
-                          or HTTPS access to a server.
+                          solving problems with HTTP or HTTPS access to a 
+                          server.
     -f FILE, --fetch=FILE Output file
-    -v, --verify-peer     Verify peer certificate.
+    -n, --no-verify-peer  Skip verification of peer certificate..

documentation/Makefile

@@ -0,0 +1,32 @@
+#
+# Makefile to generate epydoc documentation for the NDG HTTPS Client Package
+#
+# @author P J Kershaw 17/01/12
+#
+
+# @copyright: (C) 2012 STFC
+#
+# @license: BSD - see LICENSE file for details
+#
+# $Id$
+
+# Generate HTML from embedded epydoc text in source code.
+EPYDOC=epydoc
+EPYDOC_INDIR=../ndg
+EPYDOC_OUTDIR=.
+EPYDOC_NAME='NDG HTTPS Client'
+EPYDOC_LOGFILE=epydoc.log
+EPYDOC_OPTS=--no-frames --include-log --graph=all -v --debug
+ZIP=zip
+ZIP_OUTFILE=./documentation.zip
+ZIP_INFILES=./*.*
+
+epydoc:
+	${EPYDOC} ${EPYDOC_INDIR} -o ${EPYDOC_OUTDIR} --name ${EPYDOC_NAME} \
+	${EPYDOC_OPTS} > ${EPYDOC_LOGFILE}
+    
+zip:
+	${ZIP} ${ZIP_OUTFILE} ${ZIP_INFILES}    
+
+clean:
+	rm -f *.txt *.html *.gif *.css *.js *.png *.log

ndg/httpsclient/httplib_proxy.py

@@ -1,67 +0,0 @@
-'''
-Created on Jan 11, 2012
-
-@author: philipkershaw
-'''
-import socket
-from httplib import HTTPConnection as _HTTPConnection
-from httplib import HTTPException
-
-# maximal line length when calling readline().
-_MAXLINE = 65536
-
-class LineTooLong(HTTPException):
-    def __init__(self, line_type):
-        HTTPException.__init__(self, "got more than %d bytes when reading %s"
-                                     % (_MAXLINE, line_type))
-        
-
-class HTTPConnection(_HTTPConnection):
-    NDG_HTTPSCLIENT = True
-    
-    def __init__(self, *arg, **kwarg):
-        self._tunnel_host = None
-        self._tunnel_port = None
-        self._tunnel_headers = {}
-
-        _HTTPConnection.__init__(self, *arg, **kwarg)
-        
-    def set_tunnel(self, host, port=None, headers=None):
-        """ Sets up the host and the port for the HTTP CONNECT Tunnelling.
-
-        The headers argument should be a mapping of extra HTTP headers
-        to send with the CONNECT request.
-        """
-        self._tunnel_host = host
-        self._tunnel_port = port
-        if headers:
-            self._tunnel_headers = headers
-        else:
-            self._tunnel_headers.clear()
-
-    def _tunnel(self):
-        self._set_hostport(self._tunnel_host, self._tunnel_port)
-        self.send("CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port))
-        for header, value in self._tunnel_headers.iteritems():
-            self.send("%s: %s\r\n" % (header, value))
-        self.send("\r\n")
-        response = self.response_class(self.sock, strict = self.strict,
-                                       method = self._method)
-        (version, code, message) = response._read_status()
-
-        if code != 200:
-            self.close()
-            raise socket.error("Tunnel connection failed: %d %s" % (code,
-                                                            message.strip()))
-        while True:
-            line = response.fp.readline(_MAXLINE + 1)
-            if len(line) > _MAXLINE:
-                raise LineTooLong("header line")
-            if line == '\r\n': break
-            
-    def connect(self):
-        """Connect to the host and port specified in __init__."""
-        _HTTPConnection.connect(self)
-
-        if self._tunnel_host:
-            self._tunnel()

ndg/httpsclient/https.py

@@ -12,14 +12,9 @@ __contact__ = "Philip.Kershaw@stfc.ac.uk"
 __revision__ = '$Id$'
 import logging
 import socket
-import sys
 from httplib import HTTPS_PORT
-if sys.version_info < (2, 6, 2):
+from httplib import HTTPConnection
-    from ndg.httpsclient.httplib_proxy import HTTPConnection
+from urllib2 import AbstractHTTPHandler
-    from ndg.httpsclient.urllib2_proxy import AbstractHTTPHandler
-else:
-    from httplib import HTTPConnection
-    from urllib2 import AbstractHTTPHandler
 
 
 from OpenSSL import SSL
@@ -107,8 +102,8 @@ class HTTPSContextHandler(AbstractHTTPHandler):
 
     def https_open(self, req):
         """Opens HTTPS request
-        @param req - HTTP request
+        @param req: HTTP request
-        @return HTTP Response object
+        @return: HTTP Response object
         """
         # Make a custom class extending HTTPSConnection, with the SSL context
         # set as a class variable so that it is available to the connect method.

ndg/httpsclient/ssl_context_util.py

@@ -1,9 +1,20 @@
+"""ndg_httpsclient SSL Context utilities module containing convenience routines
+for setting SSL context configuration.
+
+"""
+__author__ = "P J Kershaw (STFC)"
+__date__ = "09/12/11"
+__copyright__ = "(C) 2012 Science and Technology Facilities Council"
+__license__ = "BSD - see LICENSE file in top-level directory"
+__contact__ = "Philip.Kershaw@stfc.ac.uk"
+__revision__ = '$Id$'
 import urlparse
 
 from OpenSSL import SSL
 
 from ndg.httpsclient.ssl_peer_verification import ServerSSLCertVerification
 
+
 class SSlContextConfig(object):
     """
     Holds configuration options for creating a SSL context. This is used as a
@@ -17,11 +28,13 @@ class SSlContextConfig(object):
         self.ca_dir = ca_dir
         self.verify_peer = verify_peer
 
+
 def make_ssl_context_from_config(ssl_config=False, url=None):
     return make_ssl_context(ssl_config.key_file, ssl_config.cert_file,
                             ssl_config.pem_file, ssl_config.ca_dir,
                             ssl_config.verify_peer, url)
 
+
 def make_ssl_context(key_file=None, cert_file=None, pem_file=None, ca_dir=None,
                      verify_peer=False, url=None):
     """
@@ -45,6 +58,7 @@ def make_ssl_context(key_file=None, cert_file=None, pem_file=None, ca_dir=None,
         Performs no checks and returns the status passed in.
         """
         return preverify_ok
+    
     verify_callback = _callback
 
     if verify_peer:
@@ -57,7 +71,11 @@ def make_ssl_context(key_file=None, cert_file=None, pem_file=None, ca_dir=None,
         ssl_context.set_verify(SSL.VERIFY_NONE, verify_callback)
     return ssl_context
 
-def set_peer_verification_for_url_hostname(ssl_context, url, if_verify_enabled=False):
+
+def set_peer_verification_for_url_hostname(ssl_context, url, 
+                                           if_verify_enabled=False):
+    '''Convenience routine to set peer verification callback based on
+    ServerSSLCertVerification class'''
     if not if_verify_enabled or (ssl_context.get_verify_mode() & SSL.VERIFY_PEER):
         urlObj = urlparse.urlparse(url)
         hostname = urlObj.hostname

ndg/httpsclient/ssl_peer_verification.py

@@ -1,16 +1,15 @@
+"""ndg_httpsclient - module containing SSL peer verification class.
 """
-"""
+__author__ = "P J Kershaw (STFC)"
-__author__ = "P J Kershaw"
+__date__ = "09/12/11"
-__date__ = "02/06/05"
+__copyright__ = "(C) 2012 Science and Technology Facilities Council"
-__copyright__ = "(C) 2010 Science and Technology Facilities Council"
+__license__ = "BSD - see LICENSE file in top-level directory"
-__license__ = """BSD - See LICENSE file in top-level directory"""
 __contact__ = "Philip.Kershaw@stfc.ac.uk"
-__revision__ = '$Id: client.py 7928 2011-08-12 13:16:26Z pjkersha $'
+__revision__ = '$Id$'
-
+import re
 import logging
 log = logging.getLogger(__name__)
 
-import re
 
 class ServerSSLCertVerification(object):
     """Check server identity.  If hostname doesn't match, allow match of

ndg/httpsclient/ssl_socket.py

@@ -7,7 +7,7 @@ __date__ = "21/12/10"
 __copyright__ = "(C) 2012 Science and Technology Facilities Council"
 __license__ = "BSD - see LICENSE file in top-level directory"
 __contact__ = "Philip.Kershaw@stfc.ac.uk"
-__revision__ = '$Id: pyopenssl.py 7929 2011-08-16 16:39:13Z pjkersha $'
+__revision__ = '$Id$'
 
 from datetime import datetime
 import logging

ndg/httpsclient/test/README

@@ -0,0 +1,12 @@
+NDG HTTPS Client Unit tests directory
+=====================================
+The unit tests expect to connect to a simple HTTPS server listening on port 
+4443.  An OpenSSL script is provided for this purpose in scripts/.  To run,
+
+$ ./scripts/openssl_https_server.sh
+
+Troubleshooting
+---------------
+Run it from *this* directory.  Also ensure it is has execute bits set. e.g.
+
+$ chmod 755 ./scripts/openssl_https_server.sh

ndg/httpsclient/test/__init__.py

@@ -1,4 +1,4 @@
-"""unit tests package for ndg_httpsclient
+"""Unit tests package for ndg_httpsclient
 
 PyOpenSSL utility to make a httplib-like interface suitable for use with 
 urllib2
@@ -21,7 +21,7 @@ class Constants(object):
     TEST_URI = 'https://%s:%d' % (HOSTNAME, PORT)
     TEST_URI2 = 'https://%s:%d' % (HOSTNAME, PORT2)
 
-    UNITTEST_DIR = os.path.dirname(os.path.abspath(__path__))
+    UNITTEST_DIR = os.path.dirname(os.path.abspath(__file__))
     SSL_CERT_FILENAME = 'localhost.crt'
     SSL_CERT_FILEPATH = os.path.join(UNITTEST_DIR, 'pki', SSL_CERT_FILENAME)
     SSL_PRIKEY_FILENAME = 'localhost.key'

ndg/httpsclient/test/test_https.py

@@ -1,8 +1,14 @@
-'''
+"""unit tests module for ndg.httpsclient.https.HTTPSconnection class
-Created on Jan 6, 2012
+
-
+PyOpenSSL utility to make a httplib-like interface suitable for use with 
-@author: philipkershaw
+urllib2
-'''
+"""
+__author__ = "P J Kershaw (STFC)"
+__date__ = "06/01/12"
+__copyright__ = "(C) 2012 Science and Technology Facilities Council"
+__license__ = "BSD - see LICENSE file in top-level directory"
+__contact__ = "Philip.Kershaw@stfc.ac.uk"
+__revision__ = '$Id$'
 import logging
 logging.basicConfig(level=logging.DEBUG)
 import unittest

ndg/httpsclient/test/test_urllib2.py

@@ -1,8 +1,14 @@
-'''
+"""unit tests module for ndg.httpsclient.urllib2_build_opener module
-Created on Jan 5, 2012
 
-@author: philipkershaw
+PyOpenSSL utility to make a httplib-like interface suitable for use with 
-'''
+urllib2
+"""
+__author__ = "P J Kershaw (STFC)"
+__date__ = "06/01/12"
+__copyright__ = "(C) 2012 Science and Technology Facilities Council"
+__license__ = "BSD - see LICENSE file in top-level directory"
+__contact__ = "Philip.Kershaw@stfc.ac.uk"
+__revision__ = '$Id$'
 from urllib2 import URLError
 import unittest
 

ndg/httpsclient/test/test_utils.py

@@ -1,17 +1,51 @@
-'''
+"""unit tests module for ndg.httpsclient.utils module
-Created on Jan 6, 2012
 
-@author: philipkershaw
+PyOpenSSL utility to make a httplib-like interface suitable for use with 
-'''
+urllib2
+"""
+__author__ = "P J Kershaw (STFC)"
+__date__ = "06/01/12"
+__copyright__ = "(C) 2012 Science and Technology Facilities Council"
+__license__ = "BSD - see LICENSE file in top-level directory"
+__contact__ = "Philip.Kershaw@stfc.ac.uk"
+__revision__ = '$Id$'
 import unittest
+import os
 
+from OpenSSL import SSL
 
-class TestGetModule(unittest.TestCase):
+from ndg.httpsclient.test import Constants
+from ndg.httpsclient.utils import (Configuration, fetch_from_url, open_url,
+                                   _should_use_proxy)
 
 
-    def test01(self):
+class TestUtilsModule(unittest.TestCase):
-        pass
+    '''Test ndg.httpsclient.utils module'''
 
+    def test01_configuration(self):
+        config = Configuration(SSL.Context(SSL.SSLv3_METHOD), True)
+        self.assert_(config.ssl_context)
+        self.assertEquals(config.debug, True)
 
+    def test02_fetch_from_url(self):
+        config = Configuration(SSL.Context(SSL.SSLv3_METHOD), True)
+        res = fetch_from_url(Constants.TEST_URI, config)
+        self.assert_(res)
+        
+    def test03_open_url(self):
+        config = Configuration(SSL.Context(SSL.SSLv3_METHOD), True)
+        res = open_url(Constants.TEST_URI, config)
+        self.assertEqual(res[0], 200, 
+                         'open_url for %r failed' % Constants.TEST_URI)
+        
+    def test04__should_use_proxy(self):
+        self.assertTrue(_should_use_proxy(Constants.TEST_URI), 
+                        'Expecting use proxy = True')
+        
+        os.environ['no_proxy'] = 'localhost,localhost.localdomain'
+        self.assertFalse(_should_use_proxy(Constants.TEST_URI), 
+                         'Expecting use proxy = False')
+        del os.environ['no_proxy']
+    
 if __name__ == "__main__":
     unittest.main()

ndg/httpsclient/urllib2_build_opener.py

@@ -1,27 +1,22 @@
-"""PyOpenSSL utilities including HTTPSSocket class which wraps PyOpenSSL
+"""urllib2 style build opener integrates with HTTPSConnection class from this
-SSL connection into a httplib-like interface suitable for use with urllib2
+package.
 """
 __author__ = "P J Kershaw"
 __date__ = "21/12/10"
 __copyright__ = "(C) 2011 Science and Technology Facilities Council"
 __license__ = "BSD - see LICENSE file in top-level directory"
 __contact__ = "Philip.Kershaw@stfc.ac.uk"
-__revision__ = '$Id: pyopenssl.py 7929 2011-08-16 16:39:13Z pjkersha $'
+__revision__ = '$Id$'
 import logging
 from urllib2 import (ProxyHandler, UnknownHandler, HTTPDefaultErrorHandler, 
-                     FTPHandler, FileHandler, HTTPErrorProcessor)
+                     FTPHandler, FileHandler, HTTPErrorProcessor, HTTPHandler, 
-
+                     OpenerDirector, HTTPRedirectHandler)
-import sys
-if sys.version_info < (2, 6, 2):
-    from ndg.httpsclient.urllib2_proxy import (HTTPHandler, OpenerDirector, 
-                                               HTTPRedirectHandler)
-else:
-    from urllib2 import HTTPHandler, OpenerDirector, HTTPRedirectHandler
 
 from ndg.httpsclient.https import HTTPSContextHandler
 
 log = logging.getLogger(__name__)
 
+
 # Copied from urllib2 with modifications for ssl
 def build_opener(ssl_context=None, *handlers):
     """Create an opener object from a list of handlers.

ndg/httpsclient/urllib2_proxy.py

@@ -1,262 +0,0 @@
-'''
-Created on 12 Jan 2012
-
-@author: rwilkinson
-'''
-import base64
-import socket
-import urlparse
-from urllib import unquote, addinfourl
-from urllib2 import _parse_proxy, URLError, HTTPError
-from urllib2 import (AbstractHTTPHandler as _AbstractHTTPHandler,
-                     BaseHandler as _BaseHandler,
-                     HTTPRedirectHandler as _HTTPRedirectHandler,
-                     Request as _Request,
-                     OpenerDirector as _OpenerDirector)
-
-from ndg.httpsclient.httplib_proxy import HTTPConnection
-
-
-class Request(_Request):
-
-    def __init__(self, *args, **kw):
-        _Request.__init__(self, *args, **kw)
-        self._tunnel_host = None
-
-    def set_proxy(self, host, type):
-        if self.type == 'https' and not self._tunnel_host:
-            self._tunnel_host = self.host
-        else:
-            self.type = type
-            self.__r_host = self.__original
-        self.host = host
-
-
-class BaseHandler(_BaseHandler):
-    def proxy_open(self, req, proxy, type):
-        if req.get_type() == 'https':
-            orig_type = req.get_type()
-            proxy_type, user, password, hostport = _parse_proxy(proxy)
-            if proxy_type is None:
-                proxy_type = orig_type
-            if user and password:
-                user_pass = '%s:%s' % (unquote(user), unquote(password))
-                creds = base64.b64encode(user_pass).strip()
-                req.add_header('Proxy-authorization', 'Basic ' + creds)
-            hostport = unquote(hostport)
-            req.set_proxy(hostport, proxy_type)
-            # let other handlers take care of it
-            return None
-        else:
-            return _BaseHandler.proxy_open(self, req, proxy, type)
-
-class AbstractHTTPHandler(_AbstractHTTPHandler):
-    def do_open(self, http_class, req):
-        """Return an addinfourl object for the request, using http_class.
-
-        http_class must implement the HTTPConnection API from httplib.
-        The addinfourl return value is a file-like object.  It also
-        has methods and attributes including:
-            - info(): return a mimetools.Message object for the headers
-            - geturl(): return the original request URL
-            - code: HTTP status code
-        """
-        host = req.get_host()
-        if not host:
-            raise URLError('no host given')
-
-        h = http_class(host, timeout=req.timeout) # will parse host:port
-        h.set_debuglevel(self._debuglevel)
-
-        headers = dict(req.headers)
-        headers.update(req.unredirected_hdrs)
-        # We want to make an HTTP/1.1 request, but the addinfourl
-        # class isn't prepared to deal with a persistent connection.
-        # It will try to read all remaining data from the socket,
-        # which will block while the server waits for the next request.
-        # So make sure the connection gets closed after the (only)
-        # request.
-        headers["Connection"] = "close"
-        headers = dict(
-            (name.title(), val) for name, val in headers.items())
-
-        if not hasattr(req, '_tunnel_host'):
-            pass
-        
-        if req._tunnel_host:
-            h.set_tunnel(req._tunnel_host)
-        try:
-            h.request(req.get_method(), req.get_selector(), req.data, headers)
-            r = h.getresponse()
-        except socket.error, err: # XXX what error?
-            raise URLError(err)
-
-        # Pick apart the HTTPResponse object to get the addinfourl
-        # object initialized properly.
-
-        # Wrap the HTTPResponse object in socket's file object adapter
-        # for Windows.  That adapter calls recv(), so delegate recv()
-        # to read().  This weird wrapping allows the returned object to
-        # have readline() and readlines() methods.
-
-        # XXX It might be better to extract the read buffering code
-        # out of socket._fileobject() and into a base class.
-
-        r.recv = r.read
-        fp = socket._fileobject(r, close=True)
-
-        resp = addinfourl(fp, r.msg, req.get_full_url())
-        resp.code = r.status
-        resp.msg = r.reason
-        return resp
-
-
-class HTTPHandler(AbstractHTTPHandler):
-
-    def http_open(self, req):
-        return self.do_open(HTTPConnection, req)
-
-    http_request = AbstractHTTPHandler.do_request_
-
-#if hasattr(httplib, 'HTTPS'):
-#    class HTTPSHandler(AbstractHTTPHandler):
-#
-#        def https_open(self, req):
-#            return self.do_open(httplib.HTTPSConnection, req)
-#
-#        https_request = AbstractHTTPHandler.do_request_
-
-
-class HTTPRedirectHandler(BaseHandler):
-    # maximum number of redirections to any single URL
-    # this is needed because of the state that cookies introduce
-    max_repeats = 4
-    # maximum total number of redirections (regardless of URL) before
-    # assuming we're in a loop
-    max_redirections = 10
-
-    def redirect_request(self, req, fp, code, msg, headers, newurl):
-        """Return a Request or None in response to a redirect.
-
-        This is called by the http_error_30x methods when a
-        redirection response is received.  If a redirection should
-        take place, return a new Request to allow http_error_30x to
-        perform the redirect.  Otherwise, raise HTTPError if no-one
-        else should try to handle this url.  Return None if you can't
-        but another Handler might.
-        """
-        m = req.get_method()
-        if (code in (301, 302, 303, 307) and m in ("GET", "HEAD")
-            or code in (301, 302, 303) and m == "POST"):
-            # Strictly (according to RFC 2616), 301 or 302 in response
-            # to a POST MUST NOT cause a redirection without confirmation
-            # from the user (of urllib2, in this case).  In practice,
-            # essentially all clients do redirect in this case, so we
-            # do the same.
-            # be conciliant with URIs containing a space
-            newurl = newurl.replace(' ', '%20')
-            newheaders = dict((k,v) for k,v in req.headers.items()
-                              if k.lower() not in ("content-length", "content-type")
-                             )
-            return Request(newurl,
-                           headers=newheaders,
-                           origin_req_host=req.get_origin_req_host(),
-                           unverifiable=True)
-        else:
-            raise HTTPError(req.get_full_url(), code, msg, headers, fp)
-
-    # Implementation note: To avoid the server sending us into an
-    # infinite loop, the request object needs to track what URLs we
-    # have already seen.  Do this by adding a handler-specific
-    # attribute to the Request object.
-    def http_error_302(self, req, fp, code, msg, headers):
-        # Some servers (incorrectly) return multiple Location headers
-        # (so probably same goes for URI).  Use first header.
-        if 'location' in headers:
-            newurl = headers.getheaders('location')[0]
-        elif 'uri' in headers:
-            newurl = headers.getheaders('uri')[0]
-        else:
-            return
-
-        # fix a possible malformed URL
-        urlparts = urlparse.urlparse(newurl)
-        if not urlparts.path:
-            urlparts = list(urlparts)
-            urlparts[2] = "/"
-        newurl = urlparse.urlunparse(urlparts)
-
-        newurl = urlparse.urljoin(req.get_full_url(), newurl)
-
-        # For security reasons we do not allow redirects to protocols
-        # other than HTTP, HTTPS or FTP.
-        newurl_lower = newurl.lower()
-        if not (newurl_lower.startswith('http://') or
-                newurl_lower.startswith('https://') or
-                newurl_lower.startswith('ftp://')):
-            raise HTTPError(newurl, code,
-                            msg + " - Redirection to url '%s' is not allowed" %
-                            newurl,
-                            headers, fp)
-
-        # XXX Probably want to forget about the state of the current
-        # request, although that might interact poorly with other
-        # handlers that also use handler-specific request attributes
-        new = self.redirect_request(req, fp, code, msg, headers, newurl)
-        if new is None:
-            return
-
-        # loop detection
-        # .redirect_dict has a key url if url was previously visited.
-        if hasattr(req, 'redirect_dict'):
-            visited = new.redirect_dict = req.redirect_dict
-            if (visited.get(newurl, 0) >= self.max_repeats or
-                len(visited) >= self.max_redirections):
-                raise HTTPError(req.get_full_url(), code,
-                                self.inf_msg + msg, headers, fp)
-        else:
-            visited = new.redirect_dict = req.redirect_dict = {}
-        visited[newurl] = visited.get(newurl, 0) + 1
-
-        # Don't close the fp until we are sure that we won't use it
-        # with HTTPError.
-        fp.read()
-        fp.close()
-
-        return self.parent.open(new, timeout=req.timeout)
-
-    http_error_301 = http_error_303 = http_error_307 = http_error_302
-
-    inf_msg = "The HTTP server returned a redirect error that would " \
-              "lead to an infinite loop.\n" \
-              "The last 30x error message was:\n"
-              
-
-class OpenerDirector(_OpenerDirector):
-    def open(self, fullurl, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
-        # accept a URL or a Request object
-        if isinstance(fullurl, basestring):
-            req = Request(fullurl, data)
-        else:
-            req = fullurl
-            if data is not None:
-                req.add_data(data)
-
-        req.timeout = timeout
-        protocol = req.get_type()
-
-        # pre-process request
-        meth_name = protocol+"_request"
-        for processor in self.process_request.get(protocol, []):
-            meth = getattr(processor, meth_name)
-            req = meth(req)
-
-        response = self._open(req, data)
-        
-        # post-process response
-        meth_name = protocol+"_response"
-        for processor in self.process_response.get(protocol, []):
-            meth = getattr(processor, meth_name)
-            response = meth(req, response)
-
-        return response

ndg/httpsclient/utils.py

@@ -3,12 +3,8 @@ import httplib
 import logging
 from optparse import OptionParser
 import os
-import sys
 import urllib2
-if sys.version_info < (2, 6, 2):
+from urllib2 import HTTPHandler
-    from ndg.httpsclient.urllib2_proxy import HTTPHandler
-else:
-    from urllib2 import HTTPHandler
     
 import urlparse
 
@@ -23,8 +19,9 @@ class URLFetchError(Exception):
 
 def fetch_from_url(url, config):
     """Returns data retrieved from a URL.
-    @param url - URL to attempt to open
+    @param url: URL to attempt to open
-    @param config - configuration
+    @param config: SSL context configuration
+    @type config: Configuration
     @return data retrieved from URL or None
     """
     return_code, return_message, response = open_url(url, config)
@@ -37,14 +34,14 @@ def fetch_from_url(url, config):
 
 def fetch_from_url_to_file(url, config, output_file):
     """Writes data retrieved from a URL to a file.
-    @param url - URL to attempt to open
+    @param url: URL to attempt to open
-    @param config - configuration
+    @param config: SSL context configuration
-    @param output_file - output file
+    @type config: Configuration
-    @return tuple (
+    @param output_file: output file
+    @return: tuple (
         returned HTTP status code or 0 if an error occurred
         returned message
-        boolean indicating whether access was successful
+        boolean indicating whether access was successful)
-    )
     """
     return_code, return_message, response = open_url(url, config)
     if return_code == httplib.OK:
@@ -58,13 +55,13 @@ def fetch_from_url_to_file(url, config, output_file):
 
 def open_url(url, config):
     """Attempts to open a connection to a specified URL.
-    @param url - URL to attempt to open
+    @param url: URL to attempt to open
-    @param config - configuration
+    @param config: SSL context configuration
-    @return tuple (
+    @type config: Configuration
+    @return: tuple (
         returned HTTP status code or 0 if an error occurred
         returned message or error description
-        response object
+        response object)
-    )
     """
     debuglevel = 1 if config.debug else 0
 
@@ -120,7 +117,8 @@ def open_url(url, config):
 def _should_use_proxy(url):
     """Determines whether a proxy should be used to open a connection to the 
     specified URL, based on the value of the no_proxy environment variable.
-    @param url - URL string
+    @param url: URL
+    @type url: basestring
     """
     no_proxy = os.environ.get('no_proxy', '')
 
@@ -137,9 +135,9 @@ class Configuration(object):
     """
     def __init__(self, ssl_context, debug):
         """
-        @param key_file - file containing the user's private key
+        @param ssl_context: SSL context to use with this configuration
-        @param cert_file - file containing the user's certificate
+        @type ssl_context: OpenSSL.SSL.Contex        @param debug: if True, output debugging information
-        @param debug - if True, output debugging information
+        @type debug: boo
         """
         self.ssl_context = ssl_context
         self.debug = debug