|
|
@@ -1,262 +0,0 @@
|
|
|
-'''
|
|
|
-Created on 12 Jan 2012
|
|
|
-
|
|
|
-@author: rwilkinson
|
|
|
-'''
|
|
|
-import base64
|
|
|
-import socket
|
|
|
-import urlparse
|
|
|
-from urllib import unquote, addinfourl
|
|
|
-from urllib2 import _parse_proxy, URLError, HTTPError
|
|
|
-from urllib2 import (AbstractHTTPHandler as _AbstractHTTPHandler,
|
|
|
- BaseHandler as _BaseHandler,
|
|
|
- HTTPRedirectHandler as _HTTPRedirectHandler,
|
|
|
- Request as _Request,
|
|
|
- OpenerDirector as _OpenerDirector)
|
|
|
-
|
|
|
-from ndg.httpsclient.httplib_proxy import HTTPConnection
|
|
|
-
|
|
|
-
|
|
|
-class Request(_Request):
|
|
|
-
|
|
|
- def __init__(self, *args, **kw):
|
|
|
- _Request.__init__(self, *args, **kw)
|
|
|
- self._tunnel_host = None
|
|
|
-
|
|
|
- def set_proxy(self, host, type):
|
|
|
- if self.type == 'https' and not self._tunnel_host:
|
|
|
- self._tunnel_host = self.host
|
|
|
- else:
|
|
|
- self.type = type
|
|
|
- self.__r_host = self.__original
|
|
|
- self.host = host
|
|
|
-
|
|
|
-
|
|
|
-class BaseHandler(_BaseHandler):
|
|
|
- def proxy_open(self, req, proxy, type):
|
|
|
- if req.get_type() == 'https':
|
|
|
- orig_type = req.get_type()
|
|
|
- proxy_type, user, password, hostport = _parse_proxy(proxy)
|
|
|
- if proxy_type is None:
|
|
|
- proxy_type = orig_type
|
|
|
- if user and password:
|
|
|
- user_pass = '%s:%s' % (unquote(user), unquote(password))
|
|
|
- creds = base64.b64encode(user_pass).strip()
|
|
|
- req.add_header('Proxy-authorization', 'Basic ' + creds)
|
|
|
- hostport = unquote(hostport)
|
|
|
- req.set_proxy(hostport, proxy_type)
|
|
|
- # let other handlers take care of it
|
|
|
- return None
|
|
|
- else:
|
|
|
- return _BaseHandler.proxy_open(self, req, proxy, type)
|
|
|
-
|
|
|
-class AbstractHTTPHandler(_AbstractHTTPHandler):
|
|
|
- def do_open(self, http_class, req):
|
|
|
- """Return an addinfourl object for the request, using http_class.
|
|
|
-
|
|
|
- http_class must implement the HTTPConnection API from httplib.
|
|
|
- The addinfourl return value is a file-like object. It also
|
|
|
- has methods and attributes including:
|
|
|
- - info(): return a mimetools.Message object for the headers
|
|
|
- - geturl(): return the original request URL
|
|
|
- - code: HTTP status code
|
|
|
- """
|
|
|
- host = req.get_host()
|
|
|
- if not host:
|
|
|
- raise URLError('no host given')
|
|
|
-
|
|
|
- h = http_class(host, timeout=req.timeout) # will parse host:port
|
|
|
- h.set_debuglevel(self._debuglevel)
|
|
|
-
|
|
|
- headers = dict(req.headers)
|
|
|
- headers.update(req.unredirected_hdrs)
|
|
|
- # We want to make an HTTP/1.1 request, but the addinfourl
|
|
|
- # class isn't prepared to deal with a persistent connection.
|
|
|
- # It will try to read all remaining data from the socket,
|
|
|
- # which will block while the server waits for the next request.
|
|
|
- # So make sure the connection gets closed after the (only)
|
|
|
- # request.
|
|
|
- headers["Connection"] = "close"
|
|
|
- headers = dict(
|
|
|
- (name.title(), val) for name, val in headers.items())
|
|
|
-
|
|
|
- if not hasattr(req, '_tunnel_host'):
|
|
|
- pass
|
|
|
-
|
|
|
- if req._tunnel_host:
|
|
|
- h.set_tunnel(req._tunnel_host)
|
|
|
- try:
|
|
|
- h.request(req.get_method(), req.get_selector(), req.data, headers)
|
|
|
- r = h.getresponse()
|
|
|
- except socket.error, err: # XXX what error?
|
|
|
- raise URLError(err)
|
|
|
-
|
|
|
- # Pick apart the HTTPResponse object to get the addinfourl
|
|
|
- # object initialized properly.
|
|
|
-
|
|
|
- # Wrap the HTTPResponse object in socket's file object adapter
|
|
|
- # for Windows. That adapter calls recv(), so delegate recv()
|
|
|
- # to read(). This weird wrapping allows the returned object to
|
|
|
- # have readline() and readlines() methods.
|
|
|
-
|
|
|
- # XXX It might be better to extract the read buffering code
|
|
|
- # out of socket._fileobject() and into a base class.
|
|
|
-
|
|
|
- r.recv = r.read
|
|
|
- fp = socket._fileobject(r, close=True)
|
|
|
-
|
|
|
- resp = addinfourl(fp, r.msg, req.get_full_url())
|
|
|
- resp.code = r.status
|
|
|
- resp.msg = r.reason
|
|
|
- return resp
|
|
|
-
|
|
|
-
|
|
|
-class HTTPHandler(AbstractHTTPHandler):
|
|
|
-
|
|
|
- def http_open(self, req):
|
|
|
- return self.do_open(HTTPConnection, req)
|
|
|
-
|
|
|
- http_request = AbstractHTTPHandler.do_request_
|
|
|
-
|
|
|
-#if hasattr(httplib, 'HTTPS'):
|
|
|
-# class HTTPSHandler(AbstractHTTPHandler):
|
|
|
-#
|
|
|
-# def https_open(self, req):
|
|
|
-# return self.do_open(httplib.HTTPSConnection, req)
|
|
|
-#
|
|
|
-# https_request = AbstractHTTPHandler.do_request_
|
|
|
-
|
|
|
-
|
|
|
-class HTTPRedirectHandler(BaseHandler):
|
|
|
- # maximum number of redirections to any single URL
|
|
|
- # this is needed because of the state that cookies introduce
|
|
|
- max_repeats = 4
|
|
|
- # maximum total number of redirections (regardless of URL) before
|
|
|
- # assuming we're in a loop
|
|
|
- max_redirections = 10
|
|
|
-
|
|
|
- def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
|
- """Return a Request or None in response to a redirect.
|
|
|
-
|
|
|
- This is called by the http_error_30x methods when a
|
|
|
- redirection response is received. If a redirection should
|
|
|
- take place, return a new Request to allow http_error_30x to
|
|
|
- perform the redirect. Otherwise, raise HTTPError if no-one
|
|
|
- else should try to handle this url. Return None if you can't
|
|
|
- but another Handler might.
|
|
|
- """
|
|
|
- m = req.get_method()
|
|
|
- if (code in (301, 302, 303, 307) and m in ("GET", "HEAD")
|
|
|
- or code in (301, 302, 303) and m == "POST"):
|
|
|
- # Strictly (according to RFC 2616), 301 or 302 in response
|
|
|
- # to a POST MUST NOT cause a redirection without confirmation
|
|
|
- # from the user (of urllib2, in this case). In practice,
|
|
|
- # essentially all clients do redirect in this case, so we
|
|
|
- # do the same.
|
|
|
- # be conciliant with URIs containing a space
|
|
|
- newurl = newurl.replace(' ', '%20')
|
|
|
- newheaders = dict((k,v) for k,v in req.headers.items()
|
|
|
- if k.lower() not in ("content-length", "content-type")
|
|
|
- )
|
|
|
- return Request(newurl,
|
|
|
- headers=newheaders,
|
|
|
- origin_req_host=req.get_origin_req_host(),
|
|
|
- unverifiable=True)
|
|
|
- else:
|
|
|
- raise HTTPError(req.get_full_url(), code, msg, headers, fp)
|
|
|
-
|
|
|
- # Implementation note: To avoid the server sending us into an
|
|
|
- # infinite loop, the request object needs to track what URLs we
|
|
|
- # have already seen. Do this by adding a handler-specific
|
|
|
- # attribute to the Request object.
|
|
|
- def http_error_302(self, req, fp, code, msg, headers):
|
|
|
- # Some servers (incorrectly) return multiple Location headers
|
|
|
- # (so probably same goes for URI). Use first header.
|
|
|
- if 'location' in headers:
|
|
|
- newurl = headers.getheaders('location')[0]
|
|
|
- elif 'uri' in headers:
|
|
|
- newurl = headers.getheaders('uri')[0]
|
|
|
- else:
|
|
|
- return
|
|
|
-
|
|
|
- # fix a possible malformed URL
|
|
|
- urlparts = urlparse.urlparse(newurl)
|
|
|
- if not urlparts.path:
|
|
|
- urlparts = list(urlparts)
|
|
|
- urlparts[2] = "/"
|
|
|
- newurl = urlparse.urlunparse(urlparts)
|
|
|
-
|
|
|
- newurl = urlparse.urljoin(req.get_full_url(), newurl)
|
|
|
-
|
|
|
- # For security reasons we do not allow redirects to protocols
|
|
|
- # other than HTTP, HTTPS or FTP.
|
|
|
- newurl_lower = newurl.lower()
|
|
|
- if not (newurl_lower.startswith('http://') or
|
|
|
- newurl_lower.startswith('https://') or
|
|
|
- newurl_lower.startswith('ftp://')):
|
|
|
- raise HTTPError(newurl, code,
|
|
|
- msg + " - Redirection to url '%s' is not allowed" %
|
|
|
- newurl,
|
|
|
- headers, fp)
|
|
|
-
|
|
|
- # XXX Probably want to forget about the state of the current
|
|
|
- # request, although that might interact poorly with other
|
|
|
- # handlers that also use handler-specific request attributes
|
|
|
- new = self.redirect_request(req, fp, code, msg, headers, newurl)
|
|
|
- if new is None:
|
|
|
- return
|
|
|
-
|
|
|
- # loop detection
|
|
|
- # .redirect_dict has a key url if url was previously visited.
|
|
|
- if hasattr(req, 'redirect_dict'):
|
|
|
- visited = new.redirect_dict = req.redirect_dict
|
|
|
- if (visited.get(newurl, 0) >= self.max_repeats or
|
|
|
- len(visited) >= self.max_redirections):
|
|
|
- raise HTTPError(req.get_full_url(), code,
|
|
|
- self.inf_msg + msg, headers, fp)
|
|
|
- else:
|
|
|
- visited = new.redirect_dict = req.redirect_dict = {}
|
|
|
- visited[newurl] = visited.get(newurl, 0) + 1
|
|
|
-
|
|
|
- # Don't close the fp until we are sure that we won't use it
|
|
|
- # with HTTPError.
|
|
|
- fp.read()
|
|
|
- fp.close()
|
|
|
-
|
|
|
- return self.parent.open(new, timeout=req.timeout)
|
|
|
-
|
|
|
- http_error_301 = http_error_303 = http_error_307 = http_error_302
|
|
|
-
|
|
|
- inf_msg = "The HTTP server returned a redirect error that would " \
|
|
|
- "lead to an infinite loop.\n" \
|
|
|
- "The last 30x error message was:\n"
|
|
|
-
|
|
|
-
|
|
|
-class OpenerDirector(_OpenerDirector):
|
|
|
- def open(self, fullurl, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
|
|
|
- # accept a URL or a Request object
|
|
|
- if isinstance(fullurl, basestring):
|
|
|
- req = Request(fullurl, data)
|
|
|
- else:
|
|
|
- req = fullurl
|
|
|
- if data is not None:
|
|
|
- req.add_data(data)
|
|
|
-
|
|
|
- req.timeout = timeout
|
|
|
- protocol = req.get_type()
|
|
|
-
|
|
|
- # pre-process request
|
|
|
- meth_name = protocol+"_request"
|
|
|
- for processor in self.process_request.get(protocol, []):
|
|
|
- meth = getattr(processor, meth_name)
|
|
|
- req = meth(req)
|
|
|
-
|
|
|
- response = self._open(req, data)
|
|
|
-
|
|
|
- # post-process response
|
|
|
- meth_name = protocol+"_response"
|
|
|
- for processor in self.process_response.get(protocol, []):
|
|
|
- meth = getattr(processor, meth_name)
|
|
|
- response = meth(req, response)
|
|
|
-
|
|
|
- return response
|
pjkersha