import cookielib
import httplib
import logging
from optparse import OptionParser
import os
import urllib2
import urlparse
from OpenSSL import SSL
from urllib2pyopenssl.urllib2_build_opener import urllib2_build_opener
from urllib2pyopenssl.https import HTTPSContextHandler
import urllib2pyopenssl.ssl_context_util as ssl_context_util
from urllib2pyopenssl.ssl_peer_verification import ServerSSLCertVerification
def fetch_from_url(url, config):
"""Returns data retrieved from a URL.
@param url - URL to attempt to open
@param config - configuration
@return data retrieved from URL or None
"""
(return_code, return_message, response) = open_url(url, config)
if return_code and return_code == httplib.OK:
return_data = response.read()
response.close()
return return_data
else:
raise Exception(return_message)
def fetch_from_url_to_file(url, config, output_file):
"""Writes data retrieved from a URL to a file.
@param url - URL to attempt to open
@param config - configuration
@param output_file - output file
@return tuple (
returned HTTP status code or 0 if an error occurred
returned message
boolean indicating whether access was successful
)
"""
(return_code, return_message, response) = open_url(url, config)
if return_code == httplib.OK:
return_data = response.read()
response.close()
outfile = open(output_file, "w")
outfile.write(return_data)
outfile.close()
return (return_code, return_message, (return_code == httplib.OK))
def open_url(url, config):
"""Attempts to open a connection to a specified URL.
@param url - URL to attempt to open
@param config - configuration
@return tuple (
returned HTTP status code or 0 if an error occurred
returned message or error description
response object
)
"""
debuglevel = 1 if config.debug else 0
# Set up handlers for URL opener.
cj = cookielib.CookieJar()
cookie_handler = urllib2.HTTPCookieProcessor(cj)
handlers = [cookie_handler]
if config.debug:
http_handler = urllib2.HTTPHandler(debuglevel=debuglevel)
https_handler = HTTPSContextHandler(config.ssl_context, debuglevel=debuglevel)
handlers.extend([http_handler, https_handler])
# Explicitly remove proxy handling if the host is one listed in the value of the no_proxy
# environment variable because urllib2 does use proxy settings set via http_proxy and
# https_proxy, but does not take the no_proxy value into account.
if not _should_use_proxy(url):
handlers.append(urllib2.ProxyHandler({}))
if config.debug:
print "Not using proxy"
opener = urllib2_build_opener(config.ssl_context, *handlers)
# Open the URL and check the response.
return_code = 0
return_message = ''
response = None
try:
response = opener.open(url)
if response.url == url:
return_message = response.msg
return_code = response.code
else:
return_message = ('Redirected (%s %s)' % (response.code, response.url))
if config.debug:
for index, cookie in enumerate(cj):
print index, ' : ', cookie
except urllib2.HTTPError, exc:
return_code = exc.code
return_message = ("Error: %s" % exc.msg)
if config.debug:
print exc.code, exc.msg
except Exception, exc:
return_message = ("Error: %s" % exc.__str__())
if config.debug:
print exc.__class__, exc.__str__()
return (return_code, return_message, response)
def _should_use_proxy(url):
"""Determines whether a proxy should be used to open a connection to the specified URL, based on
the value of the no_proxy environment variable.
@param url - URL string
"""
no_proxy = os.environ.get('no_proxy', '')
urlObj = urlparse.urlparse(url)
for np in [h.strip() for h in no_proxy.split(',')]:
if urlObj.hostname == np:
return False
return True
class Configuration(object):
"""Checker configuration.
"""
def __init__(self, ssl_context, debug):
"""
@param key_file - file containing the user's private key
@param cert_file - file containing the user's certificate
@param debug - if True, output debugging information
"""
self.ssl_context = ssl_context
self.debug = debug
def main():
'''Utility to fetch data using HTTP or HTTPS GET from a specified URL.
'''
parser = OptionParser(usage="%prog [options] url")
parser.add_option("-k", "--private-key", dest="key_file", metavar="FILE",
default=None,
help="Private key file.")
parser.add_option("-c", "--certificate", dest="cert_file", metavar="FILE",
default=os.path.expanduser("~/credentials.pem"),
help="Certificate file.")
parser.add_option("-t", "--ca-certificate-dir", dest="ca_dir", metavar="PATH",
default=None,
help="Trusted CA certificate file directory.")
parser.add_option("-d", "--debug", action="store_true", dest="debug", default=False,
help="Print debug information.")
parser.add_option("-f", "--fetch", dest="output_file", metavar="FILE",
default=None, help="Output file.")
parser.add_option("-v", "--verify-peer", action="store_true", dest="verify_peer", default=False,
help="Verify peer certificate.")
(options, args) = parser.parse_args()
if len(args) != 1:
parser.error("Incorrect number of arguments")
url = args[0]
# If a private key file is not specified, the key is assumed to be stored in the certificate file.
ssl_context = ssl_context_util.make_ssl_context(
options.key_file if options.key_file and os.path.exists(options.key_file) else None,
options.cert_file if options.cert_file and os.path.exists(options.cert_file) else None,
None,
options.ca_dir if options.ca_dir and os.path.exists(options.ca_dir) else None,
options.verify_peer, url)
config = Configuration(ssl_context, options.debug)
if options.output_file:
(return_code, return_message, success) = fetch_from_url_to_file(url, config,
options.output_file)
print return_code, return_message
else:
data = fetch_from_url(url, config)
print data
if __name__=='__main__':
logging.basicConfig()
main()