ndg_httpsclient/ndg/httpsclient/utils.py

import cookielib
import httplib
import logging
from optparse import OptionParser
import os
import sys
import urllib2
if sys.version_info < (2, 6, 2):
    from ndg.httpsclient.urllib2_proxy import HTTPHandler
else:
    from urllib2 import HTTPHandler
    
import urlparse

from ndg.httpsclient.urllib2_build_opener import build_opener
from ndg.httpsclient.https import HTTPSContextHandler
from ndg.httpsclient import ssl_context_util


class URLFetchError(Exception):
    """Error fetching content from URL"""
    

def fetch_from_url(url, config):
    """Returns data retrieved from a URL.
    @param url - URL to attempt to open
    @param config - configuration
    @return data retrieved from URL or None
    """
    return_code, return_message, response = open_url(url, config)
    if return_code and return_code == httplib.OK:
        return_data = response.read()
        response.close()
        return return_data
    else:
        raise URLFetchError(return_message)

def fetch_from_url_to_file(url, config, output_file):
    """Writes data retrieved from a URL to a file.
    @param url - URL to attempt to open
    @param config - configuration
    @param output_file - output file
    @return tuple (
        returned HTTP status code or 0 if an error occurred
        returned message
        boolean indicating whether access was successful
    )
    """
    return_code, return_message, response = open_url(url, config)
    if return_code == httplib.OK:
        return_data = response.read()
        response.close()
        outfile = open(output_file, "w")
        outfile.write(return_data)
        outfile.close()
    return return_code, return_message, return_code == httplib.OK


def open_url(url, config):
    """Attempts to open a connection to a specified URL.
    @param url - URL to attempt to open
    @param config - configuration
    @return tuple (
        returned HTTP status code or 0 if an error occurred
        returned message or error description
        response object
    )
    """
    debuglevel = 1 if config.debug else 0

    # Set up handlers for URL opener.
    cj = cookielib.CookieJar()
    cookie_handler = urllib2.HTTPCookieProcessor(cj)

    handlers = [cookie_handler]

    if config.debug:
        http_handler = HTTPHandler(debuglevel=debuglevel)
        https_handler = HTTPSContextHandler(config.ssl_context, 
                                            debuglevel=debuglevel)
        handlers.extend([http_handler, https_handler])

    # Explicitly remove proxy handling if the host is one listed in the value of
    # the no_proxy environment variable because urllib2 does use proxy settings 
    # set via http_proxy and https_proxy, but does not take the no_proxy value 
    # into account.
    if not _should_use_proxy(url):
        handlers.append(urllib2.ProxyHandler({}))
        if config.debug:
            print "Not using proxy"

    opener = build_opener(config.ssl_context, *handlers)

    # Open the URL and check the response.
    return_code = 0
    return_message = ''
    response = None
    try:
        response = opener.open(url)
        if response.url == url:
            return_message = response.msg
            return_code = response.code
        else:
            return_message = 'Redirected (%s  %s)' % response.code, response.url
        if config.debug:
            for index, cookie in enumerate(cj):
                print index, '  :  ', cookie        
    except urllib2.HTTPError, exc:
        return_code = exc.code
        return_message = "Error: %s" % exc.msg
        if config.debug:
            print exc.code, exc.msg
    except Exception, exc:
        return_message = "Error: %s" % exc.__str__()
        if config.debug:
            print exc.__class__, exc.__str__()
    return (return_code, return_message, response)


def _should_use_proxy(url):
    """Determines whether a proxy should be used to open a connection to the 
    specified URL, based on the value of the no_proxy environment variable.
    @param url - URL string
    """
    no_proxy = os.environ.get('no_proxy', '')

    urlObj = urlparse.urlparse(url)
    for np in [h.strip() for h in no_proxy.split(',')]:
        if urlObj.hostname == np:
            return False

    return True


class Configuration(object):
    """Checker configuration.
    """
    def __init__(self, ssl_context, debug):
        """
        @param key_file - file containing the user's private key
        @param cert_file - file containing the user's certificate
        @param debug - if True, output debugging information
        """
        self.ssl_context = ssl_context
        self.debug = debug


def main():
    '''Utility to fetch data using HTTP or HTTPS GET from a specified URL.
    '''
    parser = OptionParser(usage="%prog [options] url")
    parser.add_option("-k", "--private-key", dest="key_file", metavar="FILE",
                      default=None,
                      help="Private key file.")
    parser.add_option("-c", "--certificate", dest="cert_file", metavar="FILE",
                      default=os.path.expanduser("~/credentials.pem"),
                      help="Certificate file.")
    parser.add_option("-t", "--ca-certificate-dir", dest="ca_dir", 
                      metavar="PATH",
                      default=None,
                      help="Trusted CA certificate file directory.")
    parser.add_option("-d", "--debug", action="store_true", dest="debug", 
                      default=False,
                      help="Print debug information.")
    parser.add_option("-f", "--fetch", dest="output_file", metavar="FILE",
                      default=None, help="Output file.")
    parser.add_option("-n", "--no-verify-peer", action="store_true", 
                      dest="no_verify_peer", default=False,
                      help="Skip verification of peer certificate.")
    (options, args) = parser.parse_args()
    if len(args) != 1:
        parser.error("Incorrect number of arguments")

    url = args[0]
    
    if options.key_file and os.path.exists(options.key_file):
        key_file = options.key_file
    else:
        key_file = None
    
    if options.cert_file and os.path.exists(options.cert_file):
        cert_file = options.cert_file
    else:
        cert_file = None
    
    if options.ca_dir and os.path.exists(options.ca_dir):
        ca_dir = options.ca_dir 
    else:
        ca_dir = None
        
    verify_peer = not options.no_verify_peer
    
    # If a private key file is not specified, the key is assumed to be stored in 
    # the certificate file.
    ssl_context = ssl_context_util.make_ssl_context(key_file,
                                                    cert_file,
                                                    None,
                                                    ca_dir,
                                                    verify_peer, 
                                                    url)

    config = Configuration(ssl_context, options.debug)
    if options.output_file:
        return_code, return_message = fetch_from_url_to_file(url, 
                                                      config,
                                                      options.output_file)[:2]
        raise SystemExit(return_code, return_message)
    else:
        data = fetch_from_url(url, config)
        print(data)


if __name__=='__main__':
    logging.basicConfig()
    main()