#!@PYTHON@ # Copyright (C) 2011-2012 Internet Systems Consortium. # # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. # # THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM # DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL # INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT, # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING # FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION # WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ A standalone HTTP server for HTTP/XML interface of statistics in BIND 10 """ import sys; sys.path.append ('@@PYTHONPATH@@') import os import time import errno import select from optparse import OptionParser, OptionValueError import http.server import socket import string import xml.etree.ElementTree import urllib.parse import re import isc.cc import isc.config import isc.util.process import isc.log from isc.log_messages.stats_httpd_messages import * isc.log.init("b10-stats-httpd", buffer=True) logger = isc.log.Logger("stats-httpd") # Some constants for debug levels. DBG_STATSHTTPD_INIT = logger.DBGLVL_START_SHUT DBG_STATSHTTPD_MESSAGING = logger.DBGLVL_COMMAND # If B10_FROM_SOURCE is set in the environment, we use data files # from a directory relative to that, otherwise we use the ones # installed on the system if "B10_FROM_SOURCE" in os.environ: BASE_LOCATION = os.environ["B10_FROM_SOURCE"] + os.sep + \ "src" + os.sep + "bin" + os.sep + "stats" else: PREFIX = "@prefix@" DATAROOTDIR = "@datarootdir@" BASE_LOCATION = "@datadir@" + os.sep + "@PACKAGE@" BASE_LOCATION = BASE_LOCATION.replace("${datarootdir}", DATAROOTDIR).replace("${prefix}", PREFIX) SPECFILE_LOCATION = BASE_LOCATION + os.sep + "stats-httpd.spec" XML_TEMPLATE_LOCATION = BASE_LOCATION + os.sep + "stats-httpd-xml.tpl" XSD_TEMPLATE_LOCATION = BASE_LOCATION + os.sep + "stats-httpd-xsd.tpl" XSL_TEMPLATE_LOCATION = BASE_LOCATION + os.sep + "stats-httpd-xsl.tpl" # These variables are paths part of URL. # eg. "http://${address}" + XXX_URL_PATH XML_URL_PATH = '/bind10/statistics/xml' XSD_URL_PATH = '/bind10/statistics.xsd' XSL_URL_PATH = '/bind10/statistics.xsl' # TODO: This should be considered later. XSD_NAMESPACE = 'http://bind10.isc.org/bind10' XMLNS_XSI = "http://www.w3.org/2001/XMLSchema-instance" # constant parameter of XML XML_ROOT_ELEMENT = 'bind10:statistics' XML_ROOT_ATTRIB = { 'xsi:schemaLocation' : '%s %s' % (XSD_NAMESPACE, XSD_URL_PATH), 'xmlns:bind10' : XSD_NAMESPACE, 'xmlns:xsi' : XMLNS_XSI } # Assign this process name isc.util.process.rename() def item_name_list(element, identifier): """Return a list of strings. The strings are string expression of the first argument element which is dict type. The second argument identifier is a string for specifying the strings which are returned from this method as a list. For example, if we specify as item_name_list({'a': {'aa': [0, 1]}, 'b': [0, 1]}, 'a/aa'), then it returns ['a/aa', 'a/aa[0]', 'a/aa[1]']. If an empty string is specified in the second argument, all possible strings are returned as a list. In that example if we specify an empty string in the second argument, then it returns ['a', 'a/aa', 'a/aa[0]', 'a/aa[1]', 'b', 'b[0]', 'b[1]']. The key name of element which is in the first argument is sorted. Even if we specify as item_name_list({'xx': 0, 'a': 1, 'x': 2}, ''), then it returns ['a', 'x', 'xx']. This method internally invokes isc.cc.data.find(). The arguments of this method are passed to isc.cc.data.find(). So an exception DataNotFoundError or DataTypeError might be raised via isc.cc.data.find() depending on the arguments. See details of isc.cc.data.find() for more information about exceptions""" elem = isc.cc.data.find(element, identifier) ret = [] ident = identifier if ident: ret.append(ident) if type(elem) is dict: if ident: ident = ident + '/' for key in sorted(elem.keys()): idstr = '%s%s' % (ident, key) ret += item_name_list(element, idstr) elif type(elem) is list: for i in range(0, len(elem)): idstr = '%s[%d]' % (ident, i) ret += item_name_list(element, idstr) return ret class HttpHandler(http.server.BaseHTTPRequestHandler): """HTTP handler class for HttpServer class. The class inhrits the super class http.server.BaseHTTPRequestHandler. It implemets do_GET() and do_HEAD() and orverrides log_message()""" def do_GET(self): body = self.send_head() if body is not None: self.wfile.write(body.encode()) def do_HEAD(self): self.send_head() def send_head(self): try: req_path = self.path req_path = urllib.parse.urlsplit(req_path).path req_path = urllib.parse.unquote(req_path) body = None # In case that the requested path (req_path), # e.g. /bind10/statistics/Auth/, is started with # XML_URL_PATH + '/' if req_path.find('%s/' % XML_URL_PATH) == 0: # remove './' from the path if there is req_path = os.path.normpath(req_path) # get the strings tailing after XML_URL_PATH req_path = req_path.lstrip('%s/' % XML_URL_PATH) # remove empty dir names from the path if there are path_dirs = req_path.split('/') path_dirs = [ d for d in filter(None, path_dirs) ] req_path = '/'.join(path_dirs) # pass the complete requested path, # e.g. Auth/queries.upd, to xml_handler() body = self.server.xml_handler(req_path) # In case that the requested path (req_path) is exactly # matched with XSD_URL_PATH elif req_path == XSD_URL_PATH: body = self.server.xsd_handler() # In case that the requested path (req_path) is exactly # matched with XSL_URL_PATH elif req_path == XSL_URL_PATH: body = self.server.xsl_handler() else: if 'Host' in self.headers.keys() and \ ( req_path == '/' or req_path == XML_URL_PATH ): # redirect to XML URL only when requested with '/' or XML_URL_PATH toloc = "http://%s%s/" % (self.headers.get('Host'), XML_URL_PATH) self.send_response(302) self.send_header("Location", toloc) self.end_headers() return None else: # Couldn't find HOST self.send_error(404) return None except StatsHttpdDataError as err: # Couldn't find neither specified module name nor # specified item name self.send_error(404) logger.error(STATSHTTPD_SERVER_DATAERROR, err) return None except Exception as err: self.send_error(500) logger.error(STATSHTTPD_SERVER_ERROR, err) return None else: self.send_response(200) self.send_header("Content-type", "text/xml") self.send_header("Content-Length", len(body)) self.send_header("Last-Modified", self.date_time_string(time.time())) self.end_headers() return body def log_message(self, format, *args): """overrides the parent method log_message() to use the bind10 logging framework. """ logger.debug(DBG_STATSHTTPD_MESSAGING, STATSHTTPD_HTTPLOG, self.address_string(), format%args) class HttpServerError(Exception): """Exception class for HttpServer class. It is intended to be passed from the HttpServer object to the StatsHttpd object.""" pass class HttpServer(http.server.HTTPServer): """HTTP Server class. The class inherits the super http.server.HTTPServer. Some parameters are specified as arguments, which are xml_handler, xsd_handler, xsl_handler, and log_writer. These all are parameters which the StatsHttpd object has. The handler parameters are references of functions which return body of each document. The last parameter log_writer is reference of writer function to just write to sys.stderr.write. They are intended to be referred by HttpHandler object.""" def __init__(self, server_address, handler, xml_handler, xsd_handler, xsl_handler, log_writer): self.server_address = server_address self.xml_handler = xml_handler self.xsd_handler = xsd_handler self.xsl_handler = xsl_handler self.log_writer = log_writer http.server.HTTPServer.__init__(self, server_address, handler) class StatsHttpdError(Exception): """Exception class for StatsHttpd class. It is intended to be thrown from the the StatsHttpd object to the HttpHandler object or main routine.""" pass class StatsHttpdDataError(Exception): """Exception class for StatsHttpd class. The reason seems to be due to the data. It is intended to be thrown from the the StatsHttpd object to the HttpHandler object or main routine.""" pass class StatsHttpd: """The main class of HTTP server of HTTP/XML interface for statistics module. It handles HTTP requests, and command channel and config channel CC session. It uses select.select function while waiting for clients requests.""" def __init__(self): self.running = False self.poll_intval = 0.5 self.write_log = sys.stderr.write self.mccs = None self.httpd = [] self.open_mccs() self.config = {} self.load_config() self.http_addrs = [] self.mccs.start() try: self.open_httpd() except HttpServerError: # if some exception, e.g. address in use, is raised, then it closes mccs and httpd self.close_mccs() raise def open_mccs(self): """Opens a ModuleCCSession object""" # create ModuleCCSession logger.debug(DBG_STATSHTTPD_INIT, STATSHTTPD_STARTING_CC_SESSION) self.mccs = isc.config.ModuleCCSession( SPECFILE_LOCATION, self.config_handler, self.command_handler) self.cc_session = self.mccs._session def close_mccs(self): """Closes a ModuleCCSession object""" if self.mccs is None: return self.mccs.send_stopping() logger.debug(DBG_STATSHTTPD_INIT, STATSHTTPD_CLOSING_CC_SESSION) self.mccs.close() self.mccs = None def load_config(self, new_config={}): """Loads configuration from spec file or new configuration from the config manager""" # load config if len(self.config) == 0: self.config = dict([ (itm['item_name'], self.mccs.get_value(itm['item_name'])[0]) for itm in self.mccs.get_module_spec().get_config_spec() ]) self.config.update(new_config) # set addresses and ports for HTTP addrs = [] if 'listen_on' in self.config: for cf in self.config['listen_on']: if 'address' in cf and 'port' in cf: addrs.append((cf['address'], cf['port'])) self.http_addrs = addrs def open_httpd(self): """Opens sockets for HTTP. Iterating each HTTP address to be configured in spec file""" for addr in self.http_addrs: self.httpd.append(self._open_httpd(addr)) def _open_httpd(self, server_address): httpd = None try: # get address family for the server_address before # creating HttpServer object. If a specified address is # not numerical, gaierror may be thrown. address_family = socket.getaddrinfo( server_address[0], server_address[1], 0, socket.SOCK_STREAM, socket.IPPROTO_TCP, socket.AI_NUMERICHOST )[0][0] HttpServer.address_family = address_family httpd = HttpServer( server_address, HttpHandler, self.xml_handler, self.xsd_handler, self.xsl_handler, self.write_log) logger.info(STATSHTTPD_STARTED, server_address[0], server_address[1]) return httpd except (socket.gaierror, socket.error, OverflowError, TypeError) as err: if httpd: httpd.server_close() raise HttpServerError( "Invalid address %s, port %s: %s: %s" % (server_address[0], server_address[1], err.__class__.__name__, err)) def close_httpd(self): """Closes sockets for HTTP""" while len(self.httpd)>0: ht = self.httpd.pop() logger.info(STATSHTTPD_CLOSING, ht.server_address[0], ht.server_address[1]) ht.server_close() def start(self): """Starts StatsHttpd objects to run. Waiting for client requests by using select.select functions""" self.running = True while self.running: try: (rfd, wfd, xfd) = select.select( self.get_sockets(), [], [], self.poll_intval) except select.error as err: # select.error exception is caught only in the case of # EINTR, or in other cases it is just thrown. if err.args[0] == errno.EINTR: (rfd, wfd, xfd) = ([], [], []) else: raise # FIXME: This module can handle only one request at a # time. If someone sends only part of the request, we block # waiting for it until we time out. # But it isn't so big issue for administration purposes. for fd in rfd + xfd: if fd == self.mccs.get_socket(): self.mccs.check_command(nonblock=False) continue for ht in self.httpd: if fd == ht.socket: ht.handle_request() break self.stop() def stop(self): """Stops the running StatsHttpd objects. Closes CC session and HTTP handling sockets""" logger.info(STATSHTTPD_SHUTDOWN) self.close_httpd() self.close_mccs() self.running = False def get_sockets(self): """Returns sockets to select.select""" sockets = [] if self.mccs is not None: sockets.append(self.mccs.get_socket()) if len(self.httpd) > 0: for ht in self.httpd: sockets.append(ht.socket) return sockets def config_handler(self, new_config): """Config handler for the ModuleCCSession object. It resets addresses and ports to listen HTTP requests on.""" logger.debug(DBG_STATSHTTPD_MESSAGING, STATSHTTPD_HANDLE_CONFIG, new_config) errors = [] if not self.mccs.get_module_spec().\ validate_config(False, new_config, errors): return isc.config.ccsession.create_answer( 1, ", ".join(errors)) # backup old config old_config = self.config.copy() self.load_config(new_config) # If the http sockets aren't opened or # if new_config doesn't have'listen_on', it returns if len(self.httpd) == 0 or 'listen_on' not in new_config: return isc.config.ccsession.create_answer(0) self.close_httpd() try: self.open_httpd() except HttpServerError as err: logger.error(STATSHTTPD_SERVER_ERROR, err) # restore old config self.load_config(old_config) self.open_httpd() return isc.config.ccsession.create_answer(1, str(err)) else: return isc.config.ccsession.create_answer(0) def command_handler(self, command, args): """Command handler for the ModuleCCSesson object. It handles "status" and "shutdown" commands.""" if command == "status": logger.debug(DBG_STATSHTTPD_MESSAGING, STATSHTTPD_RECEIVED_STATUS_COMMAND) return isc.config.ccsession.create_answer( 0, "Stats Httpd is up. (PID " + str(os.getpid()) + ")") elif command == "shutdown": logger.debug(DBG_STATSHTTPD_MESSAGING, STATSHTTPD_RECEIVED_SHUTDOWN_COMMAND) self.running = False return isc.config.ccsession.create_answer(0) else: logger.debug(DBG_STATSHTTPD_MESSAGING, STATSHTTPD_RECEIVED_UNKNOWN_COMMAND, command) return isc.config.ccsession.create_answer( 1, "Unknown command: " + str(command)) def get_stats_data(self, owner=None, name=None): """Requests statistics data to the Stats daemon and returns the data which obtains from it. The first argument is the module name which owns the statistics data, the second argument is one name of the statistics items which the the module owns. The second argument cannot be specified when the first argument is not specified. It returns the statistics data of the specified module or item. When the session timeout or the session error is occurred, it raises StatsHttpdError. When the stats daemon returns none-zero value, it raises StatsHttpdDataError.""" param = {} if owner is None and name is None: param = None if owner is not None: param['owner'] = owner if name is not None: param['name'] = name try: seq = self.cc_session.group_sendmsg( isc.config.ccsession.create_command('show', param), 'Stats') (answer, env) = self.cc_session.group_recvmsg(False, seq) if answer: (rcode, value) = isc.config.ccsession.parse_answer(answer) except (isc.cc.session.SessionTimeout, isc.cc.session.SessionError) as err: raise StatsHttpdError("%s: %s" % (err.__class__.__name__, err)) else: if rcode == 0: return value else: raise StatsHttpdDataError("Stats module: %s" % str(value)) def get_stats_spec(self, owner=None, name=None): """Requests statistics data to the Stats daemon and returns the data which obtains from it. The first argument is the module name which owns the statistics data, the second argument is one name of the statistics items which the the module owns. The second argument cannot be specified when the first argument is not specified. It returns the statistics specification of the specified module or item. When the session timeout or the session error is occurred, it raises StatsHttpdError. When the stats daemon returns none-zero value, it raises StatsHttpdDataError.""" param = {} if owner is None and name is None: param = None if owner is not None: param['owner'] = owner if name is not None: param['name'] = name try: seq = self.cc_session.group_sendmsg( isc.config.ccsession.create_command('showschema', param), 'Stats') (answer, env) = self.cc_session.group_recvmsg(False, seq) if answer: (rcode, value) = isc.config.ccsession.parse_answer(answer) if rcode == 0: return value else: raise StatsHttpdDataError("Stats module: %s" % str(value)) except (isc.cc.session.SessionTimeout, isc.cc.session.SessionError) as err: raise StatsHttpdError("%s: %s" % (err.__class__.__name__, err)) def xml_handler(self, path=''): """Requests the specified statistics data and specification by using the functions get_stats_data and get_stats_spec respectively and loads the XML template file and returns the string of the XML document.The argument is a path in the requested URI. For example, the path is assumed to be like ${module_name}/${top_level_item_name}/${second_level_item_name}/...""" dirs = [ d for d in path.split("/") if len(d) > 0 ] module_name = None item_name = None if len(dirs) > 0: module_name = dirs[0] if len(dirs) > 1: item_name = dirs[1] # removed an index string when list-type value is # requested. Because such a item name can be accept by the # stats module currently. item_name = re.sub('\[\d+\]$', '', item_name) stats_spec = self.get_stats_spec(module_name, item_name) stats_data = self.get_stats_data(module_name, item_name) path_list = [] try: path_list = item_name_list(stats_data, path) except isc.cc.data.DataNotFoundError as err: raise StatsHttpdDataError( "%s: %s" % (err.__class__.__name__, err)) item_list = [] for path in path_list: dirs = path.split("/") if len(dirs) < 2: continue item = {} item['identifier'] = path value = isc.cc.data.find(stats_data, path) if type(value) is bool: value = str(value).lower() if type(value) is dict or type(value) is list: value = None if value is not None: item['value'] = str(value) owner = dirs[0] item['owner'] = owner item['uri'] = urllib.parse.quote('%s/%s' % (XML_URL_PATH, path)) item_path = '/'.join(dirs[1:]) spec = isc.config.find_spec_part(stats_spec[owner], item_path) for key in ['name', 'type', 'description', 'title', \ 'optional', 'default', 'format']: value = spec.get('item_%s' % key) if type(value) is bool: value = str(value).lower() if type(value) is dict or type(value) is list: value = None if value is not None: item[key] = str(value) item_list.append(item) xml_elem = xml.etree.ElementTree.Element( XML_ROOT_ELEMENT, attrib=XML_ROOT_ATTRIB) for item in item_list: item_elem = xml.etree.ElementTree.Element('item', attrib=item) xml_elem.append(item_elem) # The coding conversion is tricky. xml..tostring() of Python 3.2 # returns bytes (not string) regardless of the coding, while # tostring() of Python 3.1 returns a string. To support both # cases transparently, we first make sure tostring() returns # bytes by specifying utf-8 and then convert the result to a # plain string (code below assume it). # FIXME: Non-ASCII characters might be lost here. Consider how # the whole system should handle non-ASCII characters. xml_string = str(xml.etree.ElementTree.tostring(xml_elem, encoding='utf-8'), encoding='us-ascii') self.xml_body = self.open_template(XML_TEMPLATE_LOCATION).substitute( xml_string=xml_string, xsl_url_path=XSL_URL_PATH) return self.xml_body def xsd_handler(self): """Loads the XSD template file, replaces the variable strings, and returns the string of the XSD document.""" self.xsd_body = self.open_template(XSD_TEMPLATE_LOCATION).substitute( xsd_namespace=XSD_NAMESPACE) return self.xsd_body def xsl_handler(self): """Loads the XSL template file, replaces the variable strings, and returns the string of the XSL document.""" self.xsl_body = self.open_template(XSL_TEMPLATE_LOCATION).substitute( xsd_namespace=XSD_NAMESPACE) return self.xsl_body def open_template(self, file_name): """It opens a template file, and it loads all lines to a string variable and returns string. Template object includes the variable. Limitation of a file size isn't needed there. XXXX""" lines = None try: with open(file_name, 'r') as f: lines = "".join(f.readlines()) except IOError as err: raise StatsHttpdDataError( "%s: %s" % (err.__class__.__name__, err)) return string.Template(lines) if __name__ == "__main__": try: parser = OptionParser() parser.add_option( "-v", "--verbose", dest="verbose", action="store_true", help="enable maximum debug logging") (options, args) = parser.parse_args() if options.verbose: isc.log.init("b10-stats-httpd", "DEBUG", 99, buffer=True) stats_httpd = StatsHttpd() stats_httpd.start() except OptionValueError as ove: logger.fatal(STATSHTTPD_BAD_OPTION_VALUE, ove) sys.exit(1) except isc.cc.session.SessionError as se: logger.fatal(STATSHTTPD_CC_SESSION_ERROR, se) sys.exit(1) except HttpServerError as hse: logger.fatal(STATSHTTPD_START_SERVER_INIT_ERROR, hse) sys.exit(1) except KeyboardInterrupt as kie: logger.info(STATSHTTPD_STOPPED_BY_KEYBOARD) logger.info(STATSHTTPD_EXITING)