# -*- coding: utf-8 -*- from flask import Blueprint, make_response, request, Response, current_app from flask.views import MethodView from collections import OrderedDict import sys import json import datetime import time from . import utils, db from .models import ISP, CoveredArea ispdbapi = Blueprint('ispdbapi', __name__) def output_json(data, code, headers=None): """Makes a Flask response with a JSON encoded body""" def encode(obj): if isinstance(obj, datetime.datetime): return obj.isoformat() if hasattr(obj, '__json__'): return obj.__json__() indent = 4 if not request.is_xhr else None dumped = json.dumps(data, indent=indent, default=encode) dumped += '\n' resp = make_response(dumped, code) resp.headers.extend(headers or {}) return resp class REST(object): DEFAULT_MIMETYPE = 'application/json' OUTPUT_MIMETYPES = { 'application/json': output_json } @classmethod def accepted_mimetypes(cls, default_mime=DEFAULT_MIMETYPE): am = [m for m, q in request.accept_mimetypes] if default_mime: am += [default_mime] return am @classmethod def match_mimetype(cls): for accepted_mime in cls.accepted_mimetypes(): if accepted_mime in cls.OUTPUT_MIMETYPES: return accepted_mime, cls.OUTPUT_MIMETYPES[accepted_mime] @classmethod def negociated_resp(cls, data, code, headers={}): output_mime, output_func = cls.match_mimetype() resp = output_func(data, code, headers) resp.headers['Content-Type'] = output_mime return resp @classmethod def marsh_error(cls, error): return cls.negociated_resp({ 'error': dict(error) }, error.status_code) class RESTException(Exception): def __init__(self, status_code, msg, error_type=None): super(RESTException, self).__init__() self.status_code = status_code self.message = msg self.error_type = error_type def __iter__(self): return { 'error_type': self.error_type, 'message': self.message }.iteritems() def __json__(self): return { 'error': dict(self) } class RESTSimpleError(RESTException): def __init__(self): pass class ObjectNotFound(RESTSimpleError): status_code = 404 message = 'Object not found' error_type = 'ispdb.api.ObjectNotFound' class InternalError(RESTSimpleError): status_code = 500 message = 'There was an error while processing your request' error_type = 'ispdb.api.InternalError' class Resource(MethodView, REST): def __init__(self, *args, **kwargs): super(Resource, self).__init__(*args, **kwargs) def dispatch_request(self, *args, **kwargs): meth = getattr(self, request.method.lower(), None) if not meth: return self.negociated_resp(None, 405, None) # 405 Method not allowed resp = meth(*args, **kwargs) if isinstance(resp, Response): return resp data, code, headers = (None,) * 3 if isinstance(resp, tuple): data, code, headers = resp + (None,) * (3 - len(resp)) data = resp if data is None else data code = 200 if code is None else code headers = {} if headers is None else headers resp = self.negociated_resp(data, code, headers) return resp def get_range(self): range_ = request.args.get('range') if not range_: return None try: range_ = map(int, filter(None, range_.split(',', 1))) return range_ except ValueError: return None def apply_range(self, query, range_): return query.slice(*range_) if len(range_) > 1 else query.offset(range_[0]) def handle_list(self, query, cb, paginate=10, out_var=None): res = OrderedDict() res['total_items'] = query.count() range_ = self.get_range() if range_: query = self.apply_range(query, range_) items = [cb(o) for o in query] res['range'] = ','.join(map(str, range_)) elif paginate: page = request.args.get('page', 1) per_page = request.args.get('per_page', paginate) try: page = int(page) except ValueError: page = 1 try: per_page = int(per_page) except ValueError: per_page = paginate pgn = query.paginate(page, per_page=per_page, error_out=False) items = [cb(o) for o in pgn.items] res['page'] = pgn.page res['num_pages'] = pgn.pages res['per_page'] = pgn.per_page if out_var is None: out_var = query.column_descriptions[0]['name'].lower() + 's' res[out_var] = items return res class ISPResource(Resource): """ /isp/ GET - list all ISPs /isp// GET - return ISP with the given id """ def isp_to_dict(self, isp): r = OrderedDict() r['id'] = isp.id r['is_ffdn_member'] = isp.is_ffdn_member r['json_url'] = isp.json_url r['date_added'] = utils.tosystemtz(isp.date_added) if isp.last_update_success: r['last_update'] = utils.tosystemtz(isp.last_update_success) else: r['last_update'] = None r['ispformat'] = isp.json return r def get(self, isp_id=None): if isp_id is not None: s = ISP.query.filter_by(id=isp_id, is_disabled=False).scalar() if not s: raise ObjectNotFound if s.json_url: # compute next update time, based on crawler cron interval. interval = current_app.config['CRAWLER_CRON_INTERVAL'] nupdt = time.mktime(s.next_update.timetuple()) nupdt = datetime.datetime.fromtimestamp(nupdt - (nupdt % interval) + interval + 60) now = datetime.datetime.utcnow() if nupdt > now: max_age = (nupdt - now).seconds else: max_age = current_app.config['CRAWLER_MIN_CACHE_TIME'] else: # default max-age for isp using the form max_age = current_app.config['CRAWLER_MIN_CACHE_TIME'] headers = {'Cache-Control': 'max-age=' + str(max_age)} return self.isp_to_dict(s), 200, headers else: s = ISP.query.filter_by(is_disabled=False) return self.handle_list(s, self.isp_to_dict) class CoveredAreaResource(Resource): """ /covered_area/ GET - list all covered areas /covered_area// GET - return covered area with the given id /isp//covered_area/ GET - return covered areas for the given ISP """ def ca_to_dict(self, ca): r = OrderedDict() r['id'] = ca.id if not self.isp_id: r['isp'] = OrderedDict() r['isp']['id'] = ca.isp_id r['isp']['name'] = ca.isp.name r['isp']['shortname'] = ca.isp.shortname r['name'] = ca.name r['geojson'] = json.loads(ca.area_geojson) if ca.area_geojson else None return r def get(self, area_id=None, isp_id=None): self.area_id = area_id self.isp_id = isp_id if area_id is not None: raise ObjectNotFound s = CoveredArea.query.get_or_404(area_id) return self.ca_to_dict(s) else: s = CoveredArea.query.filter(ISP.is_disabled == False)\ .options(db.joinedload('isp'), db.defer('isp.json'), db.defer('area'), db.undefer('area_geojson')) if isp_id: if not ISP.query.filter_by(id=isp_id, is_disabled=False).scalar(): raise ObjectNotFound s = s.filter(CoveredArea.isp_id == isp_id) return self.handle_list(s, self.ca_to_dict, out_var='covered_areas') @ispdbapi.route('/') def path_not_found(notfound): "catch all" return REST.marsh_error(RESTException(404, 'path not found', 'ispdb.api.PathNotFound')) @ispdbapi.errorhandler(404) def resource_not_found(e): return REST.marsh_error(RESTException(404, 'not found')) @ispdbapi.errorhandler(RESTException) def handle_rest_error(e): return REST.marsh_error(e) @ispdbapi.errorhandler(Exception) def handle_generic_exception(e): "Return a REST-formated error response instead of the standard 500 html template" current_app.log_exception(sys.exc_info()) return REST.marsh_error(InternalError()) isp_view = ISPResource.as_view('isp_api') ispdbapi.add_url_rule('/v1/isp/', defaults={'isp_id': None}, view_func=isp_view, methods=['GET']) ispdbapi.add_url_rule('/v1/isp//', view_func=isp_view, methods=['GET']) @ispdbapi.route('/v1/isp/export_urls/') @ispdbapi.route('/v1/isp/all_your_urls_are_belong_to_us/') def all_urls(): """ This resource allows to simply export all ISP-format URLs in our DB without pulling all ISP data. """ isps = db.session.query(ISP.id, ISP.json_url).filter(ISP.json_url != None) return REST.negociated_resp({ 'isps': [{'id': isp.id, 'json_url': isp.json_url} for isp in isps] }, 200) ca_view = CoveredAreaResource.as_view('covered_area_api') ispdbapi.add_url_rule('/v1/covered_area/', defaults={'area_id': None}, view_func=ca_view, methods=['GET']) ispdbapi.add_url_rule('/v1/covered_area//', view_func=ca_view, methods=['GET']) ispdbapi.add_url_rule('/v1/isp//covered_areas/', view_func=ca_view, methods=['GET'])