[mod] purify this code with autopep8

ffdnispdb/crawler.py

@@ -26,31 +26,31 @@ def get_encoding(content_type):
 
 class Crawler(object):
 
-    MAX_JSON_SIZE=1*1024*1024
+    MAX_JSON_SIZE = 1 * 1024 * 1024
 
-    escape=staticmethod(lambda x: unicode(str(x), 'utf8') if type(x) != unicode else x)
+    escape = staticmethod(lambda x: unicode(str(x), 'utf8') if type(x) != unicode else x)
 
     def __init__(self):
-        self.success=False
-        self.modified=True
-        self.jdict={}
-        self.cache_info=None
-        self.jdict_max_age=self.config('DEFAULT_CACHE_TIME')
+        self.success = False
+        self.modified = True
+        self.jdict = {}
+        self.cache_info = None
+        self.jdict_max_age = self.config('DEFAULT_CACHE_TIME')
 
     def m(self, msg, evt=None):
         if not evt:
-            return u'%s\n'%msg
+            return u'%s\n' % msg
         else:
             return u''
 
     def err(self, msg, *args):
-        return self.m(u'! %s'%msg, *args)
+        return self.m(u'! %s' % msg, *args)
 
     def warn(self, msg):
-        return self.m(u'@ %s'%msg)
+        return self.m(u'@ %s' % msg)
 
     def info(self, msg):
-        return self.m(u'\u2013 %s'%msg)
+        return self.m(u'\u2013 %s' % msg)
 
     def abort(self, msg):
         raise NotImplemented
@@ -68,11 +68,11 @@ class Crawler(object):
         return self.m('')
 
     def format_validation_errors(self, errs):
-        r=[]
+        r = []
         for e in errs:
-            r.append(u'    %s: %s'%('.'.join(list(e.schema_path)[1:]), e.message))
+            r.append(u'    %s: %s' % ('.'.join(list(e.schema_path)[1:]), e.message))
 
-        return u'\n'.join(r)+'\n'
+        return u'\n'.join(r) + '\n'
 
     def pre_done_cb(self, *args):
         pass
@@ -81,40 +81,40 @@ class Crawler(object):
         pass
 
     def config(self, name):
-        return current_app.config.get('CRAWLER_'+name)
+        return current_app.config.get('CRAWLER_' + name)
 
     def parse_cache_control(self, _cachectl):
-        cachectl={}
+        cachectl = {}
         for cc in _cachectl.split(','):
-            cc=cc.strip()
+            cc = cc.strip()
             if not cc:
                 continue
-            cc=cc.split('=')
+            cc = cc.split('=')
             if cc[0] not in ('max-age', 's-maxage'):
                 continue
             try:
-                cachectl[cc[0]]=cc[1]
+                cachectl[cc[0]] = cc[1]
             except IndexError:
-                cachectl[cc[0]]=True
+                cachectl[cc[0]] = True
         return cachectl
 
     def __call__(self, url, cache_info={}):
-        esc=self.escape
+        esc = self.escape
         yield self.m('Starting the validation process...')
-        r=None
+        r = None
         try:
-            yield self.m('* Attempting to retreive %s'%self.bold(url))
-            headers={'User-Agent': 'FFDN DB validator'}
+            yield self.m('* Attempting to retreive %s' % self.bold(url))
+            headers = {'User-Agent': 'FFDN DB validator'}
             if cache_info.get('etag'):
                 headers['If-None-Match'] = cache_info['etag']
             if cache_info.get('last-modified'):
                 headers['If-Modified-Since'] = cache_info['last-modified']
-            r=requests.get(url, verify='/etc/ssl/certs/ca-certificates.crt',
+            r = requests.get(url, verify='/etc/ssl/certs/ca-certificates.crt',
                            headers=headers, stream=True, timeout=10)
         except requests.exceptions.SSLError as e:
-            yield self.err('Unable to connect, SSL Error: '+self.color('#dd1144', esc(e)))
+            yield self.err('Unable to connect, SSL Error: ' + self.color('#dd1144', esc(e)))
         except requests.exceptions.ConnectionError as e:
-            yield self.err('Unable to connect: '+self.color('#dd1144', esc(e)))
+            yield self.err('Unable to connect: ' + self.color('#dd1144', esc(e)))
         except requests.exceptions.Timeout as e:
             yield self.err('Connection timeout')
         except requests.exceptions.TooManyRedirects as e:
@@ -134,7 +134,7 @@ class Crawler(object):
 
         yield self.info('Connection established')
 
-        yield self.info('Response code: '+self.bold(str(r.status_code)+' '+esc(r.reason)))
+        yield self.info('Response code: ' + self.bold(str(r.status_code) + ' ' + esc(r.reason)))
         try:
             r.raise_for_status()
         except requests.exceptions.HTTPError as e:
@@ -142,140 +142,137 @@ class Crawler(object):
             yield self.abort('Invalid response code')
             return
 
-        _cachecontrol=r.headers.get('cache-control')
-        cachecontrol=self.parse_cache_control(_cachecontrol) if _cachecontrol else None
-        max_age=None
+        _cachecontrol = r.headers.get('cache-control')
+        cachecontrol = self.parse_cache_control(_cachecontrol) if _cachecontrol else None
+        max_age = None
         if cachecontrol:
             try:
-                _maxage=cachecontrol.get('max-age')
-                _maxage=cachecontrol.get('s-maxage', _maxage) # s-maxage takes precedence
-                max_age=int(_maxage)
+                _maxage = cachecontrol.get('max-age')
+                _maxage = cachecontrol.get('s-maxage', _maxage)  # s-maxage takes precedence
+                max_age = int(_maxage)
             except ValueError:
-                yield self.warn('Invalid max-age '+esc(_maxage))
+                yield self.warn('Invalid max-age ' + esc(_maxage))
 
-            yield self.info('Cache control: '+self.bold(esc(
-                ', '.join([k+'='+v if type(v) != bool else k for k, v in cachecontrol.iteritems()]))
+            yield self.info('Cache control: ' + self.bold(esc(
+                ', '.join([k + '=' + v if type(v) != bool else k for k, v in cachecontrol.iteritems()]))
             ))
 
-        _expires=r.headers.get('expires')
-        expires=parse_date(_expires)
+        _expires = r.headers.get('expires')
+        expires = parse_date(_expires)
         if expires:
-            _now=r.headers.get('date')
-            if _now: # use server date when possible
-                now=parse_date(_now)
+            _now = r.headers.get('date')
+            if _now:  # use server date when possible
+                now = parse_date(_now)
             else:
-                now=datetime.utcnow()
+                now = datetime.utcnow()
 
             if expires > now:
-                expires=(expires-now).total_seconds()
-                yield self.info('Expires: '+self.bold(esc(_expires)))
+                expires = (expires - now).total_seconds()
+                yield self.info('Expires: ' + self.bold(esc(_expires)))
             else:
                 yield self.warn('Invalid Expires header. Expiry date must be in the future.')
-                expires=None
+                expires = None
         elif _expires and not expires:
-            yield self.warn('Invalid Expires header %r'%esc(_expires))
+            yield self.warn('Invalid Expires header %r' % esc(_expires))
 
         if not max_age and not expires:
             yield self.warn('No valid expiration time provided ! Please provide it either '
-                             'with a Cache-Control or Expires header.')
-            max_age=self.config('DEFAULT_CACHE_TIME')
-            yield self.info('Using default expiration time of %d seconds'%(max_age))
+                            'with a Cache-Control or Expires header.')
+            max_age = self.config('DEFAULT_CACHE_TIME')
+            yield self.info('Using default expiration time of %d seconds' % (max_age))
 
         self.jdict_max_age = max_age if max_age else expires
         self.jdict_max_age = min(
             self.config('MAX_CACHE_TIME'),
             max(self.config('MIN_CACHE_TIME'), self.jdict_max_age)
         )
-        yield self.info('Next update will be in %s'%(timedelta(seconds=self.jdict_max_age)))
+        yield self.info('Next update will be in %s' % (timedelta(seconds=self.jdict_max_age)))
 
-
-        etag=r.headers.get('etag')
-        last_modified=r.headers.get('last-modified')
+        etag = r.headers.get('etag')
+        last_modified = r.headers.get('last-modified')
         if not etag and not last_modified:
             yield self.warn('Please, provide at an ETag or Last-Modified header for '
                             'conditional requests')
 
-        self.cache_info={}
+        self.cache_info = {}
         if etag:
-            self.cache_info['etag']=etag
+            self.cache_info['etag'] = etag
         if last_modified:
-            self.cache_info['last-modified']=last_modified
+            self.cache_info['last-modified'] = last_modified
 
-        if cache_info and r.status_code == 304: # not modified
-            self.m('== '+self.color('forestgreen', 'Response not modified. All good !'))
-            self.modified=False
-            self.success=True
+        if cache_info and r.status_code == 304:  # not modified
+            self.m('== ' + self.color('forestgreen', 'Response not modified. All good !'))
+            self.modified = False
+            self.success = True
             self.done_cb()
             return
 
-
-        yield self.info('Content type: '+self.bold(esc(r.headers.get('content-type', 'not defined'))))
+        yield self.info('Content type: ' + self.bold(esc(r.headers.get('content-type', 'not defined'))))
         if not r.headers.get('content-type'):
-            yield self.err('Content-type '+self.bold('MUST')+' be defined')
+            yield self.err('Content-type ' + self.bold('MUST') + ' be defined')
             yield self.abort('The file must have a proper content-type to continue')
             return
         elif r.headers.get('content-type').lower() != 'application/json':
-            yield self.warn('Content-type '+self.italics('SHOULD')+' be application/json')
+            yield self.warn('Content-type ' + self.italics('SHOULD') + ' be application/json')
 
-        encoding=get_encoding(r.headers.get('content-type'))
+        encoding = get_encoding(r.headers.get('content-type'))
         if not encoding:
             yield self.warn('Encoding not set. Assuming it\'s unicode, as per RFC4627 section 3')
 
-        yield self.info('Content length: %s'%(self.bold(esc(r.headers.get('content-length', 'not set')))))
+        yield self.info('Content length: %s' % (self.bold(esc(r.headers.get('content-length', 'not set')))))
 
-        cl=r.headers.get('content-length')
+        cl = r.headers.get('content-length')
         if not cl:
             yield self.warn('No content-length. Note that we will not process a file whose size exceed %s'
-                                                                      % (filesize_fmt(self.MAX_JSON_SIZE)))
+                            % (filesize_fmt(self.MAX_JSON_SIZE)))
         elif int(cl) > self.MAX_JSON_SIZE:
             yield self.abort('File too big ! File size must be less then %s' % (filesize_fmt(self.MAX_JSON_SIZE)))
             return
 
-
         yield self.info('Reading response into memory...')
-        b=io.BytesIO()
+        b = io.BytesIO()
         for d in r.iter_content(requests.models.CONTENT_CHUNK_SIZE):
             b.write(d)
             if b.tell() > self.MAX_JSON_SIZE:
                 yield self.abort('File too big ! File size must be less then %s'
-                                            % (filesize_fmt(self.MAX_JSON_SIZE)))
+                                 % (filesize_fmt(self.MAX_JSON_SIZE)))
                 return
-        r._content=b.getvalue()
+        r._content = b.getvalue()
         del b
-        yield self.info('Successfully read %d bytes'%len(r.content))
+        yield self.info('Successfully read %d bytes' % len(r.content))
 
-        yield self.nl()+self.m('* Parsing the JSON file')
+        yield self.nl() + self.m('* Parsing the JSON file')
         if not encoding:
-            charset=requests.utils.guess_json_utf(r.content)
+            charset = requests.utils.guess_json_utf(r.content)
             if not charset:
                 yield self.err('Unable to guess unicode charset')
                 yield self.abort('The file MUST be unicode-encoded when no explicit charset is in the content-type')
                 return
 
-            yield self.info('Guessed charset: '+self.bold(charset))
+            yield self.info('Guessed charset: ' + self.bold(charset))
 
         try:
-            txt=r.content.decode(encoding or charset)
-            yield self.info('Successfully decoded file as %s'%esc(encoding or charset))
+            txt = r.content.decode(encoding or charset)
+            yield self.info('Successfully decoded file as %s' % esc(encoding or charset))
         except LookupError as e:
-            yield self.err('Invalid/unknown charset: %s'%esc(e))
+            yield self.err('Invalid/unknown charset: %s' % esc(e))
             yield self.abort('Charset error, Cannot continue')
             return
         except UnicodeDecodeError as e:
-            yield self.err('Unicode decode error: %s'%e)
+            yield self.err('Unicode decode error: %s' % e)
             yield self.abort('Charset error, cannot continue')
             return
         except Exception:
             yield self.abort('Unexpected charset error')
             return
 
-        jdict=None
+        jdict = None
         try:
-            jdict=json.loads(txt)
+            jdict = json.loads(txt)
         except ValueError as e:
-            yield self.err('Error while parsing JSON: %s'%esc(e))
+            yield self.err('Error while parsing JSON: %s' % esc(e))
         except Exception as e:
-            yield self.err('Unexpected error while parsing JSON: %s'%esc(e))
+            yield self.err('Unexpected error while parsing JSON: %s' % esc(e))
 
         if not jdict:
             yield self.abort('Could not parse JSON')
@@ -283,11 +280,11 @@ class Crawler(object):
 
         yield self.info('JSON parsed successfully')
 
-        yield self.nl()+self.m('* Validating the JSON against the schema')
+        yield self.nl() + self.m('* Validating the JSON against the schema')
 
-        v=list(validate_isp(jdict))
+        v = list(validate_isp(jdict))
         if v:
-            yield self.err('Validation errors:')+self.format_validation_errors(v)
+            yield self.err('Validation errors:') + self.format_validation_errors(v)
             yield self.abort('Your JSON file does not follow the schema, please fix it')
             return
         else:
@@ -298,74 +295,74 @@ class Crawler(object):
                 continue
             if not check_geojson_spatialite(ca['area']):
                 yield self.err('GeoJSON data for covered area "%s" cannot '
-                               'be handled by our database'%esc(ca['name']))
+                               'be handled by our database' % esc(ca['name']))
                 yield self.abort('Please fix your GeoJSON')
                 return
 
-        ret=self.pre_done_cb(jdict)
+        ret = self.pre_done_cb(jdict)
         if ret:
             yield ret
             return
 
-        yield (self.nl()+self.m('== '+self.color('forestgreen', 'All good ! You can click on Confirm now'))+
+        yield (self.nl() + self.m('== ' + self.color('forestgreen', 'All good ! You can click on Confirm now')) +
                self.m(json.dumps({'passed': 1}), 'control'))
 
-        self.jdict=jdict
-        self.success=True
+        self.jdict = jdict
+        self.success = True
         self.done_cb()
 
 
-
 class PrettyValidator(Crawler):
 
     def __init__(self, session=None, sesskey=None, *args, **kwargs):
         super(PrettyValidator, self).__init__(*args, **kwargs)
-        self.session=session
-        self.sesskey=sesskey
-        self.escape=lambda x: escape(unicode(str(x), 'utf8') if type(x) != unicode else x)
+        self.session = session
+        self.sesskey = sesskey
+        self.escape = lambda x: escape(unicode(str(x), 'utf8') if type(x) != unicode else x)
 
     def m(self, msg, evt=None):
-        return u'%sdata: %s\n\n'%(u'event: %s\n'%evt if evt else '', msg)
+        return u'%sdata: %s\n\n' % (u'event: %s\n' % evt if evt else '', msg)
 
     def err(self, msg, *args):
-        return self.m(u'<strong style="color: crimson">!</strong> %s'%msg, *args)
+        return self.m(u'<strong style="color: crimson">!</strong> %s' % msg, *args)
 
     def warn(self, msg):
-        return self.m(u'<strong style="color: dodgerblue">@</strong> %s'%msg)
+        return self.m(u'<strong style="color: dodgerblue">@</strong> %s' % msg)
 
     def info(self, msg):
-        return self.m(u'&ndash; %s'%msg)
+        return self.m(u'&ndash; %s' % msg)
 
     def abort(self, msg):
-        return (self.m(u'<br />== <span style="color: crimson">%s</span>'%msg)+
+        return (self.m(u'<br />== <span style="color: crimson">%s</span>' % msg) +
                 self.m(json.dumps({'closed': 1}), 'control'))
 
     def bold(self, msg):
-        return u'<strong>%s</strong>'%msg
+        return u'<strong>%s</strong>' % msg
 
     def italics(self, msg):
-        return u'<em>%s</em>'%msg
+        return u'<em>%s</em>' % msg
 
     def color(self, color, msg):
-        return u'<span style="color: %s">%s</span>'%(color, msg)
+        return u'<span style="color: %s">%s</span>' % (color, msg)
 
     def format_validation_errors(self, errs):
-        lns=super(PrettyValidator, self).format_validation_errors(errs)
-        buf=u''
+        lns = super(PrettyValidator, self).format_validation_errors(errs)
+        buf = u''
         for l in lns.split('\n'):
-            buf+=self.m(self.escape(l))
+            buf += self.m(self.escape(l))
         return buf
 
     def done_cb(self):
-        self.session[self.sesskey]['validated']=True
-        self.session[self.sesskey]['jdict']=self.jdict
-        self.session[self.sesskey]['cache_info']=self.cache_info
-        self.session[self.sesskey]['last_update']=utcnow()
-        self.session[self.sesskey]['next_update']=utcnow()+timedelta(seconds=self.jdict_max_age)
+        self.session[self.sesskey]['validated'] = True
+        self.session[self.sesskey]['jdict'] = self.jdict
+        self.session[self.sesskey]['cache_info'] = self.cache_info
+        self.session[self.sesskey]['last_update'] = utcnow()
+        self.session[self.sesskey]['next_update'] = utcnow() + timedelta(seconds=self.jdict_max_age)
         self.session.save()
 
 
 class WebValidator(PrettyValidator):
+
     def pre_done_cb(self, jdict):
         # check name uniqueness
         where = (ISP.name == jdict['name'])
@@ -373,16 +370,16 @@ class WebValidator(PrettyValidator):
             where |= (ISP.shortname == jdict.get('shortname'))
         if ISP.query.filter(where).count() > 0:
             ret = self.nl()
-            ret += self.err('An ISP named "%s" already exist in our database'%self.escape(
-                jdict['name']+(' ('+jdict['shortname']+')' if jdict.get('shortname') else '')
+            ret += self.err('An ISP named "%s" already exist in our database' % self.escape(
+                jdict['name'] + (' (' + jdict['shortname'] + ')' if jdict.get('shortname') else '')
             ))
             ret += self.abort('The name of your ISP must be unique')
             return ret
 
 
 class TextValidator(Crawler):
-    def abort(self, msg):
-        res=u'FATAL ERROR: %s\n'%msg
-        pad=u'='*(len(res)-1)+'\n'
-        return self.m(pad+res+pad)
 
+    def abort(self, msg):
+        res = u'FATAL ERROR: %s\n' % msg
+        pad = u'=' * (len(res) - 1) + '\n'
+        return self.m(pad + res + pad)

ffdnispdb/cron_task.py

@@ -13,27 +13,31 @@ from ffdnispdb.models import ISP
 from ffdnispdb import create_app, db, mail, utils
 
 
-app=create_app({
+app = create_app({
     'SERVER_NAME': 'db.ffdn.org',
 })
 
-MAX_RUNTIME=15*60
+MAX_RUNTIME = 15 * 60
+
 
 class Timeout(Exception):
     pass
 
+
 class ScriptTimeout(Exception):
     """
     Script exceeded its allowed run time
     """
 
 
-strike=1
-last_isp=-1
-script_begin=datetime.now()
+strike = 1
+last_isp = -1
+script_begin = datetime.now()
+
+
 def timeout_handler(signum, frame):
     global last_isp, strike
-    if script_begin < datetime.now()-timedelta(seconds=MAX_RUNTIME):
+    if script_begin < datetime.now() - timedelta(seconds=MAX_RUNTIME):
         raise ScriptTimeout
 
     if last_isp == isp.id:
@@ -52,7 +56,7 @@ signal.alarm(6)
 
 
 def gen_reactivate_key(isp):
-    s=itsdangerous.URLSafeSerializer(app.secret_key,
+    s = itsdangerous.URLSafeSerializer(app.secret_key,
                                      salt='reactivate')
     return s.dumps([
         isp.id,
@@ -61,7 +65,7 @@ def gen_reactivate_key(isp):
 
 
 def send_warning_email(isp, debug_msg):
-    msg=Message(u"Problem while updating your ISP's data", sender=app.config['EMAIL_SENDER'])
+    msg = Message(u"Problem while updating your ISP's data", sender=app.config['EMAIL_SENDER'])
     msg.body = """
 Hello,
 
@@ -82,10 +86,10 @@ When the issue is resolved, please click on the link below to reactivate automat
 Thanks,
 The FFDN ISP Database team
 https://db.ffdn.org
-    """.strip()%(isp.complete_name, isp.json_url, debug_msg.strip(),
+    """.strip() % (isp.complete_name, isp.json_url, debug_msg.strip(),
                  url_for('ispdb.reactivate_isp', projectid=isp.id), gen_reactivate_key(isp))
     msg.add_recipient(isp.tech_email)
-    print u'    Sending notification email to %s'%(isp.tech_email)
+    print u'    Sending notification email to %s' % (isp.tech_email)
     mail.send(msg)
 
 
@@ -96,19 +100,19 @@ try:
                                 ISP.json_url != None,
                                 ISP.next_update < utils.utcnow(),
                                 ISP.update_error_strike < 3)\
-                        .order_by(ISP.last_update_success):
+            .order_by(ISP.last_update_success):
         try:
-            print u'%s: Attempting to update %s'%(datetime.now(), isp)
-            print u'    last successful update=%s'%(utils.tosystemtz(isp.last_update_success))
-            print u'    last update attempt=%s'%(utils.tosystemtz(isp.last_update_attempt))
-            print u'    next update was scheduled %s ago'%(utils.utcnow()-isp.next_update)
-            print u'    strike=%d'%(isp.update_error_strike)
+            print u'%s: Attempting to update %s' % (datetime.now(), isp)
+            print u'    last successful update=%s' % (utils.tosystemtz(isp.last_update_success))
+            print u'    last update attempt=%s' % (utils.tosystemtz(isp.last_update_attempt))
+            print u'    next update was scheduled %s ago' % (utils.utcnow() - isp.next_update)
+            print u'    strike=%d' % (isp.update_error_strike)
 
-            isp.last_update_attempt=utils.utcnow()
+            isp.last_update_attempt = utils.utcnow()
             db.session.add(isp)
             db.session.commit()
 
-            validator=TextValidator()
+            validator = TextValidator()
             log = ''
             exc, exc_trace = None, None
             try:
@@ -118,19 +122,19 @@ try:
                 exc = e
                 exc_trace = traceback.format_exc()
 
-            if not validator.success: # handle error
+            if not validator.success:  # handle error
                 isp.update_error_strike += 1
                 # reset cache info (to force refetch next time)
                 isp.cache_info = {}
-                isp.next_update = utils.utcnow()+timedelta(seconds=validator.jdict_max_age)
+                isp.next_update = utils.utcnow() + timedelta(seconds=validator.jdict_max_age)
                 db.session.add(isp)
                 db.session.commit()
-                print u'%s: Error while updating:'%(datetime.now())
+                print u'%s: Error while updating:' % (datetime.now())
                 if isp.update_error_strike >= 3:
                     print u'    three strikes, you\'re out'
                     send_warning_email(isp, log)
 
-                print log.rstrip().encode('utf-8')+'\n'
+                print log.rstrip().encode('utf-8') + '\n'
                 if exc:
                     print u'Unexpected exception in the validator: %r' % exc
                     print exc_trace
@@ -142,15 +146,15 @@ try:
             isp.cache_info = validator.cache_info
             isp.last_update_success = isp.last_update_attempt
             isp.update_error_strike = 0
-            isp.next_update = utils.utcnow()+timedelta(seconds=validator.jdict_max_age)
+            isp.next_update = utils.utcnow() + timedelta(seconds=validator.jdict_max_age)
             db.session.add(isp)
             db.session.commit()
 
-            print u'%s: Update successful !'%(datetime.now())
-            print u'    next update is scheduled for %s\n'%(isp.next_update)
+            print u'%s: Update successful !' % (datetime.now())
+            print u'    next update is scheduled for %s\n' % (isp.next_update)
         except Timeout:
-            print u'%s: Timeout while updating:'%(datetime.now())
-            isp=ISP.query.get(isp.id)
+            print u'%s: Timeout while updating:' % (datetime.now())
+            isp = ISP.query.get(isp.id)
             isp.update_error_strike += 1
             db.session.add(isp)
             db.session.commit()

ffdnispdb/default_settings.py

@@ -1,15 +1,15 @@
 # -*- coding: utf-8 -*-
 
 SQLALCHEMY_DATABASE_URI = 'sqlite:///../ffdn-db.sqlite'
-CRAWLER_MIN_CACHE_TIME = 60*60 # 1 hour
-CRAWLER_MAX_CACHE_TIME = 60*60*24*14 # 2 week
-CRAWLER_DEFAULT_CACHE_TIME = 60*60*12 # 12 hours
-CRAWLER_CRON_INTERVAL = 60*20 # used to return valid cache info in the API
+CRAWLER_MIN_CACHE_TIME = 60 * 60  # 1 hour
+CRAWLER_MAX_CACHE_TIME = 60 * 60 * 24 * 14  # 2 week
+CRAWLER_DEFAULT_CACHE_TIME = 60 * 60 * 12  # 12 hours
+CRAWLER_CRON_INTERVAL = 60 * 20  # used to return valid cache info in the API
 SYSTEM_TIME_ZONE = 'Europe/Paris'
 LANGUAGES = {
     'en': u'English',
     'fr': u'Français',
 }
-ISP_FORM_GEOJSON_MAX_SIZE = 256*1024
-ISP_FORM_GEOJSON_MAX_SIZE_TOTAL = 1024*1024
+ISP_FORM_GEOJSON_MAX_SIZE = 256 * 1024
+ISP_FORM_GEOJSON_MAX_SIZE_TOTAL = 1024 * 1024
 CACHE_NO_NULL_WARNING = True

ffdnispdb/forms.py

@@ -21,6 +21,7 @@ from .utils import check_geojson_spatialite, filesize_fmt
 
 
 class InputListWidget(ListWidget):
+
     def __call__(self, field, **kwargs):
         kwargs.setdefault('id', field.id)
         html = ['<%s %s>' % (self.html_tag, html_params(**kwargs))]
@@ -83,6 +84,7 @@ class GeoJSONField(TextField):
 
 class Unique(object):
     """ validator that checks field uniqueness """
+
     def __init__(self, model, field, message=None, allow_edit=False):
         self.model = model
         self.field = field
@@ -261,6 +263,7 @@ class ProjectForm(Form):
 
 
 class URLField(TextField):
+
     def _value(self):
         if isinstance(self.data, basestring):
             return self.data

ffdnispdb/models.py

@@ -19,11 +19,14 @@ from whoosh import fields, index, qparser
 
 
 class fakefloat(float):
+
     def __init__(self, value):
         self._value = value
+
     def __repr__(self):
         return str(self._value)
 
+
 def defaultencode(o):
     if isinstance(o, Decimal):
         # Subclass float with custom repr?
@@ -68,12 +71,12 @@ class ISP(db.Model):
     name = db.Column(db.String, nullable=False, index=True, unique=True)
     shortname = db.Column(db.String(12), index=True, unique=True)
     is_ffdn_member = db.Column(db.Boolean, default=False)
-    is_disabled = db.Column(db.Boolean, default=False) # True = ISP will not appear
+    is_disabled = db.Column(db.Boolean, default=False)  # True = ISP will not appear
     json_url = db.Column(db.String)
     date_added = db.Column(UTCDateTime, default=utcnow)
     last_update_success = db.Column(UTCDateTime)
     last_update_attempt = db.Column(UTCDateTime)
-    update_error_strike = db.Column(db.Integer, default=0) # if >= 3; then updates are disabled
+    update_error_strike = db.Column(db.Integer, default=0)  # if >= 3; then updates are disabled
     next_update = db.Column(UTCDateTime, default=utcnow)
     tech_email = db.Column(db.String)
     cache_info = db.Column(MutableDict.as_mutable(JSONEncodedDict))
@@ -120,7 +123,6 @@ class ISP(db.Model):
                 point=db.func.MakePoint(coords['longitude'], coords['latitude'], 4326)
             )
 
-
     def covered_areas_names(self):
         return [c['name'] for c in self.json.get('coveredAreas', [])]
 
@@ -131,9 +133,9 @@ class ISP(db.Model):
     @property
     def complete_name(self):
         if 'shortname' in self.json:
-            return u'%s (%s)'%(self.json['shortname'], self.json['name'])
+            return u'%s (%s)' % (self.json['shortname'], self.json['name'])
         else:
-            return u'%s'%self.json['name']
+            return u'%s' % self.json['name']
 
     @staticmethod
     def str2date(_str):
@@ -186,6 +188,7 @@ class RegisteredOffice(db.Model):
 
 geo.GeometryDDL(RegisteredOffice.__table__)
 
+
 @event.listens_for(db.metadata, 'before_create')
 def init_spatialite_metadata(target, conn, **kwargs):
     conn.execute('SELECT InitSpatialMetaData(1)')
@@ -212,7 +215,7 @@ class ISPWhoosh(object):
         step=fields.NUMERIC(signed=False),
     )
 
-    primary_key=schema._fields['id']
+    primary_key = schema._fields['id']
 
     @staticmethod
     def get_index_dir():
@@ -234,7 +237,7 @@ class ISPWhoosh(object):
         return s.search(qparser.MultifieldParser([
             'name', 'shortname', 'description', 'covered_areas'
         ], schema=cls.schema).parse(terms),
-           mask=whoosh.query.Term('is_disabled', True))
+            mask=whoosh.query.Term('is_disabled', True))
 
     @classmethod
     def search(cls, terms):
@@ -288,4 +291,3 @@ class ISPWhoosh(object):
 
 flask_sqlalchemy.models_committed.connect(ISPWhoosh._after_flush)
 event.listen(flask_sqlalchemy.Session, 'before_commit', pre_save_hook)
-

ffdnispdb/sessions.py

@@ -4,7 +4,7 @@ from flask.sessions import SessionInterface, SessionMixin
 from werkzeug.datastructures import CallbackDict
 
 from sqlalchemy import Table, Column, String, LargeBinary, DateTime,\
-                       select, delete, insert, update
+    select
 
 from random import SystemRandom, randrange
 import string
@@ -22,6 +22,7 @@ class SQLSession(CallbackDict, SessionMixin):
         self.table = table
         self.modified = False
         self.new = new
+
         def _on_update(self):
             self.modified = True
         super(SQLSession, self).__init__(initial, _on_update)
@@ -30,7 +31,7 @@ class SQLSession(CallbackDict, SessionMixin):
         if self.new:
             self.db.execute(self.table.insert({
                 'session_id': self.sid,
-                'expire': datetime.utcnow()+timedelta(hours=1),
+                'expire': datetime.utcnow() + timedelta(hours=1),
                 'value': cPickle.dumps(dict(self), -1)
             }))
             self.new = False
@@ -38,13 +39,14 @@ class SQLSession(CallbackDict, SessionMixin):
             self.db.execute(self.table.update(
                 self.table.c.session_id == self.sid,
                 {
-                    'expire': datetime.utcnow()+timedelta(hours=1),
+                    'expire': datetime.utcnow() + timedelta(hours=1),
                     'value': cPickle.dumps(dict(self), -1)
                 }
             ))
 
 
 class MySessionInterface(SessionInterface):
+
     def __init__(self, db):
         self.db = db
 
@@ -58,12 +60,12 @@ class MySessionInterface(SessionInterface):
         sid = request.cookies.get(app.session_cookie_name)
         if sid:
             res = self.db.engine.execute(select([self.table.c.value], (self.table.c.session_id == sid) &
-                                                                 (self.table.c.expire > datetime.utcnow()))).first()
+                                                (self.table.c.expire > datetime.utcnow()))).first()
             if res:
                 return SQLSession(sid, self.db.engine, self.table, False, cPickle.loads(res[0]))
 
         while True:
-            sid = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(32))
+            sid = ''.join(random.choice(string.ascii_letters + string.digits) for i in range(32))
             res = self.db.engine.execute(select([self.table.c.value], self.table.c.session_id == sid)).first()
             if not res:
                 break
@@ -72,7 +74,7 @@ class MySessionInterface(SessionInterface):
 
     def save_session(self, app, session, response):
         if not session and not session.modified:
-            return # empty/unused session
+            return  # empty/unused session
 
         if session.modified:
             session.save()

ffdnispdb/utils.py

@@ -11,25 +11,24 @@ import sys
 from . import db
 
 
-
 def dict_to_geojson(d_in):
     """
     Encode a dict representing a GeoJSON object into a JSON string.
     This is needed because spatialite's GeoJSON parser is not really
     JSON-compliant and it fails when keys are not in the right order.
     """
-    d=OrderedDict()
-    d['type']=d_in['type']
+    d = OrderedDict()
+    d['type'] = d_in['type']
 
     if 'crs' in d_in:
-        d['crs']=d_in['crs']
+        d['crs'] = d_in['crs']
     # our spatialite geo column is defined with EPSG SRID 4326 (WGS 84)
     d['crs'] = {'type': 'name', 'properties': {'name': 'urn:ogc:def:crs:EPSG:4326'}}
 
     if 'bbox' in d_in:
-        d['bbox']=d_in['bbox']
+        d['bbox'] = d_in['bbox']
 
-    d['coordinates']=d_in['coordinates']
+    d['coordinates'] = d_in['coordinates']
 
     return json.dumps(d)
 
@@ -45,7 +44,7 @@ def check_geojson_spatialite(_gjson):
     ... ]})
     True
     """
-    gjson=dict_to_geojson(_gjson)
+    gjson = dict_to_geojson(_gjson)
     return bool(db.session.query(db.func.GeomFromGeoJSON(gjson) != None).first()[0])
 
 

ffdnispdb/views.py

@@ -82,7 +82,7 @@ def isp_find_near():
         'area': {
             'id': ca.id,
             'name': ca.name,
-            }
+        }
     } for ca in q]]
 
     dst = RegisteredOffice.point.distance(db.func.MakePoint(lon, lat), 1).label('distance')

ffdnispdb/views_api.py

@@ -68,6 +68,7 @@ class REST(object):
 
 
 class RESTException(Exception):
+
     def __init__(self, status_code, msg, error_type=None):
         super(RESTException, self).__init__()
         self.status_code = status_code
@@ -87,6 +88,7 @@ class RESTException(Exception):
 
 
 class RESTSimpleError(RESTException):
+
     def __init__(self):
         pass
 
@@ -114,7 +116,7 @@ class Resource(MethodView, REST):
         if isinstance(resp, Response):
             return resp
 
-        data, code, headers = (None,)*3
+        data, code, headers = (None,) * 3
         if isinstance(resp, tuple):
             data, code, headers = resp + (None,) * (3 - len(resp))
         data = resp if data is None else data
@@ -163,7 +165,7 @@ class Resource(MethodView, REST):
             res['per_page'] = pgn.per_page
 
         if out_var is None:
-            out_var = query.column_descriptions[0]['name'].lower()+'s'
+            out_var = query.column_descriptions[0]['name'].lower() + 's'
         res[out_var] = items
         return res
 
@@ -176,6 +178,7 @@ class ISPResource(Resource):
     /isp/<int:isp_id>/
         GET - return ISP with the given id
     """
+
     def isp_to_dict(self, isp):
         r = OrderedDict()
         r['id'] = isp.id
@@ -208,7 +211,7 @@ class ISPResource(Resource):
             else:
                 # default max-age for isp using the form
                 max_age = current_app.config['CRAWLER_MIN_CACHE_TIME']
-            headers = {'Cache-Control': 'max-age='+str(max_age)}
+            headers = {'Cache-Control': 'max-age=' + str(max_age)}
             return self.isp_to_dict(s), 200, headers
         else:
             s = ISP.query.filter_by(is_disabled=False)
@@ -226,6 +229,7 @@ class CoveredAreaResource(Resource):
     /isp/<int:isp_id>/covered_area/
         GET - return covered areas for the given ISP
     """
+
     def ca_to_dict(self, ca):
         r = OrderedDict()
         r['id'] = ca.id