More work on the crawler to lay the ground for the updater cron-task

ffdnispdb/crawler.py

@@ -59,6 +59,9 @@ class Crawler(object):
 
         return u'\n'.join(r)
 
+    def pre_done_cb(self, *args):
+        pass
+
     def done_cb(self):
         pass
 
@@ -177,15 +180,9 @@ class Crawler(object):
         else:
             yield self.info('Done. No errors encountered \o')
 
-        # check name uniqueness
-        where = (ISP.name == jdict['name'])
-        if 'shortname' in jdict and jdict['shortname']:
-            where |= (ISP.shortname == jdict.get('shortname'))
-        if ISP.query.filter(where).count() > 0:
-            yield self.err('An ISP named "%s" already exist'%esc(
-                jdict['name']+(' ('+jdict['shortname']+')' if jdict.get('shortname') else '')
-            ))
-            yield self.abort('The name of your ISP must be unique')
+        ret=self.pre_done_cb(jdict)
+        if ret:
+            yield ret
             return
 
         yield (self.nl()+self.m('== '+self.color('forestgreen', 'All good ! You can click on Confirm now'))+
@@ -241,9 +238,24 @@ class PrettyValidator(Crawler):
         self.session.save()
 
 
+class WebValidator(PrettyValidator):
+    def pre_done_cb(self, jdict):
+        # check name uniqueness
+        where = (ISP.name == jdict['name'])
+        if 'shortname' in jdict and jdict['shortname']:
+            where |= (ISP.shortname == jdict.get('shortname'))
+        if ISP.query.filter(where).count() > 0:
+            ret = self.nl()
+            ret += self.err('An ISP named "%s" already exist in our database'%self.escape(
+                jdict['name']+(' ('+jdict['shortname']+')' if jdict.get('shortname') else '')
+            ))
+            ret += self.abort('The name of your ISP must be unique')
+            return ret
+
+
 class TextValidator(Crawler):
     def abort(self, msg):
-        res=u'ABORTED: %s\n'%msg
+        res=u'FATAL ERROR: %s\n'%msg
         pad=u'='*(len(res)-1)+'\n'
         return self.m(pad+res+pad)
 

ffdnispdb/forms.py

@@ -207,7 +207,7 @@ def is_url_unique(url):
     u1=urlparse.urlunsplit(t)
     t[0]='http' if t[0] == 'https' else 'https'
     u2=urlparse.urlunsplit(t)
-    if ISP.query.filter(ISP.url.startswith(u1) | ISP.url.startswith(u2)).count() > 0:
+    if ISP.query.filter(ISP.json_url.startswith(u1) | ISP.json_url.startswith(u2)).count() > 0:
         return False
     return True
 

ffdnispdb/views.py

@@ -18,7 +18,7 @@ from . import forms
 from .constants import *
 from . import app, db, cache
 from .models import ISP, ISPWhoosh
-from .crawler import PrettyValidator
+from .crawler import WebValidator
 
 
 @app.route('/')
@@ -118,7 +118,7 @@ def json_url_validator():
     else:
         session['form_json']['validator']=time()
 
-    validator=PrettyValidator(session=session._get_current_object())
+    validator=WebValidator(session=session._get_current_object())
     return Response(validator(session['form_json']['url']),
                     mimetype="text/event-stream")
 
@@ -145,7 +145,7 @@ def create_project_json_confirm():
         isp.name=jdict['name']
         if 'shortname' in jdict:
             isp.shortname=jdict['shortname']
-        isp.url=session['form_json']['url']
+        isp.json_url=session['form_json']['url']
         isp.json=jdict
         isp.tech_email=session['form_json']['tech_email']
         del session['form_json']