Add a view to allow to reactivate updates an ISP after update errors

TODO

@@ -2,8 +2,8 @@
 
 - Crawler:
 * i18n
-* http cache support
-* write a "cron task" to periodically update the data
+X http cache support
+X write a "cron task" to periodically update the data
 
 - Manual ISP insertion form:
 * date picker

ffdnispdb/crawler.py

@@ -299,9 +299,10 @@ class Crawler(object):
 
 class PrettyValidator(Crawler):
 
-    def __init__(self, session=None, *args, **kwargs):
+    def __init__(self, session=None, sesskey=None, *args, **kwargs):
         super(PrettyValidator, self).__init__(*args, **kwargs)
         self.session=session
+        self.sesskey=sesskey
         self.escape=lambda x: escape(unicode(str(x), 'utf8') if type(x) != unicode else x)
 
     def m(self, msg, evt=None):
@@ -337,8 +338,9 @@ class PrettyValidator(Crawler):
         return buf
 
     def done_cb(self):
-        self.session['form_json']['validated']=True
-        self.session['form_json']['jdict']=self.jdict
+        self.session[self.sesskey]['validated']=True
+        self.session[self.sesskey]['jdict']=self.jdict
+        self.session[self.sesskey]['cache_info']=self.cache_info
         self.session.save()
 
 

ffdnispdb/cron_task.py

@@ -7,6 +7,8 @@ from sys import stderr
 from datetime import datetime, timedelta
 from flask.ext.mail import Message
 from flask import url_for
+import itsdangerous
+
 from ffdnispdb.crawler import TextValidator
 from ffdnispdb.models import ISP
 from ffdnispdb import app, db, mail
@@ -46,6 +48,15 @@ signal.signal(signal.SIGALRM, timeout_handler)
 signal.alarm(6)
 
 
+def gen_reactivate_key(isp):
+    s=itsdangerous.URLSafeSerializer(app.secret_key,
+                                     salt='reactivate')
+    return s.dumps([
+        isp.id,
+        str(isp.last_update_attempt)
+    ])
+
+
 def send_warning_email(isp, debug_msg):
     msg=Message(u"Problem while updating your ISP's data", sender=('FFDN DB <cron@db.ffdn.org>'))
     msg.body = """
@@ -63,12 +74,14 @@ Here is some debug output to help you locate the issue:
 
 ---
 When the issue is resolved, please click on the link below to reactivate automatic updates on your ISP:
-%s
+%s?key=%s
 
 Thanks,
 The FFDN ISP Database team
-    """.strip()%(isp.complete_name, isp.json_url, debug_msg.strip(), url_for('home'))
+    """.strip()%(isp.complete_name, isp.json_url, debug_msg.strip(),
+                 url_for('reactivate_isp', projectid=isp.id), gen_reactivate_key(isp))
     msg.add_recipient(isp.tech_email)
+    print u'    Sending notification email to %s'%(isp.tech_email)
     mail.send(msg)
 
 
@@ -103,8 +116,8 @@ try:
                 db.session.commit()
                 print u'%s: Error while updating:'%(datetime.now())
                 if isp.update_error_strike >= 3:
-                    send_warning_email(isp, log)
                     print u'    three strikes, you\'re out'
+                    send_warning_email(isp, log)
 
                 print log.rstrip()+'\n'
                 continue

ffdnispdb/templates/project_json_validator.html

@@ -1,87 +1,4 @@
-{% extends "layout.html" %}
-{% block head %}
-{{ super() }}
-    <style type="text/css">
-    .cursor {
-        display: inline-block;
-        background: #dfdfdf;
-        margin-left: 1px;
-        margin-top: 2px;
-        vertical-align: sub;
-        height: 16px;
-        animation: blink 2s linear 0s infinite;
-    }
-    @keyframes blink {
-        0% { background: #fff }
-        49% { background: #fff }
-        50% { background: #dfdfdf }
-        99% { background: #dfdfdf }
-        100% { background: #fff }
-    }
-    </style>
-{%- endblock %}
-{% block script %}
-{{ super() }}
-<script type="text/javascript">
-$(function() {
-    var evt;
-    function validate() {
-        evt = new EventSource("{{ url_for('json_url_validator') }}");
-        evt.onmessage = function(e) {
-            $('#status').append(e.data+'\n');
-            var pre=$("#status").parent();
-            pre.stop();
-            pre.animate({
-                'scrollTop': pre[0].scrollHeight
-            }, 500);
-        }
-        evt.onerror = function(e) {
-            $('#status').append('<span style="color: red;">/!\\ Error with the validation API, connection was closed. Retry in a few seconds\n</span>')
-            evt.close();
-            setTimeout(function() { $('#retry').removeAttr('disabled') }, 5000);
-        }
-        evt.addEventListener('control', function(e) {
-            var msg=$.parseJSON(e.data);
-            if(!!msg.passed)
-                $('input[type="submit"]').removeAttr('disabled');
-                evt.close();
-
-            if(!!msg.closed) {
-                evt.close();
-                setTimeout(function() { $('#retry').removeAttr('disabled') }, 5000);
-            }
-        });
-    }
-    $('#retry').click(function() {
-        $("#status").html('');
-        validate();
-        $('#retry').attr('disabled', 'disabled')
-        return false;
-    });
-    validate();
-});
-</script>
-{%- endblock %}
-{% block container %}
-<div class="row">
-  <div class="span11 well">
-    <form method="post" action="{{ url_for('create_project_json_confirm') }}" class="form-horizontal">
-      <fieldset>
-        <legend>{{ _("Validating the JSON URL") }}</legend>
-        <pre style="height: 500px; background-color: white; overflow-x: auto;">
-<div style="line-height: normal"> _______ _______ ______  _______ 
-|_______|_______|_____ \|  ___  \
- ______  ______  _    \ | |   \ |   ____  _____ 
-|   ___)|   ___)| |   | | |   | |  |    \| __  |
-|  |    |  |    | |__/ /| |   | |  |  |  | __ -|
-|__|    |__|    |_____/ |_|   |_|  |____/|_____|
-</div>
-<div id="status"></div><div class="cursor"> </div></pre>
-        <div class="form-actions" style="text-align: right;">
-          <button id="retry" class="btn" disabled="disabled">{{ _("Retry") }}</button>
-          <input type="submit" disabled="disabled" class="btn btn-primary" value="{{ _("Confirm") }}" />
-        </div>
-      </fieldset>
-  </div>
-</div>
-{% endblock %}
+{% set page_title = _("Validating the JSON URL") %}
+{% set validator_url = url_for('json_url_validator') %}
+{% set confirm_url = url_for('create_project_json_confirm') %}
+{% include 'validator_generic.html' with context %}

ffdnispdb/templates/reactivate_validator.html

@@ -0,0 +1,4 @@
+{% set page_title = _("Reactivating updates on your project") %}
+{% set validator_url = url_for('reactivate_validator') %}
+{% set confirm_url = url_for('reactivate_isp', projectid=isp.id) %}
+{% include 'validator_generic.html' with context %}

ffdnispdb/templates/validator_generic.html

@@ -0,0 +1,87 @@
+{% extends "layout.html" %}
+{% block head %}
+{{ super() }}
+    <style type="text/css">
+    .cursor {
+        display: inline-block;
+        background: #dfdfdf;
+        margin-left: 1px;
+        margin-top: 2px;
+        vertical-align: sub;
+        height: 16px;
+        animation: blink 2s linear 0s infinite;
+    }
+    @keyframes blink {
+        0% { background: #fff }
+        49% { background: #fff }
+        50% { background: #dfdfdf }
+        99% { background: #dfdfdf }
+        100% { background: #fff }
+    }
+    </style>
+{%- endblock %}
+{% block script %}
+{{ super() }}
+<script type="text/javascript">
+$(function() {
+    var evt;
+    function validate() {
+        evt = new EventSource("{{ validator_url }}");
+        evt.onmessage = function(e) {
+            $('#status').append(e.data+'\n');
+            var pre=$("#status").parent();
+            pre.stop();
+            pre.animate({
+                'scrollTop': pre[0].scrollHeight
+            }, 500);
+        }
+        evt.onerror = function(e) {
+            $('#status').append('<span style="color: red;">/!\\ Error with the validation API, connection was closed. Retry in a few seconds\n</span>')
+            evt.close();
+            setTimeout(function() { $('#retry').removeAttr('disabled') }, 5000);
+        }
+        evt.addEventListener('control', function(e) {
+            var msg=$.parseJSON(e.data);
+            if(!!msg.passed)
+                $('input[type="submit"]').removeAttr('disabled');
+                evt.close();
+
+            if(!!msg.closed) {
+                evt.close();
+                setTimeout(function() { $('#retry').removeAttr('disabled') }, 5000);
+            }
+        });
+    }
+    $('#retry').click(function() {
+        $("#status").html('');
+        validate();
+        $('#retry').attr('disabled', 'disabled')
+        return false;
+    });
+    validate();
+});
+</script>
+{%- endblock %}
+{% block container %}
+<div class="row">
+  <div class="span11 well">
+    <form method="post" action="{{ confirm_url }}" class="form-horizontal">
+      <fieldset>
+        <legend>{{ page_title }}</legend>
+        <pre style="height: 500px; background-color: white; overflow-x: auto;">
+<div style="line-height: normal"> _______ _______ ______  _______ 
+|_______|_______|_____ \|  ___  \
+ ______  ______  _    \ | |   \ |   ____  _____ 
+|   ___)|   ___)| |   | | |   | |  |    \| __  |
+|  |    |  |    | |__/ /| |   | |  |  |  | __ -|
+|__|    |__|    |_____/ |_|   |_|  |____/|_____|
+</div>
+<div id="status"></div><div class="cursor"> </div></pre>
+        <div class="form-actions" style="text-align: right;">
+          <button id="retry" class="btn" disabled="disabled">{{ _("Retry") }}</button>
+          <input type="submit" disabled="disabled" class="btn btn-primary" value="{{ _("Confirm") }}" />
+        </div>
+      </fieldset>
+  </div>
+</div>
+{% endblock %}

ffdnispdb/views.py

@@ -3,6 +3,7 @@
 from flask import request, g, redirect, url_for, abort, \
     render_template, flash, json, session, Response, Markup
 from flask.ext.babel import gettext as _
+import itsdangerous
 import docutils.core
 import ispformat.specs
 
@@ -10,7 +11,6 @@ from datetime import date, time, timedelta, datetime
 from urlparse import urlunsplit
 import locale
 locale.setlocale(locale.LC_ALL, '')
-import string
 from time import time
 import os.path
 
@@ -18,7 +18,7 @@ from . import forms
 from .constants import *
 from . import app, db, cache
 from .models import ISP, ISPWhoosh
-from .crawler import WebValidator
+from .crawler import WebValidator, PrettyValidator
 
 
 @app.route('/')
@@ -119,7 +119,7 @@ def json_url_validator():
     else:
         session['form_json']['validator']=time()
 
-    validator=WebValidator(session=session._get_current_object())
+    validator=WebValidator(session._get_current_object(), 'form_json')
     return Response(validator(session['form_json']['url']),
                     mimetype="text/event-stream")
 
@@ -159,6 +159,70 @@ def create_project_json_confirm():
         return redirect(url_for('create_project_json'))
 
 
+@app.route('/isp/reactivate-validator', methods=['GET'])
+def reactivate_validator():
+    if 'form_reactivate' not in session or \
+       session['form_reactivate'].get('validated', False):
+        abort(403)
+
+    p=ISP.query.get(session['form_reactivate']['isp_id'])
+    if not p:
+        abort(403)
+
+    v=session['form_reactivate'].get('validator')
+
+    if v is not None:
+        if v > time()-5:
+            abort(429)
+    else:
+        session['form_reactivate']['validator']=time()
+
+    validator=PrettyValidator(session._get_current_object(), 'form_reactivate')
+    return Response(validator(p.json_url, p.cache_info or {}),
+                    mimetype="text/event-stream")
+
+
+@app.route('/isp/<projectid>/reactivate',  methods=['GET', 'POST'])
+def reactivate_isp(projectid):
+    """
+    Allow to reactivate an ISP after it has been disabled
+    because of problems with the JSON file.
+    """
+    p=ISP.query.filter(ISP.id==projectid, ISP.is_disabled==False,
+                       ISP.update_error_strike>=3).first_or_404()
+    if request.method == 'GET':
+        key = request.args.get('key')
+        try:
+            s=itsdangerous.URLSafeSerializer(app.secret_key,
+                                             salt='reactivate')
+            d=s.loads(key)
+        except Exception as e:
+            abort(403)
+
+        if (len(d) != 2 or d[0] != p.id or
+            d[1] != str(p.last_update_attempt)):
+            abort(403)
+
+        session['form_reactivate'] = {'isp_id': p.id}
+        return render_template('reactivate_validator.html', isp=p)
+    else:
+        if 'form_reactivate' not in session or \
+           not session['form_reactivate'].get('validated', False):
+            abort(409)
+
+        p=ISP.query.get(session['form_reactivate']['isp_id'])
+        p.json=session['form_reactivate']['jdict']
+        p.cache_info=session['form_reactivate']['cache_info']
+        p.last_update_attempt=datetime.now()
+        p.last_update_success=p.last_update_attempt
+
+        db.session.add(p)
+        db.session.commit()
+
+        flash(_(u'Automatic updates activated'), 'info')
+        return redirect(url_for('project', projectid=p.id))
+
+
 @app.route('/search', methods=['GET', 'POST'])
 def search():
     terms=request.args.get('q')