Add a special field on the manual ISP creation form to input GeoJSON

AUTHORS

@@ -20,3 +20,16 @@
 /* TRANSLATORS */
 
 	No one yet
+
+
+/* THIRD-PARTY SOFTWARE CREDITS */
+
+    Silk icon set
+    Mark James
+    http://www.famfamfam.com/lab/icons/silk/
+    Licensed under the Creative Commons Attribution 3.0 License
+
+    Deja Vu font
+    Copyright (c) 2003, Bitstream Inc.
+    parts Copyright (c) 2006, Tavmjong Bah
+    License: http://dejavu-fonts.org/wiki/License

ffdnispdb/crawler.py

@@ -10,7 +10,7 @@ import requests
 
 from ispformat.validator import validate_isp
 from .models import ISP
-from .utils import dict_to_geojson, utcnow
+from .utils import check_geojson_spatialite, utcnow
 from . import db
 
 
@@ -291,9 +291,7 @@ class Crawler(object):
         for ca in jdict.get('coveredAreas', []):
             if not 'area' in ca:
                 continue
-            gjson=dict_to_geojson(ca['area'])
-            is_valid=bool(db.session.query(db.func.GeomFromGeoJSON(gjson) != None).first()[0])
-            if not is_valid:
+            if not check_geojson_spatialite(ca['area']):
                 yield self.err('GeoJSON data for covered area "%s" cannot '
                                'be handled by our database'%esc(ca['name']))
                 yield self.abort('Please fix your GeoJSON')

ffdnispdb/default_settings.py

@@ -4,8 +4,10 @@ SQLALCHEMY_DATABASE_URI = 'sqlite:///../ffdn-db.sqlite'
 CRAWLER_MIN_CACHE_TIME = 60*60 # 1 hour
 CRAWLER_MAX_CACHE_TIME = 60*60*24*14 # 2 week
 CRAWLER_DEFAULT_CACHE_TIME = 60*60*12 # 12 hours
-SYSTEM_TIME_ZONE='Europe/Paris'
+SYSTEM_TIME_ZONE = 'Europe/Paris'
 LANGUAGES = {
     'en': 'English',
     'fr': 'Français',
 }
+ISP_FORM_GEOJSON_MAX_SIZE = 256*1024
+ISP_FORM_GEOJSON_MAX_SIZE_TOTAL = 1024*1024

ffdnispdb/forms.py

@@ -1,16 +1,22 @@
 from functools import partial
 import itertools
 import urlparse
+import json
+import collections
+from flask import current_app
 from flask.ext.wtf import Form
 from wtforms import Form as InsecureForm
 from wtforms import (TextField, DateField, DecimalField, IntegerField, SelectField,
                      SelectMultipleField, FieldList, FormField)
-from wtforms.widgets import TextInput, ListWidget, html_params, HTMLString, CheckboxInput, Select
-from wtforms.validators import DataRequired, Optional, URL, Email, Length, NumberRange, ValidationError
-from flask.ext.babel import lazy_gettext as _
+from wtforms.widgets import TextInput, ListWidget, html_params, HTMLString, CheckboxInput, Select, TextArea
+from wtforms.validators import (DataRequired, Optional, URL, Email, Length,
+                                NumberRange, ValidationError, StopValidation)
+from flask.ext.babel import lazy_gettext as _, gettext
 from babel.support import LazyProxy
+from ispformat.validator import validate_geojson
 from .constants import STEPS
 from .models import ISP
+from .utils import check_geojson_spatialite, filesize_fmt
 
 
 class InputListWidget(ListWidget):
@@ -22,6 +28,7 @@ class InputListWidget(ListWidget):
         html.append('</%s>' % self.html_tag)
         return HTMLString(''.join(html))
 
+
 class MultiCheckboxField(SelectMultipleField):
     """
     A multiple-select, except displays a list of checkboxes.
@@ -32,12 +39,47 @@ class MultiCheckboxField(SelectMultipleField):
     widget = ListWidget(prefix_label=False)
     option_widget = CheckboxInput()
 
+
 class MyFormField(FormField):
 
     @property
     def flattened_errors(self):
         return list(itertools.chain.from_iterable(self.errors.values()))
 
+
+class GeoJSONField(TextField):
+    widget = TextArea()
+
+    def process_formdata(self, valuelist):
+        if valuelist and valuelist[0]:
+            max_size = current_app.config['ISP_FORM_GEOJSON_MAX_SIZE']
+            if len(valuelist[0]) > max_size:
+                raise ValueError(_(u'JSON value too big, must be less than %(max_size)s',
+                                   max_size=filesize_fmt(max_size)))
+            try:
+                self.data = json.loads(valuelist[0], object_pairs_hook=collections.OrderedDict)
+            except Exception as e:
+                raise ValueError(_(u'Not a valid JSON value'))
+        elif valuelist and valuelist[0].strip() == '':
+            self.data = None # if an empty string was passed, set data as None
+
+    def _value(self):
+        if self.raw_data:
+            return self.raw_data[0]
+        elif self.data is not None:
+            return json.dumps(self.data)
+        else:
+            return ''
+
+    def pre_validate(self, form):
+        if self.data is not None:
+            if not validate_geojson(self.data):
+                raise StopValidation(_(u'Invalid GeoJSON, please check it'))
+            if not check_geojson_spatialite(self.data):
+                # TODO: log this
+                raise StopValidation(_(u'GeoJSON not understood by database'))
+
+
 class Unique(object):
     """ validator that checks field uniqueness """
     def __init__(self, model, field, message=None, allow_edit=False):
@@ -65,7 +107,7 @@ class CoveredArea(InsecureForm):
     name         = TextField(_(u'name'), widget=partial(TextInput(), class_='input-medium', placeholder=_(u'Area')))
     technologies = SelectMultipleField(_(u'technologies'), choices=TECHNOLOGIES_CHOICES,
                                        widget=partial(Select(True), **{'class': 'selectpicker', 'data-title': _(u'Technologies deployed')}))
-#    area          =
+    area         = GeoJSONField(_('area'), widget=partial(TextArea(), class_='geoinput'))
 
     def validate(self, *args, **kwargs):
         r=super(CoveredArea, self).validate(*args, **kwargs)
@@ -103,7 +145,7 @@ class ProjectForm(Form):
     chatrooms     = FieldList(TextField(_(u'chatrooms')), min_entries=1, widget=InputListWidget(),
                               description=[None, _(u'In URI form, e.g. <code>irc://irc.isp.net/#isp</code> or '+
                                                     '<code>xmpp:isp@chat.isp.net?join</code>')])
-    covered_areas = FieldList(MyFormField(CoveredArea, widget=partial(InputListWidget(), class_='formfield')),
+    covered_areas = FieldList(MyFormField(CoveredArea, _('Covered Areas'), widget=partial(InputListWidget(), class_='formfield')),
                                           min_entries=1, widget=InputListWidget(),
                                           description=[None, _(u'Descriptive name of the covered areas and technologies deployed')])
     latitude      = DecimalField(_(u'latitude'), validators=[Optional(), NumberRange(min=-90, max=90)],
@@ -130,6 +172,14 @@ class ProjectForm(Form):
             # not printed, whatever..
             raise ValidationError(_(u'You must specify at least one area'))
 
+        geojson_size = sum([len(ca.area.raw_data[0]) for ca in self.covered_areas if ca.area.raw_data])
+        print geojson_size
+        max_size = current_app.config['ISP_FORM_GEOJSON_MAX_SIZE_TOTAL']
+        if geojson_size > max_size:
+            # TODO: XXX This is not printed !
+            raise ValidationError(gettext(u'The size of all GeoJSON data combined must not exceed %(max_size)s',
+                                          max_size=filesize_fmt(max_size)))
+
     def to_json(self, json=None):
         if json is None:
             json={}
@@ -144,6 +194,14 @@ class ProjectForm(Form):
             if k in json or len(v):
                 json[k]=v
 
+        def transform_covered_areas(cas):
+            for ca in cas:
+                if not ca['name']:
+                    continue
+                if 'area' in ca and ca['area'] is None:
+                    del ca['area']
+                yield ca
+
         optstr('shortname', self.shortname.data)
         optstr('description', self.description.data)
         optstr('logoURL', self.logo_url.data)
@@ -158,7 +216,7 @@ class ProjectForm(Form):
         optlist('chatrooms', filter(bool, self.chatrooms.data)) # remove empty strings
         optstr('coordinates', {'latitude': self.latitude.data, 'longitude': self.longitude.data}
                                 if self.latitude.data else {})
-        optlist('coveredAreas', filter(lambda e: e['name'], self.covered_areas.data))
+        optlist('coveredAreas', list(transform_covered_areas(self.covered_areas.data)))
         return json
 
     @classmethod

ffdnispdb/static/css/style.css

@@ -77,10 +77,14 @@ label, input, button, select, textarea, input[type="text"], input[type="password
 input[type="datetime"], input[type="datetime-local"], input[type="date"],
 input[type="month"], input[type="time"], input[type="week"], input[type="number"],
 input[type="email"], input[type="url"], input[type="search"], input[type="tel"],
-input[type="color"], .uneditable-input {
+input[type="color"], .uneditable-input, .btn {
     font-size: 13px;
 }
 
+.btn-large {
+    font-size: 17px;
+}
+
 h1 {
     font-size: 34px;
 }

ffdnispdb/static/js/site.js

@@ -16,6 +16,79 @@ $(function () {
     });
     $('.selectpicker').selectpicker();
     $("[rel=tooltip]").tooltip();
+
+    var Geoinput = function(el, options, e) {
+        this.$element = $(el);
+        this.init();
+    };
+    Geoinput.prototype = {
+        constructor: Geoinput,
+
+        init: function() {
+            this.$element.hide();
+            this.$button = this.makeButton();
+            this.$modal  = this.makeModal();
+            this.$element.after(this.$button);
+            this.$element.after(this.$modal);
+
+            this.$modal.find('textarea').val(this.$element.val());
+            this.buttonIcon();
+
+            var that = this;
+            this.$button.click(function(e) {
+                e.preventDefault();
+                that.$modal.modal();
+                return false;
+            });
+            this.$modal.find('.btn-primary').click(function(e) {
+                e.preventDefault();
+                that.$modal.modal('hide');
+                that.$element.val(that.$modal.find('textarea').val());
+                that.buttonIcon.call(that);
+                return false;
+            });
+        },
+
+        buttonIcon: function() {
+            if(this.$element.val())
+                this.$button[0].firstChild.src = '/static/img/map_edit.png';
+            else
+                this.$button[0].firstChild.src = '/static/img/map.png';
+        },
+
+        makeButton: function() {
+            return $('<button/>').addClass("btn btn-default geoinput-button")
+                                 .css('padding', '4px 7px')
+                                 .attr('title', 'enter geojson')
+                                 .html('<img src="/static/img/map.png" alt="map">');
+        },
+
+        makeModal: function() {
+            return $('<div class="modal hide geoinput-modal">'+
+                     '<div class="modal-header">'+
+                     '<h3>GeoJSON Input</h3>'+
+                     '</div>'+
+                     '<div class="modal-body">'+
+                     '<p>Paste your GeoJSON here:</p>'+
+                     '<textarea style="width: 97%; height: 200px"></textarea>'+
+                     '</div>'+
+                     '<div class="modal-footer">'+
+                     '<button class="btn" data-dismiss="modal" aria-hidden="true">Cancel</button>'+
+                     '<button class="btn btn-primary">Done</button>'+
+                     '</div>'+
+                     '</div>')
+        }
+    }
+
+    $.fn.geoinput = function(options, event) {
+        return this.each(function() {
+            var $this = $(this), data = $this.data('geoinput');
+            if($this.is('input, textarea')) {
+                $this.data('geoinput', (data = new Geoinput(this, options, event)));
+            }
+        });
+    };
+    $('.geoinput').geoinput();
     init_map();
 });
 
@@ -254,7 +327,7 @@ function init_map() {
 }
 
 function change_input_num(li, new_num, reset) {
-    li.find('input,select').each(function() {
+    li.find('input,select,textarea').each(function() {
         var id = $(this).attr('id').replace(/^(.*)-\d{1,4}/, '$1-'+new_num);
         $(this).attr({'name': id, 'id': id});
         if(!!reset)
@@ -279,11 +352,14 @@ function clone_fieldlist(el) {
     var new_element = el.clone(true);
     var elem_id = new_element.find(':input')[0].id;
     var elem_num = parseInt(elem_id.replace(/^.*-(\d{1,4})/, '$1')) + 1;
-    change_input_num(new_element, elem_num, true);
     new_element.children('button').remove();
     new_element.children('.help-inline.error-list').remove();
     new_element.find('.bootstrap-select').remove();
-    append_remove_button(new_element);
+    new_element.find('.geoinput-button').remove();
+    new_element.find('.geoinput-modal').remove();
+    change_input_num(new_element, elem_num, true);
     new_element.find('.selectpicker').data('selectpicker', null).selectpicker();
+    new_element.find('.geoinput').geoinput();
+    append_remove_button(new_element);
     el.after(new_element);
 }

ffdnispdb/utils.py

@@ -5,6 +5,7 @@ from collections import OrderedDict
 from datetime import datetime
 import pytz
 import json
+from . import db
 
 
 
@@ -30,6 +31,21 @@ def dict_to_geojson(d_in):
     return json.dumps(d)
 
 
+def check_geojson_spatialite(_gjson):
+    """
+    Checks if a GeoJSON dict is understood by spatialite
+
+    >>> check_geojson_spatialite({'type': 'NOPE', 'coordinates': []})
+    False
+    >>> check_geojson_spatialite({'type': 'Polygon', 'coordinates': [
+    ...    [[0.0, 0.0], [0.0, 1.0], [1.0, 1.0], [1.0, 0.0]]
+    ... ]})
+    True
+    """
+    gjson=dict_to_geojson(_gjson)
+    return bool(db.session.query(db.func.GeomFromGeoJSON(gjson) != None).first()[0])
+
+
 def utcnow():
     """
     Return the current UTC date and time as a datetime object with proper tzinfo.
@@ -42,3 +58,12 @@ def tosystemtz(d):
     Convert the UTC datetime ``d`` to the system time zone defined in the settings
     """
     return d.astimezone(pytz.timezone(current_app.config['SYSTEM_TIME_ZONE']))
+
+
+def filesize_fmt(num):
+    fmt = lambda num, unit: "%s %s" % (format(num, '.2f').rstrip('0').rstrip('.'), unit)
+    for x in ['bytes', 'KiB', 'MiB', 'GiB']:
+        if num < 1024.0:
+            return fmt(num, x)
+        num /= 1024.0
+    return fmt(num, 'TiB')

ffdnispdb/views.py

@@ -284,6 +284,7 @@ def create_project_json_confirm():
         isp.json_url=session['form_json']['url']
         isp.json=jdict
         isp.tech_email=session['form_json']['tech_email']
+        isp.last_update_attempt=session['form_json']['last_update']
         isp.last_update_success=session['form_json']['last_update']
         isp.next_update=session['form_json']['next_update']
         isp.cache_info=session['form_json']['cache_info']