ffdn-db/ffdnispdb/cron_task.py

#!/usr/bin/env python2


import signal
import traceback
from sys import stderr
from datetime import datetime, timedelta
from ffdnispdb.crawler import TextValidator
from ffdnispdb.models import ISP
from ffdnispdb import db


MAX_RUNTIME=15*60

class Timeout(Exception):
    pass

class ScriptTimeout(Exception):
    """
    Script exceeded its allowed run time
    """


strike=1
last_isp=-1
script_begin=datetime.now()
def timeout_handler(signum, frame):
    global last_isp, strike
    if script_begin < datetime.now()-timedelta(seconds=MAX_RUNTIME):
        raise ScriptTimeout

    if last_isp == isp.id:
        strike += 1
        if strike > 2:
            # three strikes, you're out.
            print "you're out", isp
            signal.alarm(6)
            raise Timeout
    else:
        last_isp = isp.id
        strike = 1

    signal.alarm(6)

signal.signal(signal.SIGALRM, timeout_handler)
signal.alarm(6)


try:
    for isp in ISP.query.filter(ISP.is_disabled == False,
                                ISP.json_url != None,
                                ISP.next_update < datetime.now(),
                                ISP.update_error_strike < 3)\
                        .order_by(ISP.last_update_success):
        try:
            print u'%s: Attempting to update %s'%(datetime.now(), isp)
            print u'    last successful update=%s'%(isp.last_update_success)
            print u'    last update attempt=%s'%(isp.last_update_attempt)
            print u'    next update was scheduled %s ago'%(datetime.now()-isp.next_update)
            print u'    strike=%d'%(isp.update_error_strike)

            isp.last_update_attempt=datetime.now()
            db.session.add(isp)
            db.session.commit()

            validator=TextValidator()
            log=''.join(validator(isp.json_url+'ab'))
            if not validator.success: # handle error
                isp.update_error_strike += 1
                #isp.next_update = bla
                db.session.add(isp)
                db.session.commit()
                print u'%s: Error while updating:'%(datetime.now())
                if isp.update_error_strike >= 3:
                    # send email
                    print u'    three strikes, you\'re out'

                print log.rstrip()+'\n'
                continue

            isp.json = validator.jdict
            isp.last_update_success = isp.last_update_attempt
            isp.update_error_strike = 0
            #isp.next_update = bla
            db.session.add(isp)
            db.session.commit()

            print u'%s: Update successful !'%(datetime.now())
            print u'    next update is scheduled for %s\n'%(isp.next_update)
        except Timeout:
            print u'%s: Timeout while updating:'%(datetime.now())
            isp=ISP.query.get(isp.id)
            isp.update_error_strike += 1
            db.session.add(isp)
            db.session.commit()
            if isp.update_error_strike >= 3:
                # send email
                print u'    three strikes, you\'re out'
            print traceback.format_exc()

except ScriptTimeout:
    pass
except Timeout:
    pass