#!/usr/bin/env python2 import signal import traceback from sys import stderr from datetime import datetime, timedelta from ffdnispdb.crawler import TextValidator from ffdnispdb.models import ISP from ffdnispdb import db MAX_RUNTIME=15*60 class Timeout(Exception): pass class ScriptTimeout(Exception): """ Script exceeded its allowed run time """ strike=1 last_isp=-1 script_begin=datetime.now() def timeout_handler(signum, frame): global last_isp, strike if script_begin < datetime.now()-timedelta(seconds=MAX_RUNTIME): raise ScriptTimeout if last_isp == isp.id: strike += 1 if strike > 2: # three strikes, you're out. print "you're out", isp signal.alarm(6) raise Timeout else: last_isp = isp.id strike = 1 signal.alarm(6) signal.signal(signal.SIGALRM, timeout_handler) signal.alarm(6) try: for isp in ISP.query.filter(ISP.is_disabled == False, ISP.json_url != None, ISP.next_update < datetime.now(), ISP.update_error_strike < 3)\ .order_by(ISP.last_update_success): try: print u'%s: Attempting to update %s'%(datetime.now(), isp) print u' last successful update=%s'%(isp.last_update_success) print u' last update attempt=%s'%(isp.last_update_attempt) print u' next update was scheduled %s ago'%(datetime.now()-isp.next_update) print u' strike=%d'%(isp.update_error_strike) isp.last_update_attempt=datetime.now() db.session.add(isp) db.session.commit() validator=TextValidator() log=''.join(validator(isp.json_url+'ab')) if not validator.success: # handle error isp.update_error_strike += 1 #isp.next_update = bla db.session.add(isp) db.session.commit() print u'%s: Error while updating:'%(datetime.now()) if isp.update_error_strike >= 3: # send email print u' three strikes, you\'re out' print log.rstrip()+'\n' continue isp.json = validator.jdict isp.last_update_success = isp.last_update_attempt isp.update_error_strike = 0 #isp.next_update = bla db.session.add(isp) db.session.commit() print u'%s: Update successful !'%(datetime.now()) print u' next update is scheduled for %s\n'%(isp.next_update) except Timeout: print u'%s: Timeout while updating:'%(datetime.now()) isp=ISP.query.get(isp.id) isp.update_error_strike += 1 db.session.add(isp) db.session.commit() if isp.update_error_strike >= 3: # send email print u' three strikes, you\'re out' print traceback.format_exc() except ScriptTimeout: pass except Timeout: pass