Working CSV import script

coin/billing/management/commands/import_payments_from_csv.py

@@ -16,7 +16,7 @@ from django.core.management.base import BaseCommand, CommandError
 
 # Coin specific imports
 from coin.members.models import Member
-from coin.billing.models import Invoice, Payment
+from coin.billing.models import Payment
 
 # Parser / import / matcher configuration
 
@@ -47,7 +47,7 @@ properies (date, label, price) to avoid creating duplicate payments inside coin.
 
 By default, only a dry-run is perfomed to let you see what will happen ! You
 should run this command with --commit if you agree with the dry-run."""
-              
+
     def create_parser(self, *args, **kwargs):
         parser = super(Command, self).create_parser(*args, **kwargs)
         parser.formatter_class = RawTextHelpFormatter
@@ -56,7 +56,7 @@ should run this command with --commit if you agree with the dry-run."""
     def add_arguments(self, parser):
 
         parser.add_argument(
-            'filename', 
+            'filename',
             type=str,
             help="The CSV filename to be parsed"
         )
@@ -79,25 +79,37 @@ should run this command with --commit if you agree with the dry-run."""
 
         payments = self.convertCSVToDicts(self.cleanCSV(self.loadCSV(options["filename"])))
 
-        members = Member.objects.filter(status="member")
-        
-        #members = [ { "id": 5, "username": "toto",    "familyname": "Michu"  },
-        #            { "id": 3, "username": "johndoe", "familyname": "Doe"    }  ]
-
-        payments.append({ "date:":"someDate", "label":"foo ID 43 zob", "amount":30.0})
-        payments.append({ "date:":"someDate", "label":"foo JohnDoe zob", "amount":30.0})
-        payments.append({ "date:":"someDate", "label":"foo John Doe zob", "amount":30.0})
-        
-        print json.dumps(payments, indent=4, separators=(',', ': '))
-        
-        maybeMatchedPayments = self.tryToMatchPaymentWithMembers(payments, members)
-      
-        print json.dumps(maybeMatchedPayments, indent=4, separators=(',', ': '))
-        print "Number of payments found       : " + str(len(maybeMatchedPayments))
-        print "Number of payments matched     : " + str(len([p for p in maybeMatchedPayments if     p["memberMatched"]]))
-        print "Number of payments not matched : " + str(len([p for p in maybeMatchedPayments if not p["memberMatched"]]))
-        return
+        # Dummy payments for test
+        payments = []
+        payments.append({ "date": "2017-03-02", "label":"foo ID 43 zob", "amount":30.0})
+        payments.append({ "date": "2017-03-14", "label":"foo JohnDoe zob", "amount":30.0})
+        payments.append({ "date": "2017-04-03", "label":"foo John Doe zob", "amount":30.0})
+
+        payments = self.tryToMatchPaymentWithMembers(payments)
+        newPayments = self.filterAlreadyKnownPayments(payments)
+
+        numberOfAlreadyKnownPayments = len(payments)-len(newPayments)
+        numberOfNewPayments = len(newPayments)
 
+        if (numberOfNewPayments > 0) :
+            print "======================================================"
+            print "   > New payments found"
+            print json.dumps(newPayments, indent=4, separators=(',', ': '))
+        print "======================================================"
+        print "Number of already known payments found : " + str(numberOfAlreadyKnownPayments)
+        print "Number of new payments found           : " + str(numberOfNewPayments)
+        print "Number of new payments matched         : " + str(len([p for p in newPayments if     p["memberMatched"]]))
+        print "Number of payments not matched         : " + str(len([p for p in newPayments if not p["memberMatched"]]))
+        print "======================================================"
+
+        if numberOfNewPayments == 0:
+            print "Nothing to do, everything looks up to date !"
+            return
+
+        if not options["commit"]:
+            print "Use --commit to register these new payments"
+        else:
+            self.addNewPayments(newPayments)
 
 
     def isDate(self, text):
@@ -131,7 +143,7 @@ should run this command with --commit if you agree with the dry-run."""
                 #logging.warning("Ignoring the following row (bad number of elements) :")
                 #logging.warning(str(row))
                 continue
-            
+
             if not self.isDate(row[0]):
                 logging.warning("Ignoring the following row (bad format for date in the first column) :")
                 logging.warning(str(row))
@@ -148,6 +160,13 @@ should run this command with --commit if you agree with the dry-run."""
                 logging.warning(str(row))
                 continue
 
+            # Clean the date
+            row[0] = datetime.datetime.strptime(row[0], DATE_FORMAT).strftime("%Y-%m-%d")
+
+            # Clean the label ...
+            row[1] = row[1].replace('\r', ' ')
+            row[1] = row[1].replace('\n', ' ')
+
             output.append(row)
 
         return output
@@ -169,13 +188,14 @@ should run this command with --commit if you agree with the dry-run."""
         return output
 
 
+    def tryToMatchPaymentWithMembers(self, payments):
 
-    def tryToMatchPaymentWithMembers(self, payments, members):
+        members = Member.objects.filter(status="member")
 
         idregex = re.compile(ID_REGEX)
 
         for payment in payments:
-            
+
             paymentLabel = str(payment["label"])
 
             # First, attempt to match the member ID
@@ -208,7 +228,7 @@ should run this command with --commit if you agree with the dry-run."""
             if usernamematch != None:
                 payment["memberMatched"] = usernamematch
                 #print "Matched by username to "+usernamematch
-                continue 
+                continue
 
 
             # Third, attempt to match by family name
@@ -234,9 +254,54 @@ should run this command with --commit if you agree with the dry-run."""
             if familynamematch != None:
                 payment["memberMatched"] = familynamematch
                 #print "Matched by familyname to "+familynamematch
-                continue 
+                continue
 
             #print "Could not match"
             payment["memberMatched"] = None
 
         return payments
+
+
+    def filterAlreadyKnownPayments(self, payments):
+
+        newPayments = []
+
+        knownPayments = Payment.objects.all()
+
+        for payment in payments:
+
+            foundMatch = False
+            for knownPayment in knownPayments:
+
+                if  (str(knownPayment.date) == str(payment["date"])) \
+                and (str(knownPayment.label_from_bank) == str(payment["label"])) \
+                and (float(knownPayment.amount) == float(payment["amount"])):
+                    foundMatch = True
+                    break
+
+
+            if not foundMatch:
+                newPayments.append(payment)
+
+        return newPayments
+
+
+    def addNewPayments(self, newPayments):
+
+        for newPayment in newPayments:
+
+            # Get the member if there's a member matched
+            member = None
+            if newPayment["memberMatched"]:
+                member = Member.objects.filter(username=newPayment["memberMatched"])
+                assert len(member) == 1
+                member = member[0]
+
+            print "Adding new payment : "
+            print newPayment
+            # Create the payment
+            payment = Payment.objects.create(amount=float(newPayment["amount"]),
+                                             label_from_bank=str(newPayment["label"]),
+                                             date=str(newPayment["date"]),
+                                             member=member)
+

coin/billing/models.py

@@ -310,6 +310,10 @@ class Payment(models.Model):
                                                    null=True, blank=True, default=0.0,
                                                    verbose_name='montant déjà alloué')
 
+    label_from_bank = models.CharField(max_length=500,
+                                       null=True, blank=True, default="",
+                                       verbose_name='libellé du virement récuppéré via la banque')
+
     def save(self, *args, **kwargs):
         super(Payment, self).save(*args, **kwargs)