Merge remote-tracking branch 'ARN/fix-misc-stuff-in-csv-import_'

coin/billing/management/commands/import_payments_from_csv.py

@@ -27,6 +27,8 @@ import logging
 import os
 import re
 
+import unidecode
+
 # Django specific imports
 from argparse import RawTextHelpFormatter
 from django.core.management.base import BaseCommand, CommandError
@@ -45,7 +47,7 @@ DATE_FORMAT="%d/%m/%Y"
 ID_REGEX=r"(?i)(\b|_)ID[\s\-\_\/]*(\d+)(\b|_)"
 # If the label of the payment contains one of these, the payment won't be
 # matched to a member when importing it.
-KEYWORDS_TO_NOTMATCH=[ "DON", "MECENAT", "REM CHQ" ]
+KEYWORDS_TO_NOTMATCH=[ "REM CHQ" ]
 
 class Command(BaseCommand):
 
@@ -76,9 +78,10 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
 
         assert options["filename"] != ""
-
         if not os.path.isfile(options["filename"]):
             raise CommandError("This file does not exists.")
+        os.system("iconv -f ISO-8859-1 -t UTF-8 %s > %s.utf8.csv" % (options["filename"], options["filename"]))
+        options["filename"] = options["filename"] + '.utf8.csv'
 
         payments = self.convert_csv_to_dicts(self.clean_csv(self.load_csv(options["filename"])))
 
@@ -189,13 +192,14 @@ class Command(BaseCommand):
 
     def try_to_match_payment_with_members(self, payments):
 
-        members = Member.objects.filter(status="member")
+        #members = Member.objects.filter(status="member")
+        members = Member.objects.all()
 
         idregex = re.compile(ID_REGEX)
 
         for payment in payments:
 
-            payment_label = payment["label"]
+            payment_label = payment["label"].upper()
 
             # First, attempt to match the member ID
             idmatches = idregex.findall(payment_label)
@@ -211,11 +215,14 @@ class Command(BaseCommand):
             # Second, attempt to find the username
             usernamematch = None
             for member in members:
-                matches = re.compile(r"(?i)(\b|_)"+re.escape(member.username)+r"(\b|_)") \
+                username = self.flatten(member.username)
+                matches = re.compile(r"(?i)(\b|_)"+re.escape(username)+r"(\b|_)") \
                             .findall(payment_label)
+
                 # If not found, try next
                 if len(matches) == 0:
                     continue
+
                 # If we already had a match, abort the whole search because we
                 # have multiple usernames matched !
                 if usernamematch != None:
@@ -236,23 +243,31 @@ class Command(BaseCommand):
                 if member.last_name == "":
                     continue
 
-                matches = re.compile(r"(?i)(\b|_)"+re.escape(str(member.last_name))+r"(\b|_)") \
+                # "Flatten" accents in the last name... (probably the CSV
+                # don't contain 'special' chars like accents
+                member_last_name = self.flatten(member.last_name)
+
+                matches = re.compile(r"(?i)(\b|_)"+re.escape(member_last_name)+r"(\b|_)") \
                             .findall(payment_label)
+
                 # If not found, try next
                 if len(matches) == 0:
                     continue
+
                 # If this familyname was matched several time, abort the whole search
-                if len(matches) > 1:
-                    familynamematch = None
-                    break
+                #if len(matches) > 1:
+                #    print("Several matches ! Aborting !")
+                #    familynamematch = None
+                #    break
+
                 # If we already had a match, abort the whole search because we
                 # have multiple familynames matched !
                 if familynamematch != None:
                     familynamematch = None
                     break
 
-                familynamematch = str(member.last_name)
-                usernamematch = str(member.username)
+                familynamematch = member_last_name
+                usernamematch = member.username
 
             if familynamematch != None:
                 payment["member_matched"] = usernamematch
@@ -336,3 +351,5 @@ class Command(BaseCommand):
                                              date=new_payment["date"],
                                              member=member)
 
+    def flatten(self, some_string):
+        return unidecode.unidecode(some_string).upper()

requirements.txt

@@ -17,3 +17,4 @@ six==1.10.0
 WeasyPrint==0.31
 freezegun==0.3.8
 pytz>=2018.5
+unidecode>=1.0,<1.1