Add structured ISO-11649 creditor references for EUR bank transactions

.mypy.ini

@@ -94,3 +94,6 @@ ignore_missing_imports = True
 
 [mypy-flask_mailman.*]
 ignore_missing_imports = True
+
+[mypy-stdnum.*]
+ignore_missing_imports = True

apps/base/dev/tasks.py

@@ -591,7 +591,7 @@ def create_bank_accounts():
         institution="London Bank",
         address="13 Bartlett Place, London, WC1B 4NM",
         iban="GB33BUKB20201555555555",
-        swift="GB33BUKB",
+        swift="BUKBGB33",
     )
     for acct in [gbp, eur]:
         try:

apps/common/__init__.py

@@ -70,6 +70,8 @@ def format_price(price, currency=None, after=False):
 
     @app_obj.template_filter("bankref")
     def format_bankref(bankref):
+        if bankref.startswith("RF"):
+            return " ".join(wrap(bankref, 4))
         return "%s-%s" % (bankref[:4], bankref[4:])
 
     @app_obj.context_processor

apps/common/epc.py

@@ -13,7 +13,8 @@ def make_epc_qrfile(payment: BankPayment, **kwargs) -> BytesIO:
         name=payment.recommended_destination.payee_name,
         iban=payment.recommended_destination.iban,
         amount=payment.amount,
-        reference=payment.bankref,
+        reference=payment.customer_reference,
+        bic=payment.recommended_destination.swift,
         encoding=1,
     )
     qr.save(qrfile, **kwargs)

models/payment.py

@@ -2,11 +2,13 @@
 import re
 from decimal import Decimal
 from datetime import datetime, timedelta
-from typing import Optional
+from typing import Iterable, Optional
 from sqlalchemy import event, func, column
 from sqlalchemy.orm import Session, aliased
 from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
 from sqlalchemy_continuum.utils import version_class, transaction_class
+from stdnum import iso11649
+from stdnum.iso7064 import mod_97_10
 
 from main import db
 from . import (
@@ -314,6 +316,23 @@ def recommended_destination(self):
             except (MultipleResultsFound, NoResultFound):
                 continue
 
+    @property
+    def customer_reference(self):
+        if self.id is None:
+            raise Exception(
+                "Customer references can only be generated for payments that have "
+                "been persisted to the database."
+            )
+
+        # Derive an ISO-11649 payment reference for EUR-currency payments
+        if self.currency == "EUR":
+            order_check_digits = mod_97_10.calc_check_digits(f"{self.bankref}RF")
+            customer_reference = f"RF{order_check_digits}{self.bankref}"
+            assert iso11649.is_valid(customer_reference)
+            return customer_reference
+        else:
+            return self.bankref
+
 
 class BankAccount(BaseModel):
     __tablename__ = "bank_account"
@@ -423,7 +442,21 @@ def get_matching(self):
         return matching
 
     def match_payment(self) -> Optional[BankPayment]:
+        for bankref in self._recognized_bankrefs:
+            try:
+                return BankPayment.query.filter_by(bankref=bankref).one()
+            except NoResultFound:
+                continue
+
+        return None
+
+    @property
+    def _recognized_bankrefs(self) -> Iterable[str]:
         """
+        Given a customer reference text received on a bank transfer, scan for
+        substrings that appear to be valid bank references that we can match
+        against bank transfer records in the database.
+
         We need to deal with human error and character deletion without colliding.
         Unless we use some sort of coding, the minimum length of a bankref should
         be 8, although 7 is workable. For reference:
@@ -447,30 +480,24 @@ def match_payment(self) -> Optional[BankPayment]:
 
         where serial is a 6-digit number, and ref is often the payee
         name again, or REFERENCE, and always truncated to 8 chars.
+
+        We've also received ISO-11649 payment references formatted:
+
+          ref/timestamp_plus_iban
+
+        Where the timestamp contains digits and is immediately followed by
+        the payer's IBAN (no delimiter between the two).
         """
 
         ref = self.payee.upper()
+        hdr = "(RF[0-9][0-9] ?)?"  # optional ISO11649 header + check-digits
 
-        found = re.findall("[%s]{4}[- ]?[%s]{4}" % (safechars, safechars), ref)
-        for f in found:
+        found = re.findall("%s([%s]{4}[- ]?[%s]{4})" % (hdr, safechars, safechars), ref)
+        for iso_header, f in found:
             bankref = f.replace("-", "").replace(" ", "")
-            try:
-                return BankPayment.query.filter_by(bankref=bankref).one()
-            except NoResultFound:
+            if iso_header and not iso11649.is_valid(iso_header + bankref):
                 continue
-
-        # It's pretty safe to match against the last character being lost
-        found = re.findall("[%s]{4}[- ]?[%s]{3}" % (safechars, safechars), ref)
-        for f in found:
-            bankref = f.replace("-", "").replace(" ", "")
-            try:
-                return BankPayment.query.filter(
-                    BankPayment.bankref.startswith(bankref)
-                ).one()
-            except NoResultFound:
-                continue
-
-        return None
+            yield bankref
 
 
 db.Index(

pyproject.toml

@@ -56,6 +56,7 @@ pywisetransfer = "^0.3.1"
 freezegun = "^1.1.0"
 logging_tree = "^1.9"
 flask-mailman = "^0.3.0"
+python-stdnum = "^1.19"
 
 
 [tool.poetry.group.dev.dependencies]

templates/admin/_paymenthelpers.html

@@ -5,7 +5,7 @@
 <dt>Total</dt><dd>{{ payment.amount | price(payment.currency) }}</dd>
 <dt>Provider</dt><dd>{{payment.provider}}</dd>
 {%- if payment.provider == 'banktransfer' %}
-<dt>Reference</dt><dd>{{ payment.bankref | bankref }}</dd>
+<dt>Reference</dt><dd>{{ payment.customer_reference | bankref }}</dd>
 {% elif payment.provider == 'stripe' %}
 <dt>Charge ID</dt><dd><a href="https://dashboard.stripe.com/search?query={{payment.charge_id}}">{{payment.charge_id}}</a></dd>
 <dt>Intent ID</dt><dd><a href="https://dashboard.stripe.com/search?query={{payment.intent_id}}">{{payment.intent_id}}</a></dd>

templates/admin/accounts/txn-suggest-payments.html

@@ -40,7 +40,7 @@ <h2>Reconcile transaction</h2>
   <td><b>{{ payment.user.name }}</b></td>
   <td><a href="{{ url_for('.user', user_id=payment.user.id) }}">{{ payment.user.email }}</a></td>
   <td><b>{{ payment.amount | price(payment.currency) }}</b></td>
-  <td>{{ payment.bankref | bankref }}</td>
+  <td>{{ payment.customer_reference | bankref }}</td>
   <td>
     <a class="btn btn-warning" href="{{ url_for('admin.transaction_reconcile', txn_id=txn.id, payment_id=payment.id) }}">Reconcile</a>
   </td>

templates/admin/payments/payment-send-reminder.html

@@ -13,7 +13,7 @@ <h2>Payment reminder email</h2>
 <dt>Price</dt><dd>{{ payment.amount | price(payment.currency) }}</dd>
 {%- if payment.provider == 'banktransfer' %}
 <dt>Bank</dt><dd>{{ account.institution }} ({{ account.currency }})</dd>
-<dt>Reference</dt><dd>{{ payment.bankref | bankref }}</dd>
+<dt>Reference</dt><dd>{{ payment.customer_reference | bankref }}</dd>
 {% endif -%}
 <dt>Expires</dt><dd>{{ payment.expires and payment.expires.strftime('%Y-%m-%d %H:%M:%S') }}</dd>
 <dt>Already bought</dt><dd>{{ payment.user.get_owned_tickets(paid=True) | list | count }}</dd>

templates/emails/tickets-paid-email-banktransfer.txt

@@ -2,7 +2,7 @@
 {% block body %}
 Hi {{ user.name }},
 
-This is to confirm that we've received {{ payment.amount | price(payment.currency) }} for transaction {{ payment.bankref | bankref }} from
+This is to confirm that we've received {{ payment.amount | price(payment.currency) }} for transaction {{ payment.customer_reference | bankref }} from
 you as payment for {% if payment.purchases | count > 1 %}{{ payment.purchases | count }} tickets{% else %}a ticket{% endif %} for Electromagnetic Field.
 
 {% if feature_enabled('ISSUE_TICKETS') -%}

templates/payments/invoice.html

@@ -181,7 +181,7 @@ <h2>{% if mode == 'invoice' %}VAT Invoice{% else %}Receipt{% endif %}</h2>
     <dt>SWIFT</dt><dd>{{ account.swift }}</dd>
     <dt>IBAN</dt><dd>{{ account.iban | iban }}</dd>
     {% endif %}
-    <dt>Reference</dt><dd>{{ payment.bankref | bankref }}</dd>
+    <dt>Reference</dt><dd>{{ payment.customer_reference | bankref }}</dd>
     {#
       {% if payment.provider == 'banktransfer' and payment.currency == 'EUR' and payment.state == 'inprogress' %}
       <dt>EPC QR Code</dt><dd>{{ render_epc_qr(payment) }}</dd>

templates/payments/transfer-waiting.html

@@ -42,7 +42,7 @@ <h2>Thank you!</h2>
 <dt>IBAN</dt><dd>{{ account.iban | iban }}</dd>
 {% endif %}
 <dt>Amount</dt><dd>{{ payment.amount | price(payment.currency) }}</dd>
-<dt>Reference</dt><dd>{{ payment.bankref | bankref }}</dd>
+<dt>Reference</dt><dd>{{ payment.customer_reference | bankref }}</dd>
 </dl>
 
 {% if SITE_STATE != 'after-event' %}

tests/test_epc.py

@@ -1,13 +1,43 @@
+import pytest
+
 from pyzbar.pyzbar import decode
+import re
+from stdnum import iso11649
 
+from main import db
 from apps.common.epc import format_inline_epc_qr
 from models.payment import BankPayment
+from models.user import User
 
 from tests._utils import render_svg
 
 
+@pytest.mark.parametrize(
+    "customer_reference, expected_format",
+    [
+        ("VFH9K3RQ", "VFH9-K3RQ"),
+        ("RF679HKFVR8Q", "RF67 9HKF VR8Q"),
+    ],
+)
+def test_customer_reference_display_format(app, customer_reference, expected_format):
+    assert app.jinja_env.filters["bankref"](customer_reference) == expected_format
+
+
 def test_format_inline_epc_qr(app):
+    user = User("[email protected]", "test_invoice_user")
+    db.session.add(user)
+    db.session.commit()
+
     payment = BankPayment(currency="EUR", amount=10)
+    payment.user_id = user.id
+
+    # Persist the payment object so that a sequence ID is generated for it
+    db.session.add(payment)
+    db.session.commit()
+
+    # Ensure that the structured creditor reference is valid and contains the bankref
+    assert iso11649.is_valid(payment.customer_reference)
+    assert re.match(f"^RF[0-9][0-9]{payment.bankref}$", payment.customer_reference)
 
     qr_inline = format_inline_epc_qr(payment)
     qr_image = render_svg(qr_inline)
@@ -17,12 +47,12 @@ def test_format_inline_epc_qr(app):
         "002",
         "1",
         "SCT",
-        "",
+        "BUKBGB33",
         "EMF Festivals Ltd",
         "GB33BUKB20201555555555",
         "EUR10",
         "",
-        payment.bankref,
+        payment.customer_reference,
     ]
 
     decoded = decode(qr_image)

tests/test_payment.py

@@ -0,0 +1,31 @@
+import pytest
+
+from models.payment import BankTransaction
+
+
+@pytest.mark.parametrize(
+    "payee, bankref",
+    [
+        ("M87X CJ3Q", "M87XCJ3Q"),
+        ("m87x Cj3q", "M87XCJ3Q"),
+        ("prefix*M87X CJ3Q*suffix", "M87XCJ3Q"),
+        ("RF91 M87X CJ3Q", "M87XCJ3Q"),
+        ("RF91M87XCJ3Q", "M87XCJ3Q"),
+        ("name value RF52RF23MHBY type value", "RF23MHBY"),
+        ("prefix*RF52RF23MHBY*suffix", "RF23MHBY"),
+        ("RF52RF23MHBY/20250102090807GB33BUKB20201555555555", "RF23MHBY"),
+        ("RF52RF23MHBY", "RF23MHBY"),
+        ("RF33*RF52RF23MHBY*RF33", "RF23MHBY"),
+        ("RF23MHBY", "RF23MHBY"),
+        ("RF33*RF23MHBY*RF33", "RF23MHBY"),
+    ],
+)
+def test_bankref_recognition(payee, bankref):
+    transaction = BankTransaction(
+        account_id=None,
+        amount=0,
+        payee=payee,
+        posted=None,
+        type=None,
+    )
+    assert bankref in transaction._recognized_bankrefs