Subscription base setup (#100)

* Add subscription models/query/mutations

* Monkey patch weird init error

- Upgrade strawberry-django: 0.47.1 -> 0.48.1

* Restructure subscription attributes

* Add subscription alert tagging tasks

* Break Alert.is_processed_by_subscription field into 2 migrations

To avoid full rewrite of the table https://docs.djangoproject.com/en/5.1/ref/migration-operations/#addfield

* Change subscription is_active default as False

- Add test cases for subscription limit per user

* Subscription email basic setup

- Add unsubscribe_url
- Fix subscription filtering typing issue

apps/cap_feed/admin.py

@@ -153,6 +153,7 @@ class FeedAdmin(admin.ModelAdmin):
     list_filter = (
         'format',
         'country__region',
+        'enable_polling',
         AutocompleteFilterFactory('Country', 'country'),
     )
     search_fields = ['url']

apps/cap_feed/dataloaders.py

@@ -63,7 +63,23 @@ def load_feed(keys: list[int]) -> list['FeedType']:
     return _load_model(Feed, keys)  # type: ignore[reportGeneralTypeIssues]
 
 
-def load_admin1_by_alert(keys: list[int]) -> list[list['Admin1Type']]:
+def load_admin1_by_admin1s(keys_array: list[tuple[int]]) -> list[list['Admin1Type']]:
+    keys = [key for keys in keys_array for key in keys]
+    qs = Admin1.objects.filter(id__in=keys)
+
+    _map = defaultdict(list)
+    admin1_map = {admin1.pk: admin1 for admin1 in qs.all()}
+
+    for keys in keys_array:
+        for key in keys:
+            if key not in admin1_map:
+                continue
+            _map[keys].append(admin1_map[key])
+
+    return [_map[keys] for keys in keys_array]
+
+
+def load_admin1s_by_alert(keys: list[int]) -> list[list['Admin1Type']]:
     qs = (
         AlertAdmin1.objects.filter(alert__in=keys)
         .order_by()
@@ -228,9 +244,13 @@ def load_continent(self):
     def load_feed(self):
         return DataLoader(load_fn=sync_to_async(load_feed))
 
+    @cached_property
+    def load_admin1_by_admin1s(self):
+        return DataLoader(load_fn=sync_to_async(load_admin1_by_admin1s))
+
     @cached_property
     def load_admin1s_by_alert(self):
-        return DataLoader(load_fn=sync_to_async(load_admin1_by_alert))
+        return DataLoader(load_fn=sync_to_async(load_admin1s_by_alert))
 
     @cached_property
     def load_admin1s_by_country(self):

apps/cap_feed/factories.py

@@ -0,0 +1,71 @@
+import datetime
+
+import factory
+from factory.django import DjangoModelFactory
+
+from .models import Admin1, Alert, AlertInfo, Country, Feed, Region
+
+
+class RegionFactory(DjangoModelFactory):
+    ifrc_go_id = factory.Sequence(lambda n: n)
+    name = factory.Sequence(lambda n: f'Region-{n}')
+
+    class Meta:  # type: ignore[reportIncompatibleVariableOverride]
+        model = Region
+
+
+class CountryFactory(DjangoModelFactory):
+    ifrc_go_id = factory.Sequence(lambda n: n)
+    name = factory.Sequence(lambda n: f'Country-{n}')
+    iso3 = factory.Sequence(lambda n: f"{n:0>3}")
+
+    class Meta:  # type: ignore[reportIncompatibleVariableOverride]
+        model = Country
+
+
+class FeedFactory(DjangoModelFactory):
+    url = factory.Sequence(lambda n: f"https://source-{n}.com/test")
+    format = Feed.Format.RSS
+    polling_interval = Feed.PoolingInterval.I_10m
+
+    class Meta:  # type: ignore[reportIncompatibleVariableOverride]
+        model = Feed
+
+
+class Admin1Factory(DjangoModelFactory):
+    ifrc_go_id = factory.Sequence(lambda n: n)
+    name = factory.Sequence(lambda n: f'Admin1-{n}')
+
+    class Meta:  # type: ignore[reportIncompatibleVariableOverride]
+        model = Admin1
+
+
+class AlertFactory(DjangoModelFactory):
+    url = factory.Sequence(lambda n: f"https://alert-{n}.com/test")
+    identifier = "Identifier-X"
+    sender = "Sender-X"
+    sent = datetime.datetime(year=2024, month=1, day=1)
+    status = Alert.Status.ACTUAL
+    msg_type = Alert.MsgType.ALERT
+
+    class Meta:  # type: ignore[reportIncompatibleVariableOverride]
+        model = Alert
+
+    @factory.post_generation
+    def admin1s(self, create, extracted, **_):
+        if not create:
+            return
+        if extracted:
+            for author in extracted:
+                self.admin1s.add(author)  # type: ignore[reportAttributeAccessIssue]
+
+
+class AlertInfoFactory(DjangoModelFactory):
+    event = "Event-X"
+    category = AlertInfo.Category.HEALTH
+    urgency = AlertInfo.Urgency.IMMEDIATE
+    severity = AlertInfo.Severity.EXTREME
+    certainty = AlertInfo.Certainty.OBSERVED
+
+    class Meta:  # type: ignore[reportIncompatibleVariableOverride]
+        model = AlertInfo

apps/cap_feed/filters.py

@@ -12,6 +12,7 @@
 from .models import Admin1, Alert, AlertInfo, Country, Feed, Region
 
 
+# NOTE: Make sure to sync changes here with apps/subscription/serializers.py:UserAlertSubscriptionFilterSerializer
 @strawberry_django.filters.filter(Alert, lookups=True)
 class AlertFilter:
     id: strawberry.auto

apps/cap_feed/migrations/0009_alert_is_processed_by_subscription.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.13 on 2024-11-24 09:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('cap_feed', '0008_alert_cap_feed_alert_not_expired_idx'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='alert',
+            name='is_processed_by_subscription',
+            field=models.BooleanField(null=True),
+        ),
+    ]

apps/cap_feed/migrations/0010_alter_alert_is_processed_by_subscription_and_more.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2.13 on 2024-11-24 09:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('cap_feed', '0009_alert_is_processed_by_subscription'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='alert',
+            name='is_processed_by_subscription',
+            field=models.BooleanField(default=False, null=True),
+        ),
+        migrations.AddIndex(
+            model_name='alert',
+            index=models.Index(condition=models.Q(('is_processed_by_subscription', False)), fields=['is_processed_by_subscription'], name='cap_feed_alert_subscription_ix'),
+        ),
+    ]

apps/cap_feed/models.py

@@ -191,6 +191,9 @@ class Scope(models.TextChoices):  # XXX: Not used, maybe we need to use this in
 
     # This is updated by the system to filter out is_expired
     is_expired = models.BooleanField(default=False)
+    # TODO: Keep this true for existing alerts and then default=True for future alerts
+    # NOTE: null=True is to avoid full rewrite of the table: https://docs.djangoproject.com/en/5.1/ref/migration-operations/#addfield  # noqa
+    is_processed_by_subscription = models.BooleanField(default=False, null=True)
 
     identifier = models.CharField()
     sender = models.CharField()
@@ -219,7 +222,12 @@ class Meta:  # type: ignore [reportIncompatibleVariableOverride]
                 fields=['is_expired'],
                 name='%(app_label)s_%(class)s_not_expired_idx',
                 condition=models.Q(is_expired=False),
-            )
+            ),
+            models.Index(
+                fields=['is_processed_by_subscription'],
+                name='%(app_label)s_%(class)s_subscription_ix',
+                condition=models.Q(is_processed_by_subscription=False),
+            ),
         ]
 
     def __init__(self, *args, **kwargs):

apps/cap_feed/types.py

@@ -140,7 +140,7 @@ class Admin1Type:
     id: strawberry.ID
     ifrc_go_id: strawberry.ID | None
     name = string_field(Admin1.name)
-    bbox: PolygonScalar | None
+    bbox: PolygonScalar | None  # XXX: Use dataloader instead?
 
     if typing.TYPE_CHECKING:
         country_id = Admin1.country_id

apps/subscription/admin.py

@@ -0,0 +1,22 @@
+from admin_auto_filters.filters import AutocompleteFilterFactory
+from django.contrib import admin
+
+from .models import SubscriptionAlert, UserAlertSubscription
+
+
+@admin.register(UserAlertSubscription)
+class UserAlertSubscriptionAdmin(admin.ModelAdmin):
+    list_display = ["name", "is_active"]
+    search_fields = ("name",)
+    autocomplete_fields = ("user",)
+    list_filter = (
+        AutocompleteFilterFactory("User", "user"),
+        "is_active",
+    )
+
+
+@admin.register(SubscriptionAlert)
+class SubscriptionAlertAdmin(admin.ModelAdmin):
+    list_display = ["subscription", "alert"]
+    autocomplete_fields = ["subscription", "alert"]
+    list_filter = (AutocompleteFilterFactory("Subscription", "subscription"),)

apps/subscription/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class SubscriptionConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'apps.subscription'

apps/subscription/emails.py

@@ -0,0 +1,92 @@
+from datetime import timedelta
+
+from django.db import models
+from django.utils import timezone
+from django.utils.encoding import force_bytes
+from django.utils.http import urlsafe_base64_encode
+
+from apps.subscription.models import SubscriptionAlert, UserAlertSubscription
+from apps.user.models import EmailNotificationType, User
+from main.permalinks import Permalink
+from main.tokens import TokenManager
+from utils.emails import send_email
+
+
+def generate_unsubscribe_user_alert_subscription_url(subscription: UserAlertSubscription) -> str:
+    uid = urlsafe_base64_encode(force_bytes(subscription.pk))
+    # TODO: Fix typing
+    token = TokenManager.user_subscription_unsubscribe_generator.make_token(subscription)  # type: ignore[reportArgumentType]
+    return Permalink.unsubscribe_user_alert_subscription(uid, token)
+
+
+def generate_user_alert_subscription_email_context(
+    user: User,
+    email_frequency: UserAlertSubscription.EmailFrequency,
+) -> tuple[dict, models.QuerySet[UserAlertSubscription]]:
+    # NOTE: Number of subscription is static and less than UserAlertSubscription.LIMIT_PER_USER
+    subscription_qs = UserAlertSubscription.objects.filter(user=user, email_frequency=email_frequency)
+
+    if email_frequency == UserAlertSubscription.EmailFrequency.DAILY:
+        from_datetime_threshold = timezone.now() - timedelta(hours=24)
+    elif email_frequency == UserAlertSubscription.EmailFrequency.WEEKLY:
+        from_datetime_threshold = timezone.now() - timedelta(days=7)
+    elif email_frequency == UserAlertSubscription.EmailFrequency.MONTHLY:
+        # TODO: Calculate days instead of using 30 days
+        from_datetime_threshold = timezone.now() - timedelta(days=30)
+
+    subscription_data = [
+        {
+            'subscription': subscription,
+            'unsubscribe_url': generate_unsubscribe_user_alert_subscription_url(subscription),
+            'latest_alerts': [
+                subscription_alert.alert
+                # NOTE: N+1 query, but N < 10 for now
+                # TODO: Index/partition alert__sent column?
+                for subscription_alert in (
+                    SubscriptionAlert.objects.select_related('alert')
+                    .filter(
+                        subscription=subscription,
+                        alert__sent__gte=from_datetime_threshold,
+                    )
+                    .order_by('-alert__sent')[:5]
+                )
+            ],
+        }
+        for subscription in subscription_qs
+    ]
+
+    context = {
+        'subscriptions': subscription_data,
+    }
+
+    return context, subscription_qs
+
+
+def send_user_alert_subscription_email(user: User, email_frequency: UserAlertSubscription.EmailFrequency):
+    context, subscription_qs = generate_user_alert_subscription_email_context(user, email_frequency)
+    sent_at = timezone.now()
+
+    send_email(
+        user=user,
+        email_type=EmailNotificationType.ALERT_SUBSCRIPTIONS,
+        subject="Daily Alerts",  # TODO: Is this fine?
+        email_html_template='emails/subscription/body.html',
+        email_text_template='emails/subscription/body.txt',
+        context=context,
+    )
+
+    # Post action
+    subscription_qs.update(email_last_sent_at=sent_at)
+
+
+def send_user_alert_subscriptions_email(email_frequency: UserAlertSubscription.EmailFrequency):
+    # TODO: Send in parallel if email service supports it
+    users_qs = User.objects.filter(
+        id__in=UserAlertSubscription.objects.filter(email_frequency=email_frequency).values('user'),
+    )
+
+    # TODO: Handle failure
+    for user in users_qs.iterator():
+        # TODO: Trigger this as cronjob
+        # TODO: Pass timezone.now for ref time
+        send_user_alert_subscription_email(user, email_frequency)

apps/subscription/enums.py

@@ -0,0 +1,28 @@
+import strawberry
+
+from apps.cap_feed.enums import (
+    AlertInfoCategoryEnum,
+    AlertInfoCertaintyEnum,
+    AlertInfoSeverityEnum,
+    AlertInfoUrgencyEnum,
+)
+from utils.strawberry.enums import get_enum_name_from_django_field
+
+from .models import UserAlertSubscription
+
+UserAlertSubscriptionEmailFrequencyEnum = strawberry.enum(
+    UserAlertSubscription.EmailFrequency, name='UserAlertSubscriptionEmailFrequencyEnum'
+)
+
+
+enum_map = {
+    get_enum_name_from_django_field(field): enum
+    for field, enum in (
+        (UserAlertSubscription.email_frequency, UserAlertSubscriptionEmailFrequencyEnum),
+        # Filters
+        (UserAlertSubscription.filter_alert_urgencies, AlertInfoUrgencyEnum),
+        (UserAlertSubscription.filter_alert_severities, AlertInfoSeverityEnum),
+        (UserAlertSubscription.filter_alert_certainties, AlertInfoCertaintyEnum),
+        (UserAlertSubscription.filter_alert_categories, AlertInfoCategoryEnum),
+    )
+}

apps/subscription/factories.py

@@ -0,0 +1,16 @@
+import factory
+from factory.django import DjangoModelFactory
+
+from .models import UserAlertSubscription
+
+
+class UserAlertSubscriptionFactory(DjangoModelFactory):
+    name = factory.Sequence(lambda n: f'Subscription-{n}')
+    filter_alert_admin1s = []
+    filter_alert_urgencies = []
+    filter_alert_severities = []
+    filter_alert_certainties = []
+    filter_alert_categories = []
+
+    class Meta:  # type: ignore[reportIncompatibleVariableOverride]
+        model = UserAlertSubscription

apps/subscription/filters.py

@@ -0,0 +1,11 @@
+import strawberry
+import strawberry_django
+
+from .models import UserAlertSubscription
+
+
+@strawberry_django.filters.filter(UserAlertSubscription, lookups=True)
+class UserAlertSubscriptionFilter:
+    id: strawberry.auto
+    is_active: strawberry.auto
+    notify_by_email: strawberry.auto