Feature/whitenoise

.gitignore

@@ -32,6 +32,7 @@ MANIFEST
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
 *.spec
+.aider*
 
 # Installer logs
 pip-log.txt

.pre-commit-config.yaml

@@ -31,6 +31,6 @@ repos:
       - id: flake8
 
   - repo: https://github.com/RobertCraigie/pyright-python
-    rev: v1.1.364
+    rev: v1.1.389
     hooks:
       - id: pyright

Dockerfile

@@ -22,12 +22,13 @@ RUN apt-get update -y \
     && poetry --version \
     # Configure to use system instead of virtualenvs
     && poetry config virtualenvs.create false \
-    && poetry install --no-root \
+    && poetry install --no-root --no-cache --no-interaction \
     # Clean-up
     && pip uninstall -y poetry virtualenv-clone virtualenv \
     && apt-get remove -y gcc libc-dev libproj-dev \
     && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /root/.cache/
 
 
 COPY . /code/

apps/cap_feed/admin.py

@@ -153,6 +153,7 @@ class FeedAdmin(admin.ModelAdmin):
     list_filter = (
         'format',
         'country__region',
+        'enable_polling',
         AutocompleteFilterFactory('Country', 'country'),
     )
     search_fields = ['url']

apps/cap_feed/dataloaders.py

@@ -63,7 +63,23 @@ def load_feed(keys: list[int]) -> list['FeedType']:
     return _load_model(Feed, keys)  # type: ignore[reportGeneralTypeIssues]
 
 
-def load_admin1_by_alert(keys: list[int]) -> list[list['Admin1Type']]:
+def load_admin1_by_admin1s(keys_array: list[tuple[int]]) -> list[list['Admin1Type']]:
+    keys = [key for keys in keys_array for key in keys]
+    qs = Admin1.objects.filter(id__in=keys)
+
+    _map = defaultdict(list)
+    admin1_map = {admin1.pk: admin1 for admin1 in qs.all()}
+
+    for keys in keys_array:
+        for key in keys:
+            if key not in admin1_map:
+                continue
+            _map[keys].append(admin1_map[key])
+
+    return [_map[keys] for keys in keys_array]
+
+
+def load_admin1s_by_alert(keys: list[int]) -> list[list['Admin1Type']]:
     qs = (
         AlertAdmin1.objects.filter(alert__in=keys)
         .order_by()
@@ -228,9 +244,13 @@ def load_continent(self):
     def load_feed(self):
         return DataLoader(load_fn=sync_to_async(load_feed))
 
+    @cached_property
+    def load_admin1_by_admin1s(self):
+        return DataLoader(load_fn=sync_to_async(load_admin1_by_admin1s))
+
     @cached_property
     def load_admin1s_by_alert(self):
-        return DataLoader(load_fn=sync_to_async(load_admin1_by_alert))
+        return DataLoader(load_fn=sync_to_async(load_admin1s_by_alert))
 
     @cached_property
     def load_admin1s_by_country(self):

apps/cap_feed/factories.py

@@ -0,0 +1,71 @@
+import datetime
+
+import factory
+from factory.django import DjangoModelFactory
+
+from .models import Admin1, Alert, AlertInfo, Country, Feed, Region
+
+
+class RegionFactory(DjangoModelFactory):
+    ifrc_go_id = factory.Sequence(lambda n: n)
+    name = factory.Sequence(lambda n: f'Region-{n}')
+
+    class Meta:  # type: ignore[reportIncompatibleVariableOverride]
+        model = Region
+
+
+class CountryFactory(DjangoModelFactory):
+    ifrc_go_id = factory.Sequence(lambda n: n)
+    name = factory.Sequence(lambda n: f'Country-{n}')
+    iso3 = factory.Sequence(lambda n: f"{n:0>3}")
+
+    class Meta:  # type: ignore[reportIncompatibleVariableOverride]
+        model = Country
+
+
+class FeedFactory(DjangoModelFactory):
+    url = factory.Sequence(lambda n: f"https://source-{n}.com/test")
+    format = Feed.Format.RSS
+    polling_interval = Feed.PoolingInterval.I_10m
+
+    class Meta:  # type: ignore[reportIncompatibleVariableOverride]
+        model = Feed
+
+
+class Admin1Factory(DjangoModelFactory):
+    ifrc_go_id = factory.Sequence(lambda n: n)
+    name = factory.Sequence(lambda n: f'Admin1-{n}')
+
+    class Meta:  # type: ignore[reportIncompatibleVariableOverride]
+        model = Admin1
+
+
+class AlertFactory(DjangoModelFactory):
+    url = factory.Sequence(lambda n: f"https://alert-{n}.com/test")
+    identifier = "Identifier-X"
+    sender = "Sender-X"
+    sent = datetime.datetime(year=2024, month=1, day=1)
+    status = Alert.Status.ACTUAL
+    msg_type = Alert.MsgType.ALERT
+
+    class Meta:  # type: ignore[reportIncompatibleVariableOverride]
+        model = Alert
+
+    @factory.post_generation
+    def admin1s(self, create, extracted, **_):
+        if not create:
+            return
+        if extracted:
+            for author in extracted:
+                self.admin1s.add(author)  # type: ignore[reportAttributeAccessIssue]
+
+
+class AlertInfoFactory(DjangoModelFactory):
+    event = "Event-X"
+    category = AlertInfo.Category.HEALTH
+    urgency = AlertInfo.Urgency.IMMEDIATE
+    severity = AlertInfo.Severity.EXTREME
+    certainty = AlertInfo.Certainty.OBSERVED
+
+    class Meta:  # type: ignore[reportIncompatibleVariableOverride]
+        model = AlertInfo

apps/cap_feed/filters.py

@@ -12,40 +12,15 @@
 from .models import Admin1, Alert, AlertInfo, Country, Feed, Region
 
 
-@strawberry_django.filters.filter(Alert, lookups=True)
-class AlertFilter:
+@strawberry_django.filters.filter(AlertInfo, lookups=True)
+class AlertInfoFilter:
     id: strawberry.auto
-    country: strawberry.auto
-    sent: strawberry.auto
-
-    @strawberry_django.filter_field
-    def region(
-        self,
-        queryset: models.QuerySet,
-        value: strawberry.ID,
-        prefix: str,
-    ) -> tuple[models.QuerySet, models.Q]:
-        return queryset, models.Q(**{f"{prefix}country__region": value})
-
-    @strawberry_django.filter_field
-    def admin1(
-        self,
-        queryset: models.QuerySet,
-        value: strawberry.ID,
-        prefix: str,
-    ) -> tuple[models.QuerySet, models.Q]:
-        return queryset, models.Q(**{f"{prefix}admin1s": value})
 
     def _info_enum_fields(self, field, queryset, value, prefix) -> tuple[models.QuerySet, models.Q]:
         if value:
-            alias_field = f"_infos_{field}_list"
-            queryset = queryset.alias(
-                **{
-                    # NOTE: To avoid duplicate alerts when joining infos
-                    alias_field: ArrayAgg(f"{prefix}infos__{field}"),
-                }
-            )
-            return queryset, models.Q(**{f"{prefix}{alias_field}__overlap": value})
+            # NOTE: With this field, disctinct should be used by the client
+            print(f"{prefix}{field}__in")
+            return queryset, models.Q(**{f"{prefix}{field}__in": value})
         return queryset, models.Q()
 
     @strawberry_django.filter_field
@@ -85,9 +60,30 @@ def category(
         return self._info_enum_fields("category", queryset, value, prefix)
 
 
-@strawberry_django.filters.filter(AlertInfo, lookups=True)
-class AlertInfoFilter:
+@strawberry_django.filters.filter(Alert, lookups=True)
+class AlertFilter:
     id: strawberry.auto
+    country: strawberry.auto
+    sent: strawberry.auto
+    infos: AlertInfoFilter | None
+
+    @strawberry_django.filter_field
+    def region(
+        self,
+        queryset: models.QuerySet,
+        value: strawberry.ID,
+        prefix: str,
+    ) -> tuple[models.QuerySet, models.Q]:
+        return queryset, models.Q(**{f"{prefix}country__region": value})
+
+    @strawberry_django.filter_field
+    def admin1(
+        self,
+        queryset: models.QuerySet,
+        value: strawberry.ID,
+        prefix: str,
+    ) -> tuple[models.QuerySet, models.Q]:
+        return queryset, models.Q(**{f"{prefix}admin1s": value})
 
 
 @strawberry_django.filters.filter(Feed, lookups=True)

apps/cap_feed/fixtures/test_data.json

@@ -18,7 +18,8 @@
       "pk": 1,
       "fields": {
         "name": "test_country",
-        "iso3": "ISO"
+        "iso3": "ISO",
+        "region": 1
       }
     },
     {

apps/cap_feed/migrations/0009_alert_is_processed_by_subscription.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.13 on 2024-11-24 09:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('cap_feed', '0008_alert_cap_feed_alert_not_expired_idx'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='alert',
+            name='is_processed_by_subscription',
+            field=models.BooleanField(null=True),
+        ),
+    ]

apps/cap_feed/migrations/0010_alter_alert_is_processed_by_subscription_and_more.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2.13 on 2024-11-24 09:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('cap_feed', '0009_alert_is_processed_by_subscription'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='alert',
+            name='is_processed_by_subscription',
+            field=models.BooleanField(default=False, null=True),
+        ),
+        migrations.AddIndex(
+            model_name='alert',
+            index=models.Index(condition=models.Q(('is_processed_by_subscription', False)), fields=['is_processed_by_subscription'], name='cap_feed_alert_subscription_ix'),
+        ),
+    ]

apps/cap_feed/migrations/0011_alter_alertinfo_category_alter_alertinfo_certainty_and_more.py

@@ -0,0 +1,33 @@
+# Generated by Django 4.2.13 on 2024-11-26 12:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('cap_feed', '0010_alter_alert_is_processed_by_subscription_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='alertinfo',
+            name='category',
+            field=models.CharField(choices=[('Geo', 'Geo'), ('Met', 'Met'), ('Safety', 'Safety'), ('Security', 'Security'), ('Rescue', 'Rescue'), ('Fire', 'Fire'), ('Health', 'Health'), ('Env', 'Env'), ('Transport', 'Transport'), ('Infra', 'Infra'), ('CBRNE', 'CBRNE'), ('Other', 'Other')], db_index=True),
+        ),
+        migrations.AlterField(
+            model_name='alertinfo',
+            name='certainty',
+            field=models.CharField(choices=[('Observed', 'Observed'), ('Likely', 'Likely'), ('Possible', 'Possible'), ('Unlikely', 'Unlikely'), ('Unknown', 'Unknown')], db_index=True),
+        ),
+        migrations.AlterField(
+            model_name='alertinfo',
+            name='severity',
+            field=models.CharField(choices=[('Extreme', 'Extreme'), ('Severe', 'Severe'), ('Moderate', 'Moderate'), ('Minor', 'Minor'), ('Unknown', 'Unknown')], db_index=True),
+        ),
+        migrations.AlterField(
+            model_name='alertinfo',
+            name='urgency',
+            field=models.CharField(choices=[('Immediate', 'Immediate'), ('Expected', 'Expected'), ('Future', 'Future'), ('Past', 'Past'), ('Unknown', 'Unknown')], db_index=True),
+        ),
+    ]

apps/cap_feed/models.py

@@ -191,6 +191,9 @@ class Scope(models.TextChoices):  # XXX: Not used, maybe we need to use this in
 
     # This is updated by the system to filter out is_expired
     is_expired = models.BooleanField(default=False)
+    # TODO: Keep this true for existing alerts and then default=True for future alerts
+    # NOTE: null=True is to avoid full rewrite of the table: https://docs.djangoproject.com/en/5.1/ref/migration-operations/#addfield  # noqa
+    is_processed_by_subscription = models.BooleanField(default=False, null=True)
 
     identifier = models.CharField()
     sender = models.CharField()
@@ -219,7 +222,12 @@ class Meta:  # type: ignore [reportIncompatibleVariableOverride]
                 fields=['is_expired'],
                 name='%(app_label)s_%(class)s_not_expired_idx',
                 condition=models.Q(is_expired=False),
-            )
+            ),
+            models.Index(
+                fields=['is_processed_by_subscription'],
+                name='%(app_label)s_%(class)s_subscription_ix',
+                condition=models.Q(is_processed_by_subscription=False),
+            ),
         ]
 
     def __init__(self, *args, **kwargs):
@@ -300,12 +308,12 @@ class Certainty(models.TextChoices):
     alert = models.ForeignKey(Alert, on_delete=models.CASCADE, related_name='infos')
 
     language = models.CharField(blank=True, default='en-US')
-    category = models.CharField(choices=Category.choices)
+    category = models.CharField(choices=Category.choices, db_index=True)
     event = models.CharField()
     response_type = models.CharField(choices=ResponseType.choices, blank=True, null=True, default=None)
-    urgency = models.CharField(choices=Urgency.choices)
-    severity = models.CharField(choices=Severity.choices)
-    certainty = models.CharField(choices=Certainty.choices)
+    urgency = models.CharField(choices=Urgency.choices, db_index=True)
+    severity = models.CharField(choices=Severity.choices, db_index=True)
+    certainty = models.CharField(choices=Certainty.choices, db_index=True)
     audience = models.CharField(blank=True, null=True, default=None)
     event_code = models.CharField(blank=True, null=True, default=None)
     # effective = models.DateTimeField(default=Alert.objects.get(pk=alert).sent)

apps/cap_feed/queries.py

@@ -1,10 +1,17 @@
+import typing
+
 import strawberry
 import strawberry_django
 from django.db import models
 from strawberry_django.filters import apply as apply_filters
+from strawberry_django.pagination import OffsetPaginationInput
 
 from main.graphql.context import Info
-from utils.strawberry.paginations import CountList, pagination_field
+from utils.strawberry.paginations import (
+    CountList,
+    count_list_resolver,
+    pagination_field,
+)
 
 from .filters import (
     Admin1Filter,
@@ -138,6 +145,22 @@ async def alert(self, info: Info, pk: strawberry.ID) -> AlertType | None:
     async def alert_info(self, info: Info, pk: strawberry.ID) -> AlertInfoType | None:
         return await AlertInfoType.get_queryset(None, None, info).filter(pk=pk).afirst()
 
+    @strawberry_django.field
+    async def historical_alerts(
+        self,
+        info: Info,
+        filters: typing.Optional[AlertFilter] = strawberry.UNSET,
+        pagination: typing.Optional[OffsetPaginationInput] = strawberry.UNSET,
+    ) -> CountList[AlertType]:
+        queryset = get_alert_queryset(None, is_active=False)
+        return count_list_resolver(
+            info,
+            queryset,
+            AlertType,
+            filters=filters,  # type: ignore[reportArgumentType]
+            pagination=pagination,  # type: ignore[reportArgumentType]
+        )
+
 
 @strawberry.type
 class PrivateQuery: