From 67a36af11b0b89a1ff7f8e3e40d35e9468fa0abc Mon Sep 17 00:00:00 2001 From: Navin Ayer Date: Wed, 27 Nov 2024 22:16:15 +0545 Subject: [PATCH] Add index for historical data (#101) - Change filter structure to use index - Fix ID-name issues for filterAlertCountry - Add historical alerts node --- apps/cap_feed/filters.py | 61 +++++++++---------- ...gory_alter_alertinfo_certainty_and_more.py | 33 ++++++++++ apps/cap_feed/models.py | 8 +-- apps/cap_feed/queries.py | 25 +++++++- apps/subscription/queries.py | 11 +++- apps/subscription/tests/test_mutations.py | 8 +-- schema.graphql | 10 +-- 7 files changed, 107 insertions(+), 49 deletions(-) create mode 100644 apps/cap_feed/migrations/0011_alter_alertinfo_category_alter_alertinfo_certainty_and_more.py diff --git a/apps/cap_feed/filters.py b/apps/cap_feed/filters.py index 908b10bb..7928b520 100644 --- a/apps/cap_feed/filters.py +++ b/apps/cap_feed/filters.py @@ -12,41 +12,15 @@ from .models import Admin1, Alert, AlertInfo, Country, Feed, Region -# NOTE: Make sure to sync changes here with apps/subscription/serializers.py:UserAlertSubscriptionFilterSerializer -@strawberry_django.filters.filter(Alert, lookups=True) -class AlertFilter: +@strawberry_django.filters.filter(AlertInfo, lookups=True) +class AlertInfoFilter: id: strawberry.auto - country: strawberry.auto - sent: strawberry.auto - - @strawberry_django.filter_field - def region( - self, - queryset: models.QuerySet, - value: strawberry.ID, - prefix: str, - ) -> tuple[models.QuerySet, models.Q]: - return queryset, models.Q(**{f"{prefix}country__region": value}) - - @strawberry_django.filter_field - def admin1( - self, - queryset: models.QuerySet, - value: strawberry.ID, - prefix: str, - ) -> tuple[models.QuerySet, models.Q]: - return queryset, models.Q(**{f"{prefix}admin1s": value}) def _info_enum_fields(self, field, queryset, value, prefix) -> tuple[models.QuerySet, models.Q]: if value: - alias_field = f"_infos_{field}_list" - queryset = queryset.alias( - **{ - # NOTE: To avoid duplicate alerts when joining infos - alias_field: ArrayAgg(f"{prefix}infos__{field}"), - } - ) - return queryset, models.Q(**{f"{prefix}{alias_field}__overlap": value}) + # NOTE: With this field, disctinct should be used by the client + print(f"{prefix}{field}__in") + return queryset, models.Q(**{f"{prefix}{field}__in": value}) return queryset, models.Q() @strawberry_django.filter_field @@ -86,9 +60,30 @@ def category( return self._info_enum_fields("category", queryset, value, prefix) -@strawberry_django.filters.filter(AlertInfo, lookups=True) -class AlertInfoFilter: +@strawberry_django.filters.filter(Alert, lookups=True) +class AlertFilter: id: strawberry.auto + country: strawberry.auto + sent: strawberry.auto + infos: AlertInfoFilter | None + + @strawberry_django.filter_field + def region( + self, + queryset: models.QuerySet, + value: strawberry.ID, + prefix: str, + ) -> tuple[models.QuerySet, models.Q]: + return queryset, models.Q(**{f"{prefix}country__region": value}) + + @strawberry_django.filter_field + def admin1( + self, + queryset: models.QuerySet, + value: strawberry.ID, + prefix: str, + ) -> tuple[models.QuerySet, models.Q]: + return queryset, models.Q(**{f"{prefix}admin1s": value}) @strawberry_django.filters.filter(Feed, lookups=True) diff --git a/apps/cap_feed/migrations/0011_alter_alertinfo_category_alter_alertinfo_certainty_and_more.py b/apps/cap_feed/migrations/0011_alter_alertinfo_category_alter_alertinfo_certainty_and_more.py new file mode 100644 index 00000000..11d2e7af --- /dev/null +++ b/apps/cap_feed/migrations/0011_alter_alertinfo_category_alter_alertinfo_certainty_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.13 on 2024-11-26 12:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('cap_feed', '0010_alter_alert_is_processed_by_subscription_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='alertinfo', + name='category', + field=models.CharField(choices=[('Geo', 'Geo'), ('Met', 'Met'), ('Safety', 'Safety'), ('Security', 'Security'), ('Rescue', 'Rescue'), ('Fire', 'Fire'), ('Health', 'Health'), ('Env', 'Env'), ('Transport', 'Transport'), ('Infra', 'Infra'), ('CBRNE', 'CBRNE'), ('Other', 'Other')], db_index=True), + ), + migrations.AlterField( + model_name='alertinfo', + name='certainty', + field=models.CharField(choices=[('Observed', 'Observed'), ('Likely', 'Likely'), ('Possible', 'Possible'), ('Unlikely', 'Unlikely'), ('Unknown', 'Unknown')], db_index=True), + ), + migrations.AlterField( + model_name='alertinfo', + name='severity', + field=models.CharField(choices=[('Extreme', 'Extreme'), ('Severe', 'Severe'), ('Moderate', 'Moderate'), ('Minor', 'Minor'), ('Unknown', 'Unknown')], db_index=True), + ), + migrations.AlterField( + model_name='alertinfo', + name='urgency', + field=models.CharField(choices=[('Immediate', 'Immediate'), ('Expected', 'Expected'), ('Future', 'Future'), ('Past', 'Past'), ('Unknown', 'Unknown')], db_index=True), + ), + ] diff --git a/apps/cap_feed/models.py b/apps/cap_feed/models.py index bdd170ca..47d4aac6 100644 --- a/apps/cap_feed/models.py +++ b/apps/cap_feed/models.py @@ -308,12 +308,12 @@ class Certainty(models.TextChoices): alert = models.ForeignKey(Alert, on_delete=models.CASCADE, related_name='infos') language = models.CharField(blank=True, default='en-US') - category = models.CharField(choices=Category.choices) + category = models.CharField(choices=Category.choices, db_index=True) event = models.CharField() response_type = models.CharField(choices=ResponseType.choices, blank=True, null=True, default=None) - urgency = models.CharField(choices=Urgency.choices) - severity = models.CharField(choices=Severity.choices) - certainty = models.CharField(choices=Certainty.choices) + urgency = models.CharField(choices=Urgency.choices, db_index=True) + severity = models.CharField(choices=Severity.choices, db_index=True) + certainty = models.CharField(choices=Certainty.choices, db_index=True) audience = models.CharField(blank=True, null=True, default=None) event_code = models.CharField(blank=True, null=True, default=None) # effective = models.DateTimeField(default=Alert.objects.get(pk=alert).sent) diff --git a/apps/cap_feed/queries.py b/apps/cap_feed/queries.py index fb1ca2a0..31ac83a4 100644 --- a/apps/cap_feed/queries.py +++ b/apps/cap_feed/queries.py @@ -1,10 +1,17 @@ +import typing + import strawberry import strawberry_django from django.db import models from strawberry_django.filters import apply as apply_filters +from strawberry_django.pagination import OffsetPaginationInput from main.graphql.context import Info -from utils.strawberry.paginations import CountList, pagination_field +from utils.strawberry.paginations import ( + CountList, + count_list_resolver, + pagination_field, +) from .filters import ( Admin1Filter, @@ -138,6 +145,22 @@ async def alert(self, info: Info, pk: strawberry.ID) -> AlertType | None: async def alert_info(self, info: Info, pk: strawberry.ID) -> AlertInfoType | None: return await AlertInfoType.get_queryset(None, None, info).filter(pk=pk).afirst() + @strawberry_django.field + async def historical_alerts( + self, + info: Info, + filters: typing.Optional[AlertFilter] = strawberry.UNSET, + pagination: typing.Optional[OffsetPaginationInput] = strawberry.UNSET, + ) -> CountList[AlertType]: + queryset = get_alert_queryset(None, is_active=False) + return count_list_resolver( + info, + queryset, + AlertType, + filters=filters, # type: ignore[reportArgumentType] + pagination=pagination, # type: ignore[reportArgumentType] + ) + @strawberry.type class PrivateQuery: diff --git a/apps/subscription/queries.py b/apps/subscription/queries.py index 12694f0f..4b9c8aa1 100644 --- a/apps/subscription/queries.py +++ b/apps/subscription/queries.py @@ -6,7 +6,7 @@ from apps.cap_feed.filters import AlertFilter from apps.cap_feed.orders import AlertOrder -from apps.cap_feed.types import AlertType +from apps.cap_feed.types import AlertType, get_alert_queryset from main.graphql.context import Info from utils.strawberry.paginations import ( CountList, @@ -40,10 +40,15 @@ async def subscripted_alerts( order: typing.Optional[AlertOrder] = strawberry.UNSET, pagination: typing.Optional[OffsetPaginationInput] = strawberry.UNSET, ) -> CountList[AlertType]: - queryset = AlertType.get_queryset(None, None, info).filter( + # XXX: Add DISTINCT as default to avoid duplicate alerts + if filters is strawberry.UNSET: + filters = AlertFilter(DISTINCT=True) # type: ignore[reportCallIssue] + else: + filters.DISTINCT = True # type: ignore[reportCallIssue] + + queryset = get_alert_queryset(None, is_active=False).filter( subscriptions__in=UserAlertSubscription.objects.filter(user=info.context.request.user).all(), ) - # TODO: Handle duplicates from filters(frontend side) or manually return count_list_resolver( info, queryset, diff --git a/apps/subscription/tests/test_mutations.py b/apps/subscription/tests/test_mutations.py index f2525943..155f5d67 100644 --- a/apps/subscription/tests/test_mutations.py +++ b/apps/subscription/tests/test_mutations.py @@ -31,8 +31,8 @@ class Mutation: # Filters # -- ForeignKey - filterAlertCountry - filterAlertCountryDisplay { + filterAlertCountryId + filterAlertCountry { id name } @@ -78,8 +78,8 @@ class Mutation: # Filters # -- ForeignKey - filterAlertCountry - filterAlertCountryDisplay { + filterAlertCountryId + filterAlertCountry { id name } diff --git a/schema.graphql b/schema.graphql index 3f57b6f1..a40cb832 100644 --- a/schema.graphql +++ b/schema.graphql @@ -34,16 +34,13 @@ input AlertFilter { id: IDBaseFilterLookup country: DjangoModelFilterInput sent: DatetimeDatetimeFilterLookup + infos: AlertInfoFilter AND: AlertFilter OR: AlertFilter NOT: AlertFilter DISTINCT: Boolean region: ID admin1: ID - urgency: [AlertInfoUrgencyEnum!] - severity: [AlertInfoSeverityEnum!] - certainty: [AlertInfoCertaintyEnum!] - category: [AlertInfoCategoryEnum!] } type AlertInfoAreaCircleType { @@ -106,6 +103,10 @@ input AlertInfoFilter { OR: AlertInfoFilter NOT: AlertInfoFilter DISTINCT: Boolean + urgency: [AlertInfoUrgencyEnum!] + severity: [AlertInfoSeverityEnum!] + certainty: [AlertInfoCertaintyEnum!] + category: [AlertInfoCategoryEnum!] } input AlertInfoOrder { @@ -644,6 +645,7 @@ type PublicQuery { feed(pk: ID!): FeedType alert(pk: ID!): AlertType alertInfo(pk: ID!): AlertInfoType + historicalAlerts(filters: AlertFilter, pagination: OffsetPaginationInput): AlertTypeCountList! id: ID! }