fix export and mail (#236)

vespadb/observations/admin.py

@@ -18,6 +18,7 @@
 from django.utils.timezone import now
 from django.utils.translation import gettext_lazy as _
 from rest_framework.test import APIRequestFactory
+from django.conf import settings
 
 from vespadb.observations.filters import MunicipalityExcludeFilter, ObserverReceivedEmailFilter, ProvinceFilter
 from vespadb.observations.forms import SendEmailForm
@@ -305,7 +306,7 @@ def send_email_view(self, request: HttpRequest) -> HttpResponse:
                         fail_list.append(observation.id)
                         continue
                     try:
-                        send_mail(subject, message, "[email protected]", [observation.observer_email])
+                        send_mail(subject, message, settings.DEFAULT_FROM_EMAIL, [observation.observer_email])
                         logger.debug(f"Email sent to {observation.observer_email} for observation {observation.id}")
                         observation.observer_received_email = True
                         observation.save()

vespadb/observations/utils.py

@@ -1,7 +1,12 @@
 """Utility functions for the observations app."""
 
 from django.contrib.gis.geos import Point
+import time
+from functools import wraps
+from django.db import connection, OperationalError
+from typing import Callable, TypeVar, Any, cast
 
+F = TypeVar("F", bound=Callable[..., Any])
 
 def get_municipality_from_coordinates(longitude: float, latitude: float):  # type: ignore[no-untyped-def]
     """Get the municipality for a given long and lat."""
@@ -24,3 +29,29 @@ def check_if_point_in_anb_area(longitude: float, latitude: float) -> bool:
 
     anb_areas_containing_point = ANB.objects.filter(polygon__contains=point_to_check)
     return bool(anb_areas_containing_point)
+
+def db_retry(retries: int = 3, delay: int = 5) -> Callable[[F], F]:
+    """
+    Decorator to retry a database operation in case of an OperationalError.
+
+    Args:
+        retries (int): Number of retry attempts. Defaults to 3.
+        delay (int): Delay between retries in seconds. Defaults to 5.
+
+    Returns:
+        Callable: The wrapped function with retry logic.
+    """
+    def decorator(func: F) -> F:
+        @wraps(func)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            for attempt in range(retries):
+                try:
+                    return func(*args, **kwargs)
+                except OperationalError:
+                    if attempt < retries - 1:
+                        time.sleep(delay)
+                        connection.close()
+                    else:
+                        raise
+        return cast(F, wrapper)
+    return decorator

vespadb/observations/views.py

@@ -4,6 +4,7 @@
 import datetime
 import io
 import json
+import time
 import logging
 from typing import TYPE_CHECKING, Any
 
@@ -18,6 +19,7 @@
 from django.db.models.functions import Coalesce
 from django.db.utils import IntegrityError, OperationalError
 from django.http import HttpResponse, JsonResponse
+from django.db import connection
 from django.utils.decorators import method_decorator
 from django.utils.timezone import now
 from django.views.decorators.http import require_GET
@@ -40,7 +42,8 @@
 
 from vespadb.observations.cache import invalidate_geojson_cache, invalidate_observation_cache
 from vespadb.observations.filters import ObservationFilter
-from vespadb.observations.helpers import parse_and_convert_to_utc, retry_with_backoff
+from vespadb.observations.helpers import parse_and_convert_to_utc
+from vespadb.observations.utils import db_retry
 from vespadb.observations.models import Municipality, Observation, Province
 from vespadb.observations.serializers import (
     MunicipalitySerializer,
@@ -616,7 +619,7 @@ def save_observations(self, valid_data: list[dict[str, Any]]) -> Response:
             return Response(
                 {"error": f"An error occurred during bulk import: {e!s}"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR
             )
-
+            
     @swagger_auto_schema(
         method="get",
         manual_parameters=[
@@ -633,41 +636,40 @@ def save_observations(self, valid_data: list[dict[str, Any]]) -> Response:
     @method_decorator(ratelimit(key="ip", rate="60/m", method="GET", block=True))
     @action(detail=False, methods=["get"], permission_classes=[AllowAny])
     def export(self, request: Request) -> Response:
-        try:
-            export_format = request.query_params.get("export_format", "csv").lower()
-            queryset = self.filter_queryset(self.get_queryset())
-            paginator = Paginator(queryset, 1000)  # Process in batches of 1000 items
+        retries = 3
+        delay = 5
 
-            serialized_data = []
-            errors = []
+        for attempt in range(retries):
+            try:
+                export_format = request.query_params.get("export_format", "csv").lower()
+                queryset = self.filter_queryset(self.get_queryset())
 
-            # Loop through each page in the paginator
-            for page_number in paginator.page_range:
-                page = paginator.page(page_number)
+                serialized_data = []
+                errors = []
 
-                try:
-                    # Try serializing the page
-                    serializer = self.get_serializer(page, many=True)
-                    serialized_data.extend(serializer.data)
-                except Exception as e:
-                    # Retry with backoff for individual objects on failure
-                    for obj in page.object_list:
-                        try:
-                            serialized_obj = retry_with_backoff(lambda: self.get_serializer(obj).data)
-                            serialized_data.append(serialized_obj)
-                        except Exception as inner_error:
-                            errors.append({
-                                "id": obj.id,
-                                "error": str(inner_error)
-                            })
-
-            if export_format == "csv":
-                return self.export_as_csv(serialized_data)
-
-            return JsonResponse({"data": serialized_data, "errors": errors}, safe=False)
-        except Exception as e:
-            return JsonResponse({"errors": errors}, safe=False)
-
+                for obj in queryset.iterator(chunk_size=1000):
+                    try:
+                        serialized_obj = self.get_serializer(obj).data
+                        serialized_data.append(serialized_obj)
+                    except Exception as inner_error:
+                        errors.append({
+                            "id": obj.id,
+                            "error": str(inner_error)
+                        })
+
+                if export_format == "csv":
+                    return self.export_as_csv(serialized_data)
+                return JsonResponse({"data": serialized_data, "errors": errors}, safe=False)
+
+            except OperationalError:
+                if attempt < retries - 1:
+                    time.sleep(delay)
+                    connection.close()
+                else:
+                    return JsonResponse({"error": "Database connection failed after retries"}, safe=False)
+            except Exception as e:
+                return JsonResponse({"errors": str(e)}, safe=False)
+                
     def export_as_csv(self, data: list[dict[str, Any]]) -> HttpResponse:
         """Export the data as a CSV file."""
         response = HttpResponse(content_type="text/csv")

vespadb/settings.py

@@ -33,6 +33,7 @@
     "CELERY_BROKER_URL": os.getenv("CELERY_BROKER_URL"),
     "REDIS_REFRESH_RATE_MIN": os.getenv("REDIS_REFRESH_RATE_MIN", "15"),
     "DJANGO_DEBUG": os.getenv("DJANGO_DEBUG", "False"),
+    "DEFAULT_FROM_EMAIL": os.getenv("DEFAULT_FROM_EMAIL", "[email protected]")
 }
 
 # Core settings
@@ -266,4 +267,4 @@
 EMAIL_BACKEND = "django_ses.SESBackend"
 AWS_SES_REGION_NAME = "eu-west-1"
 AWS_SES_REGION_ENDPOINT = "email.eu-west-1.amazonaws.com"
-DEFAULT_FROM_EMAIL = "[email protected]"
+DEFAULT_FROM_EMAIL = secrets["DEFAULT_FROM_EMAIL"]