diff --git a/vespadb/observations/admin.py b/vespadb/observations/admin.py index b828f25..ea66c46 100644 --- a/vespadb/observations/admin.py +++ b/vespadb/observations/admin.py @@ -18,6 +18,7 @@ from django.utils.timezone import now from django.utils.translation import gettext_lazy as _ from rest_framework.test import APIRequestFactory +from django.conf import settings from vespadb.observations.filters import MunicipalityExcludeFilter, ObserverReceivedEmailFilter, ProvinceFilter from vespadb.observations.forms import SendEmailForm @@ -305,7 +306,7 @@ def send_email_view(self, request: HttpRequest) -> HttpResponse: fail_list.append(observation.id) continue try: - send_mail(subject, message, "noreply@vespawatch.be", [observation.observer_email]) + send_mail(subject, message, settings.DEFAULT_FROM_EMAIL, [observation.observer_email]) logger.debug(f"Email sent to {observation.observer_email} for observation {observation.id}") observation.observer_received_email = True observation.save() diff --git a/vespadb/observations/utils.py b/vespadb/observations/utils.py index eb2cdaf..f7d2258 100644 --- a/vespadb/observations/utils.py +++ b/vespadb/observations/utils.py @@ -1,7 +1,12 @@ """Utility functions for the observations app.""" from django.contrib.gis.geos import Point +import time +from functools import wraps +from django.db import connection, OperationalError +from typing import Callable, TypeVar, Any, cast +F = TypeVar("F", bound=Callable[..., Any]) def get_municipality_from_coordinates(longitude: float, latitude: float): # type: ignore[no-untyped-def] """Get the municipality for a given long and lat.""" @@ -24,3 +29,29 @@ def check_if_point_in_anb_area(longitude: float, latitude: float) -> bool: anb_areas_containing_point = ANB.objects.filter(polygon__contains=point_to_check) return bool(anb_areas_containing_point) + +def db_retry(retries: int = 3, delay: int = 5) -> Callable[[F], F]: + """ + Decorator to retry a database operation in case of an OperationalError. + + Args: + retries (int): Number of retry attempts. Defaults to 3. + delay (int): Delay between retries in seconds. Defaults to 5. + + Returns: + Callable: The wrapped function with retry logic. + """ + def decorator(func: F) -> F: + @wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Any: + for attempt in range(retries): + try: + return func(*args, **kwargs) + except OperationalError: + if attempt < retries - 1: + time.sleep(delay) + connection.close() + else: + raise + return cast(F, wrapper) + return decorator diff --git a/vespadb/observations/views.py b/vespadb/observations/views.py index fcbdc29..0527f01 100644 --- a/vespadb/observations/views.py +++ b/vespadb/observations/views.py @@ -4,6 +4,7 @@ import datetime import io import json +import time import logging from typing import TYPE_CHECKING, Any @@ -18,6 +19,7 @@ from django.db.models.functions import Coalesce from django.db.utils import IntegrityError, OperationalError from django.http import HttpResponse, JsonResponse +from django.db import connection from django.utils.decorators import method_decorator from django.utils.timezone import now from django.views.decorators.http import require_GET @@ -40,7 +42,8 @@ from vespadb.observations.cache import invalidate_geojson_cache, invalidate_observation_cache from vespadb.observations.filters import ObservationFilter -from vespadb.observations.helpers import parse_and_convert_to_utc, retry_with_backoff +from vespadb.observations.helpers import parse_and_convert_to_utc +from vespadb.observations.utils import db_retry from vespadb.observations.models import Municipality, Observation, Province from vespadb.observations.serializers import ( MunicipalitySerializer, @@ -616,7 +619,7 @@ def save_observations(self, valid_data: list[dict[str, Any]]) -> Response: return Response( {"error": f"An error occurred during bulk import: {e!s}"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR ) - + @swagger_auto_schema( method="get", manual_parameters=[ @@ -633,41 +636,40 @@ def save_observations(self, valid_data: list[dict[str, Any]]) -> Response: @method_decorator(ratelimit(key="ip", rate="60/m", method="GET", block=True)) @action(detail=False, methods=["get"], permission_classes=[AllowAny]) def export(self, request: Request) -> Response: - try: - export_format = request.query_params.get("export_format", "csv").lower() - queryset = self.filter_queryset(self.get_queryset()) - paginator = Paginator(queryset, 1000) # Process in batches of 1000 items + retries = 3 + delay = 5 - serialized_data = [] - errors = [] + for attempt in range(retries): + try: + export_format = request.query_params.get("export_format", "csv").lower() + queryset = self.filter_queryset(self.get_queryset()) - # Loop through each page in the paginator - for page_number in paginator.page_range: - page = paginator.page(page_number) + serialized_data = [] + errors = [] - try: - # Try serializing the page - serializer = self.get_serializer(page, many=True) - serialized_data.extend(serializer.data) - except Exception as e: - # Retry with backoff for individual objects on failure - for obj in page.object_list: - try: - serialized_obj = retry_with_backoff(lambda: self.get_serializer(obj).data) - serialized_data.append(serialized_obj) - except Exception as inner_error: - errors.append({ - "id": obj.id, - "error": str(inner_error) - }) - - if export_format == "csv": - return self.export_as_csv(serialized_data) - - return JsonResponse({"data": serialized_data, "errors": errors}, safe=False) - except Exception as e: - return JsonResponse({"errors": errors}, safe=False) - + for obj in queryset.iterator(chunk_size=1000): + try: + serialized_obj = self.get_serializer(obj).data + serialized_data.append(serialized_obj) + except Exception as inner_error: + errors.append({ + "id": obj.id, + "error": str(inner_error) + }) + + if export_format == "csv": + return self.export_as_csv(serialized_data) + return JsonResponse({"data": serialized_data, "errors": errors}, safe=False) + + except OperationalError: + if attempt < retries - 1: + time.sleep(delay) + connection.close() + else: + return JsonResponse({"error": "Database connection failed after retries"}, safe=False) + except Exception as e: + return JsonResponse({"errors": str(e)}, safe=False) + def export_as_csv(self, data: list[dict[str, Any]]) -> HttpResponse: """Export the data as a CSV file.""" response = HttpResponse(content_type="text/csv") diff --git a/vespadb/settings.py b/vespadb/settings.py index c24c194..418e69b 100644 --- a/vespadb/settings.py +++ b/vespadb/settings.py @@ -33,6 +33,7 @@ "CELERY_BROKER_URL": os.getenv("CELERY_BROKER_URL"), "REDIS_REFRESH_RATE_MIN": os.getenv("REDIS_REFRESH_RATE_MIN", "15"), "DJANGO_DEBUG": os.getenv("DJANGO_DEBUG", "False"), + "DEFAULT_FROM_EMAIL": os.getenv("DEFAULT_FROM_EMAIL", "noreply@uat.vespawatch.be") } # Core settings @@ -266,4 +267,4 @@ EMAIL_BACKEND = "django_ses.SESBackend" AWS_SES_REGION_NAME = "eu-west-1" AWS_SES_REGION_ENDPOINT = "email.eu-west-1.amazonaws.com" -DEFAULT_FROM_EMAIL = "noreply@vespawatch.be" +DEFAULT_FROM_EMAIL = secrets["DEFAULT_FROM_EMAIL"]