From ab7a639707f53b4eee427b8d9ef209e0759adc40 Mon Sep 17 00:00:00 2001 From: steve <82342664+stevegerrits@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:31:15 +0100 Subject: [PATCH] dec release 2 (#275) --- .../ObservationDetailsComponent.vue | 54 ++--- .../observations/tasks/observation_mapper.py | 35 ++- .../observations/tasks/observation_sync.py | 2 +- vespadb/observations/views.py | 214 ++++++++++-------- 4 files changed, 169 insertions(+), 136 deletions(-) diff --git a/src/components/ObservationDetailsComponent.vue b/src/components/ObservationDetailsComponent.vue index 8321eb1..f860cb5 100644 --- a/src/components/ObservationDetailsComponent.vue +++ b/src/components/ObservationDetailsComponent.vue @@ -205,53 +205,42 @@ -
- +

+ {{ selectedObservation.nest_type ? nestTypeEnum[selectedObservation.nest_type] : + 'Geen' }} +

- +

+ {{ selectedObservation.nest_location ? + nestLocationEnum[selectedObservation.nest_location] : + 'Geen' }} +

- +

+ {{ selectedObservation.nest_size ? nestSizeEnum[selectedObservation.nest_size] : + 'Geen' }} +

- +

+ {{ selectedObservation.nest_height ? + nestHeightEnum[selectedObservation.nest_height] : + 'Geen' }} +

@@ -322,7 +311,8 @@
-

{{ selectedObservation.observer_phone_number }}

+

{{ selectedObservation.observer_phone_number }} +

@@ -498,10 +488,6 @@ export default { }; const editableFields = [ - "nest_height", - "nest_size", - "nest_location", - "nest_type", "observation_datetime", "eradication_date", "admin_notes", diff --git a/vespadb/observations/tasks/observation_mapper.py b/vespadb/observations/tasks/observation_mapper.py index d815c6f..6121c04 100644 --- a/vespadb/observations/tasks/observation_mapper.py +++ b/vespadb/observations/tasks/observation_mapper.py @@ -2,7 +2,6 @@ import logging from datetime import datetime -from difflib import get_close_matches from typing import Any, cast import pytz @@ -28,8 +27,10 @@ mapping_dict: dict[int, dict[str, str]] = { 329: { + "hoger_dan_4_meter": "hoger_dan_4_meter", "Hoger dan 4 meter": "hoger_dan_4_meter", "Higher than 4 meters": "hoger_dan_4_meter", + "lager_dan_4_meter": "lager_dan_4_meter", "Lager dan 4 meter": "lager_dan_4_meter", "Lower than 4 meters": "lager_dan_4_meter", }, @@ -39,7 +40,7 @@ "Larger than 25cm": "groter_dan_25_cm", "Smaller than 25cm": "kleiner_dan_25_cm", }, - 331 : { + 331: { "Binnen, in gebouw of constructie": "binnen_in_gebouw_of_constructie", "Buiten, maar overdekt door constructie": "buiten_maar_overdekt_door_constructie", "Buiten, natuurlijk overdekt": "buiten_natuurlijk_overdekt", @@ -50,7 +51,28 @@ "Outside, natural cover": "buiten_natuurlijk_overdekt", "Outside, uncovered in a tree or bush": "buiten_onbedekt_in_boom_of_struik", "Outside, uncovered on building": "buiten_onbedekt_op_gebouw", - } + }, + 368: { + "Actief embryonaal nest": "actief_embryonaal_nest", + "Actief embryonaal nest (van maart tot eind juni, nest met enkel koningin)": "actief_embryonaal_nest", + "Actief primair nest": "actief_primair_nest", + "Actief primair nest (van juni tot eind november, nest met werksters op lage hoogte (tot 6 meter))": "actief_primair_nest", + "Actief secundair nest": "actief_secundair_nest", + "Actief secundair nest (van augustus tot eind november, nest met werksters op grote hoogte (tot 30m))": "actief_secundair_nest", + "Active embryonic nest": "actief_embryonaal_nest", + "Active embryonic nest (from march to the end of june, nest with queen only)": "actief_embryonaal_nest", + "Active primary nest": "actief_primair_nest", + "Active primary nest (from june to the end of november, nest with workers at low altitude (up to 6m))": "actief_primair_nest", + "Active secondary nest": "actief_secundair_nest", + "Active secondary nest (from aug. to the end of nov., nest with workers at high altitude (up to 30m))": "actief_secundair_nest", + "Inactief/leeg nest (typisch tijdens wintermaanden, een leeg nest hoog in een boom)": "inactief_leeg_nest", + "Inactief/leeg nest (typisch tijdens wintermaanden, een leeg netst oog in een boom)": "inactief_leeg_nest", + "Inactive/empty nest": "inactief_leeg_nest", + "Inactive/empty nest (typically during the winter months, an empty nest high in a tree)": "inactief_leeg_nest", + "Potential nest": None, + "Potentieel nest": None, + "Potentieel nest (onzeker van de soort)": None, + }, } ENUMS_MAPPING: dict[str, type[TextChoices]] = { @@ -67,6 +89,7 @@ 329: "nest_height", 330: "nest_size", 331: "nest_location", + 368: "nest_type", } # Literal mapping functions def map_nest_height_attribute_to_enum(value: str) -> Any | None: @@ -81,6 +104,10 @@ def map_nest_location_attribute_to_enum(value: str) -> str | None: """Maps Nest location values to enums based on literal mapping.""" return mapping_dict[331].get(value.strip()) +def map_nest_type_attribute_to_enum(value: str) -> str | None: + """Maps Nest location values to enums based on literal mapping.""" + return mapping_dict[368].get(value.strip()) + def map_attribute_to_enum(attribute_id: int, value: str) -> str | None: """ Maps a single attribute value to an enum using literal mapping functions. @@ -91,6 +118,8 @@ def map_attribute_to_enum(attribute_id: int, value: str) -> str | None: return map_nest_size_attribute_to_enum(value) elif attribute_id == 331: return map_nest_location_attribute_to_enum(value) + elif attribute_id == 368: + return map_nest_type_attribute_to_enum(value) else: return None diff --git a/vespadb/observations/tasks/observation_sync.py b/vespadb/observations/tasks/observation_sync.py index e234ea2..332e632 100644 --- a/vespadb/observations/tasks/observation_sync.py +++ b/vespadb/observations/tasks/observation_sync.py @@ -325,4 +325,4 @@ def fetch_and_update_observations(self: Task, since_week: int | None = None, dat logger.info("Finished processing observations") manage_observations_visibility(token) - logger.info("Finished managing observations visibility") + logger.info("Finished managing observations visibility") \ No newline at end of file diff --git a/vespadb/observations/views.py b/vespadb/observations/views.py index 41b95d6..7246067 100644 --- a/vespadb/observations/views.py +++ b/vespadb/observations/views.py @@ -9,6 +9,9 @@ import csv import json from typing import TYPE_CHECKING, Any, Generator, Any, Union +from django.http import FileResponse +import os +import tempfile from django.contrib.gis.db.models.functions import Transform from django.contrib.gis.geos import GEOSGeometry @@ -62,13 +65,14 @@ BBOX_LENGTH = 4 GEOJSON_REDIS_CACHE_EXPIRATION = 900 # 15 minutes GET_REDIS_CACHE_EXPIRATION = 86400 # 1 day -BATCH_SIZE = 150 CSV_HEADERS = [ "id", "created_datetime", "modified_datetime", "latitude", "longitude", "source", "source_id", "nest_height", "nest_size", "nest_location", "nest_type", "observation_datetime", "province", "eradication_date", "municipality", "images", "anb_domain", "notes", "eradication_result", "wn_id", "wn_validation_status", "nest_status" ] +BATCH_SIZE = 1000 + class ObservationsViewSet(ModelViewSet): # noqa: PLR0904 """ViewSet for the Observation model.""" @@ -642,127 +646,141 @@ def save_observations(self, valid_data: list[dict[str, Any]]) -> Response: ) @method_decorator(ratelimit(key="ip", rate="60/m", method="GET", block=True)) @action(detail=False, methods=["get"], permission_classes=[AllowAny]) - def export(self, request: HttpRequest) -> Union[StreamingHttpResponse, JsonResponse]: + def export(self, request: HttpRequest) -> FileResponse: """ - Export observations as CSV with dynamically controlled fields based on user permissions. - - Observations from municipalities the user has access to will display full details; - others will show limited fields as per public access. + Export observations as CSV using batch processing. """ - if request.query_params.get("export_format", "csv").lower() != "csv": - return JsonResponse({"error": "Only CSV export is supported"}, status=400) - - # Determine user permissions - if request.user.is_authenticated: - user_municipality_ids = set(request.user.municipalities.values_list("id", flat=True)) - is_admin = request.user.is_superuser - else: - user_municipality_ids = set() - is_admin = False - - # Set CSV headers directly from CSV_HEADERS as a base - dynamic_csv_headers = CSV_HEADERS - - # Prepare response - queryset = self.filter_queryset(self.get_queryset()) - response = StreamingHttpResponse( - self.generate_csv_rows(queryset, dynamic_csv_headers, user_municipality_ids, is_admin), - content_type="text/csv" - ) - response["Content-Disposition"] = 'attachment; filename="observations_export.csv"' - return response - - def generate_csv_rows( - self, queryset: QuerySet, headers: list[str], user_municipality_ids: set, is_admin: bool - ) -> Generator[bytes, None, None]: - """Generate CSV rows with headers and filtered data according to user permissions.""" - # Yield headers - yield self._csv_line(headers) - - for observation in queryset.iterator(chunk_size=500): - # Determine fields to include based on user permissions for each observation - if is_admin or (observation.municipality_id in user_municipality_ids): - # Full access for admins and assigned municipalities - allowed_fields = user_read_fields + try: + # Validate export format + if request.query_params.get("export_format", "csv").lower() != "csv": + return JsonResponse({"error": "Only CSV export is supported"}, status=400) + + # Get user permissions + if request.user.is_authenticated: + user_municipality_ids = set(request.user.municipalities.values_list("id", flat=True)) + is_admin = request.user.is_superuser else: - # Restricted access for other municipalities - allowed_fields = public_read_fields - - # Add essential fields for export - allowed_fields.extend(["source_id", "latitude", "longitude", "anb_domain", "nest_status"]) + user_municipality_ids = set() + is_admin = False - # Serialize the observation with restricted fields as needed - row = self.serialize_observation(observation, headers, allowed_fields) - yield self._csv_line(row) + # Get filtered queryset + queryset = self.filter_queryset(self.get_queryset()) - def parse_location(self, srid_str: str) -> tuple[float, float]: + # Create temporary file + with tempfile.NamedTemporaryFile(mode='w+', newline='', delete=False, suffix='.csv') as temp_file: + writer = csv.writer(temp_file) + + # Write headers + writer.writerow(CSV_HEADERS) + + # Process in batches + total_processed = 0 + while True: + # Get batch of observations + batch = queryset[total_processed:total_processed + BATCH_SIZE] + if not batch: + break + + # Process each observation in the batch + for observation in batch: + row_data = self._prepare_row_data( + observation, + is_admin, + user_municipality_ids + ) + writer.writerow(row_data) + + total_processed += len(batch) + logger.info(f"Processed {total_processed} observations") + + # Create response with the temporary file + response = FileResponse( + open(temp_file.name, 'rb'), + content_type='text/csv', + as_attachment=True, + filename=f"observations_export_{datetime.datetime.now().strftime('%Y%m%d_%H%M%S')}.csv" + ) + + # Delete the temporary file after it's sent + response.close = lambda: os.unlink(temp_file.name) + + return response + + except Exception as e: + logger.exception("Export failed") + return JsonResponse( + {"error": "Export failed. Please try again or contact support."}, + status=500 + ) + def _prepare_row_data( + self, + observation: Observation, + is_admin: bool, + user_municipality_ids: set[str] + ) -> list[str]: """ - Parse SRID string to extract latitude and longitude. + Prepare a single row of data for the CSV export. """ - # Convert the SRID location string to GEOSGeometry - geom = GEOSGeometry(srid_str) + # Determine allowed fields based on permissions + if is_admin or (observation.municipality_id in user_municipality_ids): + allowed_fields = user_read_fields + else: + allowed_fields = public_read_fields + + # Add essential fields + allowed_fields.extend(["source_id", "latitude", "longitude", "anb_domain", "nest_status"]) - # Extract latitude and longitude - longitude = geom.x - latitude = geom.y - return latitude, longitude - - def serialize_observation(self, obj: Observation, headers: list[str], allowed_fields: list[str]) -> list[str]: - """Serialize an observation for CSV export with specified fields.""" - data = [] - for field in headers: + row_data = [] + for field in CSV_HEADERS: if field not in allowed_fields: - data.append("") # Add empty string for restricted fields + row_data.append("") continue - # Handle custom formatting for certain fields - if field == "latitude" or field == "longitude": - if obj.location: - srid_location_str = f"SRID=4326;POINT ({obj.location.x} {obj.location.y})" - latitude, longitude = self.parse_location(srid_location_str) - logger.info('Latitude: %s, Longitude: %s', latitude, longitude) - if field == "latitude": - data.append(str(latitude)) - elif field == "longitude": - data.append(str(longitude)) - else: - data.append("") + # Handle special fields + if field == "latitude": + row_data.append(str(observation.location.y) if observation.location else "") + elif field == "longitude": + row_data.append(str(observation.location.x) if observation.location else "") elif field in ["created_datetime", "modified_datetime", "observation_datetime"]: - datetime_val = getattr(obj, field, None) + datetime_val = getattr(observation, field, None) if datetime_val: - # Remove milliseconds and ensure ISO format with 'Z' datetime_val = datetime_val.replace(microsecond=0) - # Convert to ISO format and replace +00:00 with Z if present - iso_datetime = datetime_val.isoformat() - if iso_datetime.endswith('+00:00'): - iso_datetime = iso_datetime[:-6] + 'Z' - elif not iso_datetime.endswith('Z'): - iso_datetime += 'Z' - data.append(iso_datetime) + row_data.append(datetime_val.isoformat() + "Z") else: - data.append("") + row_data.append("") elif field == "province": - data.append(obj.province.name if obj.province else "") + row_data.append(observation.province.name if observation.province else "") elif field == "municipality": - data.append(obj.municipality.name if obj.municipality else "") + row_data.append(observation.municipality.name if observation.municipality else "") elif field == "anb_domain": - data.append(str(obj.anb)) - elif field == "eradication_result": - data.append(obj.eradication_result if obj.eradication_result else "") + row_data.append(str(observation.anb)) elif field == "nest_status": - logger.info("Getting status for observation %s", obj.eradication_result) - data.append(self.get_status(obj)) + row_data.append(self.get_status(observation)) elif field == "source_id": - data.append(str(obj.source_id) if obj.source_id is not None else "") + row_data.append(str(observation.source_id) if observation.source_id is not None else "") else: - value = getattr(obj, field, "") - data.append(str(value) if value is not None else "") - return data + value = getattr(observation, field, "") + row_data.append(str(value) if value is not None else "") + + return row_data + + + def parse_location(self, srid_str: str) -> tuple[float, float]: + """ + Parse SRID string to extract latitude and longitude. + """ + # Convert the SRID location string to GEOSGeometry + geom = GEOSGeometry(srid_str) + + # Extract latitude and longitude + longitude = geom.x + latitude = geom.y + return latitude, longitude def get_status(self, observation: Observation) -> str: """Determine observation status based on eradication data.""" - logger.info("Getting status for observation %s", observation.eradication_result) - if observation.eradication_result == EradicationResultEnum.SUCCESSFUL: + logger.debug("Getting status for observation %s", observation.eradication_result) + if observation.eradication_result: return "eradicated" if observation.reserved_by: return "reserved"