Merge pull request #279 from inbo/export-r5

fix export uat

nginx.conf

@@ -8,6 +8,12 @@ http {
     include       mime.types;
     default_type  application/octet-stream;
 
+    # HTTP/2 specific settings
+    http2_max_field_size 16k;
+    http2_max_header_size 32k;
+    http2_max_requests 1000;
+    http2_idle_timeout 5m;
+
     # Global timeout settings
     proxy_connect_timeout 300;
     proxy_send_timeout 300;
@@ -18,13 +24,14 @@ http {
     keepalive_timeout  65;
 
     # Buffering settings for large responses
-    proxy_buffering on;
+    proxy_buffering off;
+    proxy_request_buffering off;
     proxy_buffer_size 16k;
     proxy_buffers 8 16k;
     proxy_busy_buffers_size 32k;
 
     server {
-        listen 80;
+        listen 80 http2;  # Added http2
         server_name uat.vespadb.be;
 
         # Increase client body size limit if needed
@@ -45,14 +52,23 @@ http {
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_set_header X-Forwarded-Proto $scheme;
 
+            # HTTP/2 specific
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+
+            # Timeouts
             proxy_connect_timeout 300s;
             proxy_send_timeout 300s;
             proxy_read_timeout 300s;
+
+            # Buffer settings
+            proxy_buffering off;
+            proxy_request_buffering off;
         }
     }
 
     server {
-        listen 80;
+        listen 80 http2;  # Added http2
         server_name data.vespawatch.be;
 
         # Increase client body size limit if needed
@@ -73,9 +89,18 @@ http {
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_set_header X-Forwarded-Proto $scheme;
 
+            # HTTP/2 specific
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+
+            # Timeouts
             proxy_connect_timeout 300s;
             proxy_send_timeout 300s;
             proxy_read_timeout 300s;
+
+            # Buffer settings
+            proxy_buffering off;
+            proxy_request_buffering off;
         }
     }
 }

vespadb/observations/views.py

@@ -19,6 +19,7 @@
     before_log,
     after_log,
 )
+import time
 from typing import Generator, Optional
 from django.db import OperationalError, connection, transaction
 from django.core.exceptions import ValidationError
@@ -783,6 +784,52 @@ def get_chunk_with_retries(
                 time.sleep(wait_time)
         return None
 
+    def _generate_csv_content(
+        self, 
+        queryset: QuerySet,
+        is_admin: bool,
+        user_municipality_ids: Set[str]
+    ) -> Generator[str, None, None]:
+        """Generate CSV content in smaller chunks."""
+        buffer = io.StringIO()
+        writer = csv.writer(buffer)
+
+        # Write headers first
+        writer.writerow(CSV_HEADERS)
+        data = buffer.getvalue()
+        buffer.seek(0)
+        buffer.truncate()
+        yield data
+
+        # Process in smaller chunks
+        chunk_size = 100  # Kleinere chunk size
+        total = queryset.count()
+
+        for start in range(0, total, chunk_size):
+            chunk = queryset.select_related(
+                'province', 
+                'municipality', 
+                'reserved_by'
+            )[start:start + chunk_size]
+
+            for observation in chunk:
+                try:
+                    row_data = self._prepare_row_data(
+                        observation, 
+                        is_admin, 
+                        user_municipality_ids
+                    )
+                    writer.writerow(row_data)
+                    data = buffer.getvalue()
+                    buffer.seek(0)
+                    buffer.truncate()
+                    yield data
+                except Exception as e:
+                    logger.error(f"Error processing observation {observation.id}: {str(e)}")
+                    continue
+
+        buffer.close()
+
     def create_csv_generator(
         self,
         queryset: QuerySet,
@@ -857,11 +904,9 @@ def export(self, request: HttpRequest) -> StreamingHttpResponse:
         Export observations as CSV using streaming response with improved error handling
         and performance optimizations.
         """
-        export_format = request.query_params.get("export_format", "csv").lower()
-
         try:
-            # Input validation
-            if export_format != "csv":
+            # Validate export format
+            if request.query_params.get("export_format", "csv").lower() != "csv":
                 return JsonResponse({"error": "Only CSV export is supported"}, status=400)
 
             # Get user permissions
@@ -875,39 +920,26 @@ def export(self, request: HttpRequest) -> StreamingHttpResponse:
             # Get filtered queryset
             queryset = self.filter_queryset(self.get_queryset())
 
-            # Create streaming response
+            # Create the StreamingHttpResponse
             response = StreamingHttpResponse(
-                streaming_content=self.create_csv_generator(
-                    queryset=queryset,
-                    is_admin=is_admin,
-                    user_municipality_ids=user_municipality_ids
+                streaming_content=self._generate_csv_content(
+                    queryset, is_admin, user_municipality_ids
                 ),
                 content_type='text/csv'
             )
 
-            # Set headers
+            # Important headers
             filename = f"observations_export_{datetime.datetime.now().strftime('%Y%m%d_%H%M%S')}.csv"
             response['Content-Disposition'] = f'attachment; filename="{filename}"'
-            response['X-Accel-Buffering'] = 'no'  # Disable nginx buffering
+            response['X-Accel-Buffering'] = 'no'
+            response['Cache-Control'] = 'no-cache'
 
             return response
 
-        except QueryTimeoutError:
-            logger.exception("Query timeout during export")
-            return JsonResponse(
-                {"error": "Export timed out. Please try with a smaller date range or fewer filters."},
-                status=503
-            )
-        except ExportError as e:
-            logger.exception("Export error")
-            return JsonResponse(
-                {"error": f"Export failed: {str(e)}. Please try again or contact support."},
-                status=500
-            )
         except Exception as e:
-            logger.exception("Unexpected error during export")
+            logger.exception("Export failed")
             return JsonResponse(
-                {"error": "An unexpected error occurred. Please try again or contact support."},
+                {"error": "Export failed. Please try again or contact support."}, 
                 status=500
             )