export

entrypoint.sh

@@ -34,8 +34,14 @@ echo "Load waarnemingen observation data via: python manage.py load_waarnemingen
 # Start Gunicorn
 echo "Starting Gunicorn..."
 gunicorn --workers 3 \
+         --worker-class gthread \
+         --threads 4 \
+         --worker-connections 1000 \
          --timeout 1800 \
+         --graceful-timeout 300 \
          --keep-alive 65 \
+         --max-requests 1000 \
+         --max-requests-jitter 50 \
          --bind 0.0.0.0:8000 \
          vespadb.wsgi:application &
 

nginx.conf

@@ -113,4 +113,4 @@ http {
             proxy_busy_buffers_size 256k;
         }
     }
-}
+}

pyproject.toml

@@ -46,6 +46,7 @@ types-python-dateutil = "^2.9.0.20240316"
 whitenoise = "^6.6.0"
 django-ses = "^4.2.0"
 tenacity = "^9.0.0"
+django-extensions = "^3.2.3"
 [tool.poetry.group.dev.dependencies]  # https://python-poetry.org/docs/master/managing-dependencies/
 coverage = { extras = ["toml"], version = ">=7.4.1" }
 ipython = ">=8.20.0"

src/components/MapPage.vue

@@ -15,6 +15,9 @@
       <div class="loading-screen" v-if="isMapLoading">
         Observaties laden...
       </div>
+      <div class="loading-screen" v-if="isExporting">
+        Exporteren...
+      </div>
       <div class="map-legend" v-if="map && !isMapLoading">
         <div>
           <span class="legend-icon bg-reported"></span> Gerapporteerd nest
@@ -61,6 +64,7 @@ export default {
   setup() {
     const vespaStore = useVespaStore();
     const searchQuery = ref('');
+    const isExporting = computed(() => vespaStore.isExporting);
     const router = useRouter();
     const selectedObservation = computed(() => vespaStore.selectedObservation);
     const isEditing = computed(() => vespaStore.isEditing);
@@ -327,6 +331,7 @@ export default {
       updateMarkerColor,
       searchQuery,
       searchAddress,
+      isExporting,
     };
   },
 };

src/components/NavbarComponent.vue

@@ -19,7 +19,15 @@
             Export
           </button>
           <ul class="dropdown-menu">
-            <li><button class="dropdown-item" @click="exportData('csv')">CSV</button></li>
+              <li>
+                  <button 
+                      class="dropdown-item" 
+                      @click="exportData('csv')"
+                      :disabled="isExporting"
+                  >
+                      CSV
+                  </button>
+              </li>
           </ul>
         </div>
 
@@ -67,6 +75,7 @@ export default {
     const isModalVisible = ref(false);
     const modalTitle = ref('');
     const modalMessage = ref('');
+    const isExporting = computed(() => vespaStore.isExporting);
 
     watch(() => vespaStore.error, (newError) => {
       if (newError) {
@@ -94,10 +103,18 @@ export default {
     };
 
     const exportData = async (format) => {
-      await vespaStore.exportData(format);
-    };
+      try {
+          if (vespaStore.isExporting) return;
+          
+          await vespaStore.exportData(format);
+      } catch (error) {
+          modalTitle.value = 'Error';
+          modalMessage.value = 'Er is een fout opgetreden tijdens het exporteren.';
+          isModalVisible.value = true;
+      }
+  };
 
-    return { isLoggedIn, loadingAuth, username, logout, navigateToChangePassword, exportData, fileInput, isModalVisible, modalTitle, modalMessage };
+    return { isLoggedIn, loadingAuth, username, logout, navigateToChangePassword, exportData, fileInput, isModalVisible, modalTitle, modalMessage, isExporting };
   },
   mounted() {
     var dropdownElementList = [].slice.call(document.querySelectorAll('.dropdown-toggle'));

src/stores/vespaStore.js

@@ -27,6 +27,7 @@ export const useVespaStore = defineStore('vespaStore', {
         isEditing: false,
         map: null,
         viewMode: 'map',
+        isExporting: false,
         filters: {
             municipalities: [],
             provinces: [],
@@ -307,21 +308,55 @@ export const useVespaStore = defineStore('vespaStore', {
             }
         },
         async exportData(format) {
-            const filterQuery = this.createFilterQuery();
-            const url = `/observations/export?export_format=${format}&${filterQuery}`;
-
             try {
-                const response = await ApiService.get(url, { responseType: 'blob' });
-                const blob = new Blob([response.data], { type: response.headers['content-type'] });
-                const downloadUrl = window.URL.createObjectURL(blob);
-                const link = document.createElement('a');
-                link.href = downloadUrl;
-                link.setAttribute('download', `export.${format}`);
-                document.body.appendChild(link);
-                link.click();
-                link.remove();
+                this.isExporting = true;  // Start loading indicator
+                const response = await ApiService.get(
+                    `/observations/export?${this.createFilterQuery()}`
+                );
+
+                if (response.status === 200) {
+                    const { export_id } = response.data;
+
+                    const checkStatus = async () => {
+                        const statusResponse = await ApiService.get(
+                            `/observations/export_status?export_id=${export_id}`
+                        );
+
+                        if (statusResponse.data.status === 'completed') {
+                            const downloadResponse = await ApiService.get(
+                                `/observations/download_export/?export_id=${export_id}`,
+                                { responseType: 'blob' }
+                            );
+
+                            const blob = new Blob([downloadResponse.data], { type: 'text/csv' });
+                            const url = window.URL.createObjectURL(blob);
+                            const link = document.createElement('a');
+                            link.href = url;
+                            link.setAttribute('download', `observations_export_${new Date().getTime()}.csv`);
+                            document.body.appendChild(link);
+                            link.click();
+                            document.body.removeChild(link);
+                            window.URL.revokeObjectURL(url);
+                            this.isExporting = false;  // Stop loading indicator
+                            return true;
+                        } else if (statusResponse.data.status === 'failed') {
+                            this.isExporting = false;  // Stop loading indicator on error
+                            throw new Error(statusResponse.data.error || 'Export failed');
+                        }
+
+                        return new Promise(resolve => {
+                            setTimeout(async () => {
+                                resolve(await checkStatus());
+                            }, 2000);
+                        });
+                    };
+
+                    await checkStatus();
+                }
             } catch (error) {
+                this.isExporting = false;  // Stop loading indicator on error
                 console.error('Error exporting data:', error);
+                throw error;
             }
         },
         async fetchMunicipalitiesByProvinces(provinceIds) {

vespadb/observations/migrations/0033_export.py

@@ -0,0 +1,31 @@
+# Generated by Django 5.1.4 on 2024-12-18 16:03
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('observations', '0032_rename_wn_notes_observation_notes'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Export',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('filters', models.JSONField(default=dict, help_text='Filters applied to the export')),
+                ('status', models.CharField(choices=[('pending', 'Pending'), ('processing', 'Processing'), ('completed', 'Completed'), ('failed', 'Failed')], default='pending', help_text='Status of the export', max_length=20)),
+                ('progress', models.IntegerField(default=0, help_text='Progress percentage of the export')),
+                ('file_path', models.CharField(blank=True, help_text='Path to the exported file', max_length=255, null=True)),
+                ('created_at', models.DateTimeField(auto_now_add=True, help_text='Datetime when the export was created')),
+                ('completed_at', models.DateTimeField(blank=True, help_text='Datetime when the export was completed', null=True)),
+                ('error_message', models.TextField(blank=True, help_text='Error message if the export failed', null=True)),
+                ('task_id', models.CharField(blank=True, help_text='Celery task ID for the export', max_length=255, null=True)),
+                ('user', models.ForeignKey(blank=True, help_text='User who requested the export', null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+    ]

vespadb/observations/models.py

@@ -389,3 +389,29 @@ def save(self, *args: Any, **kwargs: Any) -> None:
             self.province = municipality.province if municipality else None
 
         super().save(*args, **kwargs)
+
+class Export(models.Model):
+    """Model for tracking observation exports."""
+    STATUS_CHOICES = (
+        ('pending', 'Pending'),
+        ('processing', 'Processing'),
+        ('completed', 'Completed'),
+        ('failed', 'Failed'),
+    )
+
+    id = models.AutoField(primary_key=True)
+    user = models.ForeignKey(
+        settings.AUTH_USER_MODEL,
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+        help_text="User who requested the export",
+    )
+    filters = models.JSONField(default=dict, help_text="Filters applied to the export")
+    status = models.CharField(max_length=20, choices=STATUS_CHOICES, default='pending', help_text="Status of the export")
+    progress = models.IntegerField(default=0, help_text="Progress percentage of the export")
+    file_path = models.CharField(max_length=255, blank=True, null=True, help_text="Path to the exported file")
+    created_at = models.DateTimeField(auto_now_add=True, help_text="Datetime when the export was created")
+    completed_at = models.DateTimeField(blank=True, null=True, help_text="Datetime when the export was completed")
+    error_message = models.TextField(blank=True, null=True, help_text="Error message if the export failed")
+    task_id = models.CharField(max_length=255, blank=True, null=True, help_text="Celery task ID for the export")

vespadb/observations/serializers.py

@@ -13,7 +13,7 @@
 from rest_framework.request import Request
 
 from vespadb.observations.helpers import parse_and_convert_to_cet, parse_and_convert_to_utc
-from vespadb.observations.models import EradicationResultEnum, Municipality, Observation, Province
+from vespadb.observations.models import EradicationResultEnum, Municipality, Observation, Province, Export
 from vespadb.observations.utils import get_municipality_from_coordinates
 from vespadb.users.models import VespaUser
 
@@ -484,3 +484,9 @@ class Meta:
 
         model = Province
         fields = ["id", "name"]
+
+
+class ExportSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = Export
+        fields = '__all__'