Add S3/Azure config for static storage (#102)

helm/templates/config/configmap.yaml

@@ -11,20 +11,29 @@ data:
   DJANGO_APP_ENVIRONMENT: {{ .Values.environment | quote | upper }}
   DJANGO_DEBUG: {{ required "env.DJANGO_DEBUG" .Values.env.DJANGO_DEBUG | quote }}
   DJANGO_TIME_ZONE: {{ required "env.DJANGO_TIME_ZONE" .Values.env.DJANGO_TIME_ZONE | quote }}
+
   # App Domain
   APP_DOMAIN: {{ required "env.APP_DOMAIN" .Values.env.APP_DOMAIN | quote }}
   APP_HTTP_PROTOCOL: {{ required "env.APP_HTTP_PROTOCOL" .Values.env.APP_HTTP_PROTOCOL | quote }}
   APP_FRONTEND_HOST: {{ required "env.APP_FRONTEND_HOST" .Values.env.APP_FRONTEND_HOST | quote }}
   DJANGO_ALLOWED_HOSTS: {{ required "env.DJANGO_ALLOWED_HOSTS" .Values.env.DJANGO_ALLOWED_HOSTS | quote }}
   SESSION_COOKIE_DOMAIN: {{ required "env.SESSION_COOKIE_DOMAIN" .Values.env.SESSION_COOKIE_DOMAIN | quote }}
   CSRF_COOKIE_DOMAIN: {{ required "env.CSRF_COOKIE_DOMAIN" .Values.env.CSRF_COOKIE_DOMAIN | quote }}
+
   # CORS
   CORS_ALLOWED_ORIGINS: {{ required "env.CORS_ALLOWED_ORIGINS" .Values.env.CORS_ALLOWED_ORIGINS | quote }}
   CORS_ALLOWED_ORIGIN_REGEXES: {{ default "" .Values.env.CORS_ALLOWED_ORIGIN_REGEXES | quote }}
+
   # Misc
   UPTIME_WORKER_HEARTBEAT: {{ .Values.env.UPTIME_WORKER_HEARTBEAT | quote }}
   DEFAULT_FROM_EMAIL: {{ required "env.DEFAULT_FROM_EMAIL" .Values.env.DEFAULT_FROM_EMAIL | quote }}
   EMAIL_USE_TLS: {{ required "env.EMAIL_USE_TLS" .Values.env.EMAIL_USE_TLS | quote }}
+
   # Sentry
   SENTRY_TRACES_SAMPLE_RATE: {{ required "env.SENTRY_TRACES_SAMPLE_RATE" .Values.env.SENTRY_TRACES_SAMPLE_RATE | quote }}
   SENTRY_PROFILE_SAMPLE_RATE: {{ required "env.SENTRY_PROFILE_SAMPLE_RATE" .Values.env.SENTRY_PROFILE_SAMPLE_RATE | quote }}
+
+  # Additional configs
+  {{- range $name, $value := .Values.envAdditional }}
+  {{ $name }}: {{ $value | quote }}
+  {{- end }}

helm/templates/config/secret.yaml

@@ -11,22 +11,27 @@ metadata:
 type: Opaque
 stringData:
   DJANGO_SECRET_KEY: {{ required "secrets.DJANGO_SECRET_KEY" .Values.secrets.DJANGO_SECRET_KEY | quote }}
+
   # Database
   DB_NAME: {{ required "secrets.DB_NAME" .Values.secrets.DB_NAME | quote }}
   DB_USER: {{ required "secrets.DB_USER" .Values.secrets.DB_USER | quote }}
   DB_PASSWORD: {{ required "secrets.DB_PASSWORD" .Values.secrets.DB_PASSWORD | quote }}
   DB_HOST: {{ required "secrets.DB_HOST" .Values.secrets.DB_HOST | quote }}
   DB_PORT: {{ required "secrets.DB_PORT" .Values.secrets.DB_PORT | quote }}
+
   # Sentry
   SENTRY_DSN: {{ required "secrets.SENTRY_DSN" .Values.secrets.SENTRY_DSN | quote }}
+
   # Email
   EMAIL_HOST: {{ required "secrets.EMAIL_HOST" .Values.secrets.EMAIL_HOST | quote }}
   EMAIL_PORT: {{ required "secrets.EMAIL_PORT" .Values.secrets.EMAIL_PORT | quote }}
   EMAIL_HOST_USER: {{ required "secrets.EMAIL_HOST_USER" .Values.secrets.EMAIL_HOST_USER | quote }}
   EMAIL_HOST_PASSWORD: {{ required "secrets.EMAIL_HOST_PASSWORD" .Values.secrets.EMAIL_HOST_PASSWORD | quote }}
+
   # Hcaptcha
   HCAPTCHA_SECRET: {{ required "secrets.HCAPTCHA_SECRET" .Values.secrets.HCAPTCHA_SECRET | quote }}
   HCAPTCHA_SITEKEY: {{ required "secrets.HCAPTCHA_SITEKEY" .Values.secrets.HCAPTCHA_SITEKEY | quote }}
+
   # Redis
   {{- if .Values.redis.enabled }}
   CELERY_BROKER_URL: "redis://{{ printf "%s-master" (include "common.names.fullname" .Subcharts.redis) }}:6379/0"
@@ -36,4 +41,9 @@ stringData:
   CACHE_REDIS_URL: {{ required "env.CACHE_REDIS_URL" .Values.env.CACHE_REDIS_URL | quote }}
   {{- end }}
 
+  # Additional secrets
+  {{- range $name, $value := .Values.secretsAdditional }}
+  {{ $name }}: {{ $value | quote }}
+  {{- end }}
+
 {{- end }}

helm/values-test.yaml

@@ -25,6 +25,9 @@ env:
   CORS_ALLOWED_ORIGINS: dummy.com
   # Misc
   DEFAULT_FROM_EMAIL: [email protected]
+envAdditional:
+  ENABLE_MAGIC: "true"
+  MAGIC_TYPE: fun
 
 secretsName: ""
 secrets:
@@ -45,3 +48,6 @@ secrets:
   # Hcaptcha
   HCAPTCHA_SITEKEY: "10000000-ffff-ffff-ffff-000000000001"
   HCAPTCHA_SECRET: "0x0000000000000000000000000000000000000000"
+secretsAdditional:
+  ENABLE_MAGIC_SECRET: "true"
+  MAGIC_KEY: to-much-fun

helm/values.yaml

@@ -90,6 +90,10 @@ env:
   # Sentry
   SENTRY_TRACES_SAMPLE_RATE: 0.2
   SENTRY_PROFILE_SAMPLE_RATE: 0.2
+# NOTE: Used to pass additional configs to api/worker containers
+# NOTE: Not used by azure vault
+envAdditional:
+  # USE_S3_BUCKET: "true"
 
 secretsName: ""
 secrets:
@@ -115,6 +119,10 @@ secrets:
   # Hcaptcha
   HCAPTCHA_SITEKEY:
   HCAPTCHA_SECRET:
+# NOTE: Used to pass additional secrets to api/worker containers
+# NOTE: Not used by azure vault
+secretsAdditional:
+  # AWS_S3_ACCESS_KEY_ID: "VERY-SENSITIVE-KEY-ID"
 
 # Azure configurations
 azure:

main/settings.py

@@ -14,6 +14,7 @@
 from pathlib import Path
 
 import environ
+from azure.identity import DefaultAzureCredential
 from django.utils.translation import gettext_lazy as _
 
 from main import sentry
@@ -59,6 +60,24 @@
     EMAIL_HOST_USER=str,
     EMAIL_HOST_PASSWORD=str,
     DEFAULT_FROM_EMAIL=str,
+    # Storage
+    # -- S3
+    USE_S3_BUCKET=(bool, False),
+    AWS_S3_AWS_ENDPOINT_URL=str,
+    AWS_S3_ACCESS_KEY_ID=str,
+    AWS_S3_SECRET_ACCESS_KEY=str,
+    AWS_S3_REGION=str,
+    S3_STATIC_BUCKET_NAME=str,
+    S3_MEDIA_BUCKET_NAME=str,
+    # -- Azure blob storage
+    USE_AZURE_STORAGE=(bool, False),
+    AZURE_STORAGE_MEDIA_CONTAINER=str,  # media
+    AZURE_STORAGE_STATIC_CONTAINER=str,  # static
+    AZURE_STORAGE_CONNECTION_STRING=(str, None),
+    AZURE_STORAGE_ACCOUNT_NAME=str,
+    AZURE_STORAGE_ACCOUNT_KEY=(str, None),
+    AZURE_STORAGE_TOKEN_CREDENTIAL=(str, None),
+    AZURE_STORAGE_MANAGED_IDENTITY=(bool, False),
     # Sentry
     SENTRY_DSN=(str, None),
     SENTRY_TRACES_SAMPLE_RATE=(float, 0.2),
@@ -238,11 +257,79 @@
 # TODO: Use custom config for static files
 STATICFILES_DIRS = (str(BASE_DIR.joinpath('static')),)
 
-STATIC_URL = env('DJANGO_STATIC_URL')
-MEDIA_URL = env('DJANGO_MEDIA_URL')
 
-STATIC_ROOT = env('DJANGO_STATIC_ROOT')
-MEDIA_ROOT = env('DJANGO_MEDIA_ROOT')
+STATIC_URL = env("DJANGO_STATIC_URL")
+MEDIA_URL = env("DJANGO_MEDIA_URL")
+
+# Django storage
+
+if env("USE_AZURE_STORAGE"):
+
+    AZURE_STORAGE_CONFIG_OPTIONS = {
+        "connection_string": env("AZURE_STORAGE_CONNECTION_STRING"),
+        "overwrite_files": False,
+    }
+
+    if not env("AZURE_STORAGE_CONNECTION_STRING"):
+        AZURE_STORAGE_CONFIG_OPTIONS.update(
+            {
+                "account_name": env("AZURE_STORAGE_ACCOUNT_NAME"),
+                "account_key": env("AZURE_STORAGE_ACCOUNT_KEY"),
+                "token_credential": env("AZURE_STORAGE_TOKEN_CREDENTIAL"),
+            }
+        )
+
+        if env("AZURE_STORAGE_MANAGED_IDENTITY"):
+            AZURE_STORAGE_CONFIG_OPTIONS["token_credential"] = DefaultAzureCredential()
+
+    STORAGES = {
+        "default": {
+            "BACKEND": "storages.backends.azure_storage.AzureStorage",
+            "OPTIONS": {
+                **AZURE_STORAGE_CONFIG_OPTIONS,
+                "azure_container": env("AZURE_STORAGE_MEDIA_CONTAINER"),
+            },
+        },
+        "staticfiles": {
+            "BACKEND": "storages.backends.azure_storage.AzureStorage",
+            "OPTIONS": {
+                **AZURE_STORAGE_CONFIG_OPTIONS,
+                "azure_container": env("AZURE_STORAGE_STATIC_CONTAINER"),
+                "overwrite_files": True,
+            },
+        },
+    }
+
+elif env("USE_S3_BUCKET"):
+    AWS_S3_ENDPOINT_URL = env("AWS_S3_AWS_ENDPOINT_URL")
+
+    AWS_S3_ACCESS_KEY_ID = env("AWS_S3_ACCESS_KEY_ID")
+    AWS_S3_SECRET_ACCESS_KEY = env("AWS_S3_SECRET_ACCESS_KEY")
+    AWS_S3_REGION_NAME = env("AWS_S3_REGION")
+
+    STORAGES = {
+        "default": {
+            "BACKEND": "storages.backends.s3.S3Storage",
+            "OPTIONS": {
+                "bucket_name": env("S3_MEDIA_BUCKET_NAME"),
+                "location": "media/",
+                "file_overwrite": False,
+            },
+        },
+        "staticfiles": {
+            "BACKEND": "storages.backends.s3.S3Storage",
+            "OPTIONS": {
+                "bucket_name": env("S3_STATIC_BUCKET_NAME"),
+                "location": "static/",
+                "file_overwrite": True,
+            },
+        },
+    }
+
+else:
+    STATIC_ROOT = env("DJANGO_STATIC_ROOT")
+    MEDIA_ROOT = env("DJANGO_MEDIA_ROOT")
+
 
 # Default primary key field type
 # https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field