Fix the mounts....

- Removed the deprecated docker-compose.ci.yml file, consolidating configurations into docker-compose.yml.
- Updated environment variable mappings to use OLYMPIA_UID and OLYMPIA_MOUNT for improved clarity and consistency.
- Enhanced entrypoint script to adjust user IDs based on the new environment variables.
- Modified setup.py to determine the appropriate olympia mount based on the target environment.
- Updated GitHub Actions workflows to reflect changes in environment variable usage and remove references to the old compose file.

.github/actions/run-docker/action.yml

@@ -12,42 +12,33 @@ inputs:
   run:
     description: 'Run command in container'
     required: true
-  compose_file:
-    description: 'The docker-compose file to use'
-    required: false
-    default: 'docker-compose.yml:docker-compose.ci.yml'
   logs:
     description: 'Show logs'
     required: false
   data_backup_skip:
     description: 'Skip data backup'
     required: false
     default: 'true'
+  mount:
+    description: 'Mount olympia files from host'
+    required: false
+    default: 'production'
 
 runs:
   using: 'composite'
   steps:
-    - id: id
-      shell: bash
-      run: |
-        echo "id=$(id -u)" >> $GITHUB_OUTPUT
-
     - name: Run Docker Container
       shell: bash
-      env:
-        DOCKER_VERSION: ${{ inputs.version }}
-        DOCKER_DIGEST: ${{ inputs.digest }}
-        COMPOSE_FILE: ${{ inputs.compose_file }}
-        HOST_UID: ${{ steps.id.outputs.id }}
-        DATA_BACKUP_SKIP: ${{ inputs.data_backup_skip }}
-        # In CI, we should use the docker-compose wait flag to ensure
-        # healthchecks are passing before running any commands on the containers.
-        # This comes at a performance cost, but ensures containers are ready
-        # to accept commands before CI continues to execute.
-        DOCKER_WAIT: true
       run: |
         # Start the specified services
-        make up
+        make up \
+          DOCKER_VERSION="${{ inputs.version }}" \
+          DOCKER_DIGEST="${{ inputs.digest }}" \
+          OLYMPIA_UID="$(id -u)" \
+          OLYMPIA_MOUNT="${{ inputs.mount }}" \
+          DATA_BACKUP_SKIP="${{ inputs.data_backup_skip }}" \
+          DOCKER_WAIT="true"
+
 
         # Exec the run command in the container
         # quoted 'EOF' to prevent variable expansion

.github/workflows/_test.yml

@@ -41,29 +41,24 @@ jobs:
           -
             name: Needs Locale Compilation
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: |
               make compile_locales
               make test_needs_locales_compilation
           -
             name: Static Assets
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make test_static_assets
           -
             name: Internal Routes
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make test_internal_routes_allowed
           -
             name: Elastic Search
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make test_es_tests
           -
             name: Codestyle
             services: web
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make lint-codestyle
     steps:
       - uses: actions/checkout@v4
@@ -73,5 +68,4 @@ jobs:
           version: ${{ inputs.version }}
           digest: ${{ inputs.digest }}
           services: ${{ matrix.services }}
-          compose_file: ${{ matrix.compose_file }}
           run: ${{ matrix.run }}

.github/workflows/_test_check.yml

@@ -45,16 +45,16 @@ jobs:
     runs-on: ubuntu-latest
     name: |
       version: '${{ matrix.version }}' |
-      compose_file: '${{ matrix.compose_file }}'
+      mount: '${{ matrix.mount }}'
     strategy:
       fail-fast: false
       matrix:
         version:
           - local
           - ${{ inputs.version }}
-        compose_file:
-          - docker-compose.yml
-          - docker-compose.yml:docker-compose.ci.yml
+        mount:
+          - development
+          - production
     steps:
       - uses: actions/checkout@v4
       - shell: bash
@@ -63,7 +63,7 @@ jobs:
           cat <<EOF
             Values passed to the action:
             version: ${{ matrix.version }}
-            compose_file: ${{ matrix.compose_file }}
+            mount: ${{ matrix.mount }}
           EOF
       - name: ${{ matrix.version == 'local' && 'Uncached Build' || 'Pull' }} Check
         uses: ./.github/actions/run-docker
@@ -73,14 +73,14 @@ jobs:
           DOCKER_VERSION: 'not-expected'
         with:
           version: ${{ matrix.version }}
-          compose_file: ${{ matrix.compose_file }}
+          mount: ${{ matrix.mount }}
           run: make check
       - name: Cached Build Check
         uses: ./.github/actions/run-docker
         if: ${{ matrix.version == 'local' }}
         with:
           version: ${{ matrix.version }}
-          compose_file: ${{ matrix.compose_file }}
+          mount: ${{ matrix.mount }}
           run: echo true
 
   test_make_docker_configuration:

.github/workflows/_test_main.yml

@@ -88,7 +88,7 @@ jobs:
           services: ''
           digest: ${{ inputs.digest }}
           version: ${{ inputs.version }}
-          compose_file: docker-compose.yml
+          mount: development
           run: |
             split="--splits ${{ needs.test_config.outputs.splits }}"
             group="--group ${{ matrix.group }}"

.github/workflows/ci.yml

@@ -88,7 +88,7 @@ jobs:
         with:
           digest: ${{ needs.build.outputs.digest }}
           version: ${{ needs.build.outputs.version }}
-          compose_file: docker-compose.yml
+          mount: development
           run: |
             make docs
 
@@ -139,7 +139,7 @@ jobs:
         with:
           digest: ${{ needs.build.outputs.digest }}
           version: ${{ needs.build.outputs.version }}
-          compose_file: docker-compose.yml
+          mount: development
           run: make extract_locales
 
       - name: Push Locales

.gitignore

@@ -47,7 +47,7 @@ src/olympia/discovery/strings.jinja2
 static-build/*
 static/css/node_lib/*
 static/js/node_lib/*
-storage
+storage/*
 tmp/*
 
 # End of .gitignore. Please keep this in sync with the top section of .dockerignore
@@ -56,3 +56,4 @@ tmp/*
 !docker-compose.ci.yml
 !docker-compose.private.yml
 !private/README.md
+!storage/.gitignore

Makefile-docker

@@ -51,6 +51,8 @@ check_django: ## check if the django app is configured properly
 
 .PHONY: check_nginx
 check_nginx:  ## check if the nginx config for local development is configured properly
+	id -u
+	ls -lan /data/olympia/storage
 	mkdir -p /data/olympia/storage/shared_storage/uploads
 	echo "OK" > /data/olympia/storage/shared_storage/uploads/.check
 	@if [ "$$(curl -sf http://nginx/user-media/.check)" != "OK" ]; then echo "Requesting http://nginx/user-media/.check failed"; exit 1; fi

docker-compose.yml

@@ -20,10 +20,20 @@ x-env-mapping: &env
     - HISTIGNORE=ls:exit:"cd .."
     - HISTCONTROL=erasedups
     - CIRCLECI
-    - HOST_UID
-    - DEBUG
     - DATA_BACKUP_SKIP
 
+# Value is determined by the setup.py script
+# production: (data_olympia):/data/olympia/storage
+# development: (./):/data/olympia/storage
+x-olympia-mount: &olympia-mount
+  ${HOST_MOUNT_SOURCE:?}:/data/olympia
+
+# Value is determined by the setup.py script
+# production: (data_olympia)_storage:/data/olympia/storage
+# development: (./)_storage:/data/olympia/storage
+x-storage-mount: &storage-mount
+  ${HOST_MOUNT_SOURCE:?}storage:/data/olympia/storage
+
 x-site-static-mount: &site-static-mount
   data_site_static:/data/olympia/site-static
 
@@ -49,7 +59,10 @@ services:
     # so we just sleep indefinitely instead.
     command: ["sleep", "infinity"]
     volumes:
+      # used by: web, worker, nginx
+      - *olympia-mount
       - *site-static-mount
+      - *storage-mount
   worker:
     <<: *olympia
     command: [
@@ -64,7 +77,8 @@ services:
       "celery -A olympia.amo.celery:app worker -E -c 2 --loglevel=INFO",
     ]
     volumes:
-      - .:/data/olympia
+      - *olympia-mount
+      - *storage-mount
     extra_hosts:
      - "olympia.test:127.0.0.1"
     restart: on-failure:5
@@ -74,6 +88,7 @@ services:
       retries: 3
       start_interval: 1s
     depends_on:
+      - olympia_volumes
       - mysqld
       - elasticsearch
       - redis
@@ -96,15 +111,14 @@ services:
       # and would otherwiser be deleted by mounting the cwd volume above
       - data_static_build:/data/olympia/static-build
       - *site-static-mount
-    depends_on:
-      - olympia_volumes
 
   nginx:
     image: nginx
     volumes:
       - data_nginx:/etc/nginx/conf.d
-      - .:/srv
+      - *olympia-mount
       - *site-static-mount
+      - *storage-mount
     ports:
       - "80:80"
     networks:
@@ -139,7 +153,7 @@ services:
       # Disable performance schema for faster startup
       - --performance-schema=OFF
     healthcheck:
-      test: ["CMD-SHELL", "mysql -u root --silent --execute='SELECT 1;'"]
+      test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "--silent"]
       start_interval: 1s
       timeout: 2s
       start_period: 10s
@@ -207,6 +221,16 @@ volumes:
   # mounted from the host.
   data_static_build:
   data_site_static:
+  # Volume for the production olympia mounts
+  # allowing to conditionally mount directories
+  # from the host or from the image to <path>
+  # in the running docker container.
+  # If DATA_HOST_MOUNT_PREFIX matches (data_olympia)
+  # then we use the production volume mounts. Otherwise
+  # it will map to the current directory ./<name>
+  # (data_olympia)<name>:/<path>
+  data_olympia_:
+  data_olympia_storage:
   # Volume for rabbitmq/redis to avoid anonymous volumes
   data_rabbitmq:
   data_redis:

docker/entrypoint.sh

@@ -6,7 +6,7 @@
 ### id of the olympia user sometimes should match the host user's id
 ### to avoid permission issues with mounted volumes.
 
-set -ue
+set -xue
 
 if [[ $(id -u) -ne 0 ]]; then
   echo "This script must be run as root"
@@ -18,13 +18,26 @@ OLYMPIA_USER="olympia"
 function get_olympia_uid() { echo "$(id -u "$OLYMPIA_USER")"; }
 function get_olympia_gid() { echo "$(id -g "$OLYMPIA_USER")"; }
 
-if [[ -n "${HOST_UID:-}" ]]; then
+OLD_HOST_UID=$(get_olympia_uid)
+
+# If the olympia user's uid is different in the container than from the build,
+# we need to update the olympia user's uid to match the new one.
+if [[ "${HOST_UID}" != "${OLD_HOST_UID}" ]]; then
   usermod -u ${HOST_UID} ${OLYMPIA_USER}
-  echo "${OLYMPIA_USER} UID: ${OLYMPIA_UID} -> ${HOST_UID}"
+  echo "${OLYMPIA_USER} UID: ${OLD_HOST_UID} -> ${HOST_UID}"
+fi
+
+NEW_HOST_UID=$(get_olympia_uid)
+OLYMPIA_ID_STRING="${NEW_HOST_UID}:$(get_olympia_gid)"
+
+# If we are on production mode, update the ownership of /data/olympia and /deps to match the new id
+if [[ "${HOST_MOUNT}" == "production" ]]; then
+  echo "Updating ownership of /data/olympia and /deps to ${OLYMPIA_ID_STRING}"
+  chown -R ${OLYMPIA_ID_STRING} /data/olympia /deps
 fi
 
 cat <<EOF | su -s /bin/bash $OLYMPIA_USER
-  echo "Running command as ${OLYMPIA_USER} $(get_olympia_uid):$(get_olympia_gid)"
+  echo "Running command as ${OLYMPIA_USER} ${OLYMPIA_ID_STRING}"
   set -xue
   $@
 EOF

docker/nginx/addons.conf

@@ -7,11 +7,11 @@ server {
 
     location /data/olympia/storage/ {
         internal;
-        alias /srv/storage/;
+        alias /data/olympia/storage/;
     }
 
     location /static/ {
-         alias /srv/static/;
+         alias /data/olympia/static/;
 
          # Fallback to the uwsgi server if the file is not found in the static files directory.
          # This will happen for vendor files from pytnon or npm dependencies that won't be available
@@ -20,7 +20,7 @@ server {
     }
 
     location /user-media/ {
-        alias /srv/storage/shared_storage/uploads/;
+        alias /data/olympia/storage/shared_storage/uploads/;
     }
 
     location ~ ^/api/ {