OLYPIA_MOUNT to control host files inclusion + merge docker-compose f…

…iles.

.dockerignore

@@ -53,7 +53,6 @@ tmp/*
 
 # Additionally ignore these files from the docker build that are not in .gitignore
 
-.dockerignore
 .github
 docs
 private

.github/actions/run-docker/action.yml

@@ -12,33 +12,27 @@ inputs:
   run:
     description: 'Run command in container'
     required: true
-  compose_file:
-    description: 'The docker-compose file to use'
-    required: false
-    default: 'docker-compose.yml:docker-compose.ci.yml'
   logs:
     description: 'Show logs'
     required: false
   data_backup_skip:
     description: 'Skip data backup'
     required: false
     default: 'true'
+  mount:
+    description: 'Name of the mount for the host directory (development|production)'
+    required: false
+    default: 'production'
 
 runs:
   using: 'composite'
   steps:
-    - id: id
-      shell: bash
-      run: |
-        echo "id=$(id -u)" >> $GITHUB_OUTPUT
-
     - name: Run Docker Container
       shell: bash
       env:
         DOCKER_VERSION: ${{ inputs.version }}
         DOCKER_DIGEST: ${{ inputs.digest }}
-        COMPOSE_FILE: ${{ inputs.compose_file }}
-        HOST_UID: ${{ steps.id.outputs.id }}
+        OLYMPIA_MOUNT_INPUT: ${{ inputs.mount }}
         DATA_BACKUP_SKIP: ${{ inputs.data_backup_skip }}
       run: |
         # Start the specified services

.github/workflows/_test.yml

@@ -41,29 +41,24 @@ jobs:
           -
             name: Needs Locale Compilation
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: |
               make compile_locales
               make test_needs_locales_compilation
           -
             name: Static Assets
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make test_static_assets
           -
             name: Internal Routes
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make test_internal_routes_allowed
           -
             name: Elastic Search
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make test_es_tests
           -
             name: Codestyle
             services: web
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make lint-codestyle
     steps:
       - uses: actions/checkout@v4
@@ -73,5 +68,4 @@ jobs:
           version: ${{ inputs.version }}
           digest: ${{ inputs.digest }}
           services: ${{ matrix.services }}
-          compose_file: ${{ matrix.compose_file }}
           run: ${{ matrix.run }}

.github/workflows/_test_check.yml

@@ -45,16 +45,16 @@ jobs:
     runs-on: ubuntu-latest
     name: |
       version: '${{ matrix.version }}' |
-      compose_file: '${{ matrix.compose_file }}'
+      mount: '${{ matrix.mount }}' |
     strategy:
       fail-fast: false
       matrix:
         version:
           - local
           - ${{ inputs.version }}
-        compose_file:
-          - docker-compose.yml
-          - docker-compose.yml:docker-compose.ci.yml
+        mount:
+          - development
+          - production
     steps:
       - uses: actions/checkout@v4
       - shell: bash
@@ -63,7 +63,7 @@ jobs:
           cat <<EOF
             Values passed to the action:
             version: ${{ matrix.version }}
-            compose_file: ${{ matrix.compose_file }}
+            mount: ${{ matrix.mount }}
           EOF
       - uses: ./.github/actions/run-docker
         # Set environment variables that are expected to be ignored
@@ -72,7 +72,7 @@ jobs:
           DOCKER_VERSION: 'not-expected'
         with:
           version: ${{ matrix.version }}
-          compose_file: ${{ matrix.compose_file }}
+          mount: ${{ matrix.mount }}
           run: make check
 
   test_make_docker_configuration:

.github/workflows/_test_main.yml

@@ -88,7 +88,7 @@ jobs:
           services: ''
           digest: ${{ inputs.digest }}
           version: ${{ inputs.version }}
-          compose_file: docker-compose.yml
+          mount: development
           run: |
             split="--splits ${{ needs.test_config.outputs.splits }}"
             group="--group ${{ matrix.group }}"

.github/workflows/ci.yml

@@ -88,7 +88,7 @@ jobs:
         with:
           digest: ${{ needs.build.outputs.digest }}
           version: ${{ needs.build.outputs.version }}
-          compose_file: docker-compose.yml
+          mount: development
           run: |
             make docs
 
@@ -139,7 +139,7 @@ jobs:
         with:
           digest: ${{ needs.build.outputs.digest }}
           version: ${{ needs.build.outputs.version }}
-          compose_file: docker-compose.yml
+          mount: development
           run: make extract_locales
 
       - name: Push Locales

.gitignore

@@ -53,6 +53,5 @@ tmp/*
 # End of .gitignore. Please keep this in sync with the top section of .dockerignore
 
 # do not ignore the following files
-!docker-compose.ci.yml
 !docker-compose.private.yml
 !private/README.md

docker-compose.yml

@@ -20,10 +20,14 @@ x-env-mapping: &env
     - HISTIGNORE=ls:exit:"cd .."
     - HISTCONTROL=erasedups
     - CIRCLECI
-    - HOST_UID
-    - DEBUG
     - DATA_BACKUP_SKIP
 
+# Define the volume to use for the olympia mount
+# production: mount the files from the container, effectively disabling the mount
+# development: mount the host files, mapping host files into the container
+x-olympia-mount: &olympia-mount
+  data_olympia_${OLYMPIA_MOUNT}:/data/olympia
+
 x-site-static-mount: &site-static-mount
   data_site_static:/data/olympia/site-static
 
@@ -49,6 +53,9 @@ services:
     # so we just sleep indefinitely instead.
     command: ["sleep", "infinity"]
     volumes:
+      # used by: web, worker, nginx
+      - *olympia-mount
+      # used by: web, worker, nginx
       - *site-static-mount
   worker:
     <<: *olympia
@@ -63,7 +70,7 @@ services:
       "celery -A olympia.amo.celery:app worker -E -c 2 --loglevel=INFO",
     ]
     volumes:
-      - .:/data/olympia
+      - *olympia-mount
 
     extra_hosts:
      - "olympia.test:127.0.0.1"
@@ -81,6 +88,7 @@ services:
       # The start period is 60s
       start_period: 120s
     depends_on:
+      - olympia_volumes
       - mysqld
       - elasticsearch
       - redis
@@ -106,14 +114,12 @@ services:
       - *site-static-mount
       - ./package.json:/deps/package.json
       - ./package-lock.json:/deps/package-lock.json
-    depends_on:
-      - olympia_volumes
 
   nginx:
     image: nginx
     volumes:
       - ./docker/nginx/addons.conf:/etc/nginx/conf.d/addons.conf
-      - .:/srv
+      - *olympia-mount
       - *site-static-mount
     ports:
       - "80:80"
@@ -201,6 +207,17 @@ volumes:
   # Volumes for static files that should not be
   # mounted from the host.
   data_site_static:
+  # volumes for conditionally mounting host files
+  # we use two volumes to prvent corrupting the
+  # production volume when switching between the two
+  data_olympia_production:
+  # The development volume is mounted from the host files
+  data_olympia_development:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${PWD}
   data_mysqld:
     # Keep this value in sync with Makefile-os
     # External volumes must be manually created/destroyed

docker/entrypoint.sh

@@ -18,9 +18,9 @@ OLYMPIA_USER="olympia"
 function get_olympia_uid() { echo "$(id -u "$OLYMPIA_USER")"; }
 function get_olympia_gid() { echo "$(id -g "$OLYMPIA_USER")"; }
 
-if [[ -n "${HOST_UID:-}" ]]; then
-  usermod -u ${HOST_UID} ${OLYMPIA_USER}
-  echo "${OLYMPIA_USER} UID: ${OLYMPIA_UID} -> ${HOST_UID}"
+if [[ "${OLYMPIA_MOUNT}" == "development" ]]; then
+  usermod -u ${OLYMPIA_UID} ${OLYMPIA_USER}
+  echo "Updating ${OLYMPIA_USER} UID to ${OLYMPIA_UID}"
 fi
 
 cat <<EOF | su -s /bin/bash $OLYMPIA_USER

docker/nginx/addons.conf

@@ -7,11 +7,11 @@ server {
 
     location /data/olympia/storage/ {
         internal;
-        alias /srv/storage/;
+        alias /data/olympia/storage/;
     }
 
     location /static/ {
-         alias /srv/static/;
+         alias /data/olympia/static/;
 
          # Fallback to the uwsgi server if the file is not found in the static files directory.
          # This will happen for vendor files from pytnon or npm dependencies that won't be available
@@ -20,7 +20,7 @@ server {
     }
 
     location /user-media/ {
-        alias /srv/storage/shared_storage/uploads/;
+        alias /data/olympia/storage/shared_storage/uploads/;
     }
 
     location ~ ^/api/ {

docs/topics/development/building_and_running_services.md

@@ -16,9 +16,9 @@ The Dockerfile for the **addons-server** project uses a multi-stage build to opt
    - **Mounting Local Repository**: The volume `.:/data/olympia` mounts the local Git repository into the container, allowing real-time changes to files within the container.
    - **Mounting Dependencies**: The volume `./deps:/deps` mounts the dependencies directory, enabling better caching across builds and providing visibility for debugging directly on the host.
 
-4. **Environment Variables for OLYMPIA_USER**:
-   - **Development Setup**: The `HOST_UID` environment variable is set to the host user ID, ensuring that the container runs with the correct permissions.
-   - **CI Setup**: In CI environments, such as defined in `docker-compose.ci.yml`, the user ID is reset to the default 9500, and the Olympia mount is removed. This makes the container a closed system, mimicking production behavior closely.
+4. **Environment Variables**:
+   - **Development Setup**: The `OLYMPIA_UID` and `OLYMPIA_MOUNT_INPUT` environment variables are set to the host user ID and mount host, ensuring that the container runs with the correct permissions and mounts the correct host directory.
+   - **CI Setup**: In CI environments, the Olympia mount is removed by default. This makes the container a closed system, mimicking production behavior closely.
 
 ### Best Practices for the Dockerfile
 
@@ -146,7 +146,7 @@ We use docker compose under the hood to orchestrate container both locally and i
 The `docker-compose.yml` file defines the services, volumes, and networks required for the project.
 
 Our docker compose project is split into a root [docker-compose.yml](../../../docker-compose.yml) file and additional files for specific environments,
-such as [docker-compose.ci.yml](../../../docker-compose.ci.yml) for CI environments.
+such as [docker-compose.override.yml](../../../docker-compose.override.yml) for CI environments.
 
 ### Healthchecks
 
@@ -156,14 +156,13 @@ The health checks ensure the django wsgi server and celery worker node are runni
 ### Environment specific compose files
 
 - **Local Development**: The `docker-compose.yml` file is used for local development. It defines services like `web`, `db`, `redis`, and `elasticsearch`.
-- **CI Environment**: The `docker-compose.ci.yml` file is used for CI environments. It overrides the HOST_UID as well as removing volumes to make the container more production like.
 - **Private**: This file includes the customs service that is not open source and should therefore not be included by default.
 - **Override**: This file allows modifying the default configuration without changing the main `docker-compose.yml` file. This file is larglely obsolete and should not be used.
 
 To mount with a specific  set of docker compose files you can add the COMPOSE_FILE argument to make up. This will persist your setting to .env.
 
 ```sh
-make up COMPOSE_FILE=docker-compose.yml:docker-compose.ci.yml
+make up COMPOSE_FILE=docker-compose.yml:docker-compose.override.yml
 ```
 
 Files should be separated with a colon.

docs/topics/development/setup_and_configuration.md

@@ -187,7 +187,6 @@ Though it is **highly recommended to use the make commands** instead of directly
 ### Docker Compose Files
 
 - **[docker-compose.yml][docker-compose]**: The primary Docker Compose file defining services, networks, and volumes for local and CI environments.
-- **[docker-compose.ci.yml][docker-compose-ci]**: Overrides certain configurations for CI-specific needs, ensuring the environment is optimized for automated testing and builds.
 - **[docker-compose.private.yml][docker-compose-private]**: Runs addons-server with the _customs_ service that is only available to Mozilla employees
 
 Our docker compose files rely on substituted values, all of which are included in our .env file for direct CLI compatibility.
@@ -317,7 +316,6 @@ and docker-comose.yml file locally.
 To fix this error `rm -f .env` to remove your .env and `make up` to restart the containers.
 
 [docker-compose]: ../../../docker-compose.yml
-[docker-compose-ci]: ../../../docker-compose.ci.yml
 [docker-compose-private]: ../../../docker-compose.private.yml
 [docker-image-digest]: https://github.com/opencontainers/.github/blob/main/docs/docs/introduction/digests.md
 [addons-server-tags]: https://hub.docker.com/r/mozilla/addons-server/tags

scripts/setup.py

@@ -91,32 +91,33 @@ def main():
     docker_target = get_value(
         'DOCKER_TARGET', ('development' if is_local else 'production')
     )
+    # On development images, we ignore the user provided OLYMPIA_MOUNT_INPUT
+    # and hard code the volume to development. This is because neither
+    # the image nor the volume would provide the files needed by the container.
+    # That is also why the value saved to .env is different from the input value.
+    # This way the docker-compose.yml and the container only read the computed
+    # OLYMPIA_MOUNT value and not the OLYMPIA_MOUNT_INPUT.
+    data_olympia_mount = (
+        docker_target
+        if docker_target == 'development'
+        else get_value('OLYMPIA_MOUNT_INPUT', get_value('OLYMPIA_MOUNT', docker_target))
+    )
 
     is_production = docker_target == 'production'
 
-    # The default value for which compose files to use is based on the target
-    # but can be freely overridden by the user.
-    # E.g running a production image in development mode with source code changes
-    compose_file = get_value(
-        'COMPOSE_FILE',
-        (
-            'docker-compose.yml:docker-compose.ci.yml'
-            if is_production
-            else 'docker-compose.yml'
-        ),
-    )
-
     # DEBUG is special, as we should allow the user to override it
     # but we should not set a default to the previously set value but instead
     # to the most sensible default.
     debug = os.environ.get('DEBUG', str(False if is_production else True))
+    # OLYMPIA_UID should always be set to the current user's UID
+    host_uid = os.getuid()
 
     set_env_file(
         {
-            'COMPOSE_FILE': compose_file,
             'DOCKER_TAG': docker_tag,
             'DOCKER_TARGET': docker_target,
-            'HOST_UID': get_value('HOST_UID', os.getuid()),
+            'OLYMPIA_UID': host_uid,
+            'OLYMPIA_MOUNT': data_olympia_mount,
             'DEBUG': debug,
         }
     )