OLYPIA_MOUNT named host volume mount:

- Updated `docker-compose.ci.yml` and `docker-compose.yml` to support separate volume mounts for development and production.
- Introduced `OLYMPIA_MOUNT` environment variable to toggle between development and production settings.
- Modified `entrypoint.sh` to conditionally adjust user permissions based on the `OLYMPIA_MOUNT` setting.
- Enhanced `host_check` function in `apps.py` to validate the presence of ignored files in production.
- Added tests to ensure that production settings correctly prevent the presence of certain files.

This commit improves the flexibility and reliability of the Docker setup for different environments.

docker-compose.ci.yml

@@ -2,9 +2,10 @@ services:
   olympia_volumes:
     environment:
       - HOST_UID=9500
+      - OLYMPIA_MOUNT=production
     volumes:
+      - data_olympia_production:/data/olympia
       - storage:/data/olympia/storage
-      - /data/olympia
 
   worker:
     extends:
@@ -17,9 +18,14 @@ services:
 
   nginx:
     volumes:
+      - data_olympia_production:/srv
       - storage:/srv/storage
     depends_on:
       - olympia_volumes
 
 volumes:
   storage:
+  # volumes for conditionally mounting host files
+  # we use two volumes to prvent corrupting the
+  # production volume when switching between the two
+  data_olympia_production:

docker-compose.yml

@@ -21,9 +21,16 @@ x-env-mapping: &env
     - HISTCONTROL=erasedups
     - CIRCLECI
     - HOST_UID
+    - OLYMPIA_MOUNT=development
     - DEBUG
     - DATA_BACKUP_SKIP
 
+# Define the volume to use for the olympia mount
+# production: mount the files from the container, effectively disabling the mount
+# development: mount the host files, mapping host files into the container
+x-olympia-mount: &olympia-mount
+  data_olympia_development:/data/olympia
+
 x-site-static-mount: &site-static-mount
   data_site_static:/data/olympia/site-static
 
@@ -49,6 +56,9 @@ services:
     # so we just sleep indefinitely instead.
     command: ["sleep", "infinity"]
     volumes:
+      # used by: web, worker, nginx
+      - *olympia-mount
+      # used by: web, worker, nginx
       - *site-static-mount
   worker:
     <<: *olympia
@@ -63,7 +73,7 @@ services:
       "celery -A olympia.amo.celery:app worker -E -c 2 --loglevel=INFO",
     ]
     volumes:
-      - .:/data/olympia
+      - *olympia-mount
 
     extra_hosts:
      - "olympia.test:127.0.0.1"
@@ -81,6 +91,7 @@ services:
       # The start period is 60s
       start_period: 120s
     depends_on:
+      - olympia_volumes
       - mysqld
       - elasticsearch
       - redis
@@ -106,14 +117,12 @@ services:
       - *site-static-mount
       - ./package.json:/deps/package.json
       - ./package-lock.json:/deps/package-lock.json
-    depends_on:
-      - olympia_volumes
 
   nginx:
     image: nginx
     volumes:
       - ./docker/nginx/addons.conf:/etc/nginx/conf.d/addons.conf
-      - .:/srv
+      - data_olympia_development:/srv
       - *site-static-mount
     ports:
       - "80:80"
@@ -201,6 +210,13 @@ volumes:
   # Volumes for static files that should not be
   # mounted from the host.
   data_site_static:
+  # The development volume is mounted from the host files
+  data_olympia_development:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${PWD}
   data_mysqld:
     # Keep this value in sync with Makefile-os
     # External volumes must be manually created/destroyed

docker/entrypoint.sh

@@ -18,7 +18,7 @@ OLYMPIA_USER="olympia"
 function get_olympia_uid() { echo "$(id -u "$OLYMPIA_USER")"; }
 function get_olympia_gid() { echo "$(id -g "$OLYMPIA_USER")"; }
 
-if [[ -n "${HOST_UID:-}" ]]; then
+if [[ "${OLYMPIA_MOUNT}" == "development" && -n "${HOST_UID}" ]]; then
   usermod -u ${HOST_UID} ${OLYMPIA_USER}
   echo "${OLYMPIA_USER} UID: ${OLYMPIA_UID} -> ${HOST_UID}"
 fi

settings.py

@@ -19,6 +19,7 @@
 DEV_MODE = DOCKER_TARGET != 'production'
 
 HOST_UID = os.environ.get('HOST_UID')
+OLYMPIA_MOUNT = os.environ.get('OLYMPIA_MOUNT')
 
 WSGI_APPLICATION = 'olympia.wsgi.application'
 

src/olympia/core/apps.py

@@ -43,20 +43,48 @@ def uwsgi_check(app_configs, **kwargs):
 def host_check(app_configs, **kwargs):
     """Check that the host settings are valid."""
     errors = []
+    is_production = (
+        settings.OLYMPIA_MOUNT is None or settings.OLYMPIA_MOUNT == 'production'
+    )
 
     # In production, we expect settings.HOST_UID to be None and so
     # set the expected uid to 9500, otherwise we expect the uid
     # passed to the environment to be the expected uid.
-    expected_uid = 9500 if settings.HOST_UID is None else int(settings.HOST_UID)
-    actual_uid = os.getuid()
+    expected_uid = (
+        9500
+        if is_production or settings.HOST_UID is None
+        else int(settings.HOST_UID)
+    )
+    actual_user_uid = os.getuid()
+
+    if actual_user_uid != expected_uid:
+        errors.append(
+            Error(
+                f'Expected user uid to be {expected_uid}, received {actual_user_uid}',
+                id='setup.E002',
+            )
+        )
 
-    if actual_uid != expected_uid:
-        return [
+    # check the ownership of the files in the /data/olympia directory are owned by actual_uid
+    actual_dir_uid = os.stat('/data/olympia').st_uid
+    if actual_dir_uid != expected_uid:
+        errors.append(
             Error(
-                f'Expected user uid to be {expected_uid}, received {actual_uid}',
+                f'Expected /data/olympia directory to be owned by {expected_uid}, '
+                f'received {actual_dir_uid}',
                 id='setup.E002',
             )
-        ]
+        )
+
+    # In production we files matching the dockerignore file to be excluded
+    if is_production and os.path.exists('/data/olympia/Makefile-os'):
+        errors.append(
+            Error(
+                f'Makefile-os should be excluded by '
+                'dockerignore in production images',
+                id='setup.E002',
+            )
+        )
 
     return errors
 

src/olympia/core/tests/test_apps.py

@@ -1,12 +1,59 @@
+import os
 from unittest import mock
 
 from django.core.management import call_command
 from django.core.management.base import SystemCheckError
 from django.test import TestCase
 from django.test.utils import override_settings
 
+from olympia.core.apps import host_check
+
 
 class SystemCheckIntegrationTest(TestCase):
+    @override_settings(OLYMPIA_MOUNT='production', HOST_UID='9500')
+    def test_unexpected_data_olympia_directory_uid_check(self):
+        """
+        Check that the /data/olympia directory is owned by the expected uid.
+        """
+        original_stat = os.stat
+
+        def mock_stat(path, *args, **kwargs):
+            if path == '/data/olympia':
+                return mock.Mock(st_uid=1000)
+            return original_stat(path)
+
+        with mock.patch('olympia.core.apps.os.stat', side_effect=mock_stat):
+            with self.assertRaisesMessage(
+                SystemCheckError,
+                'Expected /data/olympia directory to be owned by 9500, received 1000',
+            ):
+                call_command('check')
+
+    def test_illegal_host_files_exists_check(self):
+        """
+        In production, or when the host mount is set to production, we expect
+        not to find docker ignored files like Makefile-os in the file system.
+        """
+        original_exists = os.path.exists
+
+        def mock_exists(path):
+            if path == '/data/olympia/Makefile-os':
+                return True
+            return original_exists(path)
+
+        for host_mount in (None, 'production'):
+            with self.subTest(host_mount=host_mount):
+                with override_settings(OLYMPIA_MOUNT=host_mount, HOST_UID=9500):
+                    with mock.patch(
+                        'olympia.core.apps.os.path.exists', side_effect=mock_exists
+                    ):
+                        with self.assertRaisesMessage(
+                            SystemCheckError,
+                            'Makefile-os should be excluded by '
+                            'dockerignore in production images',
+                        ):
+                            call_command('check')
+
     @mock.patch('olympia.core.apps.os.getuid')
     def test_illegal_override_uid_check(self, mock_getuid):
         """
@@ -26,7 +73,7 @@ def test_illegal_override_uid_check(self, mock_getuid):
                     ):
                         call_command('check')
 
-        with override_settings(HOST_UID=dummy_uid):
+        with override_settings(HOST_UID=dummy_uid, OLYMPIA_MOUNT='development'):
             mock_getuid.return_value = int(olympia_uid)
             with self.assertRaisesMessage(
                 SystemCheckError,

src/olympia/lib/settings_base.py

@@ -73,6 +73,7 @@ def path(*folders):
 
 # Host info that is hard coded for production images.
 HOST_UID = None
+OLYMPIA_MOUNT = None
 
 # Used to determine if django should serve static files.
 # For local deployments we want nginx to proxy static file requests to the

tests/make/make.spec.js

@@ -97,7 +97,7 @@ describe('docker-compose.yml', () => {
         expect(service.volumes).toEqual(
           expect.arrayContaining([
             expect.objectContaining({
-              ...(isProd ? {} : { source: expect.any(String) }),
+              source: `data_olympia_${target}`,
               target: '/data/olympia',
             }),
             expect.objectContaining(
@@ -159,7 +159,7 @@ describe('docker-compose.yml', () => {
           }),
           // mapping for local host directory to /data/olympia
           expect.objectContaining({
-            source: expect.any(String),
+            source: `data_olympia_${target}`,
             target: '/srv',
           }),
           // mapping for local host directory to /data/olympia/storage