TMP: fix

.github/actions/run-docker/action.yml

@@ -27,25 +27,16 @@ inputs:
 runs:
   using: 'composite'
   steps:
-    - id: id
-      shell: bash
-      run: |
-        echo "id=$(id -u)" >> $GITHUB_OUTPUT
-
     - name: Run Docker Container
       shell: bash
       run: |
         # Start the specified services
         make up \
           DOCKER_VERSION="${{ inputs.version }}" \
           DOCKER_DIGEST="${{ inputs.digest }}" \
-          OLYMPIA_UID="${{ steps.id.outputs.id }}" \
+          OLYMPIA_UID="$(id -u)" \
           OLYMPIA_MOUNT="${{ inputs.mount }}" \
           DATA_BACKUP_SKIP="${{ inputs.data_backup_skip }}" \
-          # In CI, we should use the docker-compose wait flag to ensure \
-          # healthchecks are passing before running any commands on the containers. \
-          # This comes at a performance cost, but ensures containers are ready \
-          # to accept commands before CI continues to execute. \
           DOCKER_WAIT="true"
 
 

.gitignore

@@ -47,7 +47,7 @@ src/olympia/discovery/strings.jinja2
 static-build/*
 static/css/node_lib/*
 static/js/node_lib/*
-storage
+storage/*
 tmp/*
 
 # End of .gitignore. Please keep this in sync with the top section of .dockerignore
@@ -56,3 +56,4 @@ tmp/*
 !docker-compose.ci.yml
 !docker-compose.private.yml
 !private/README.md
+!storage/.gitignore

Makefile-docker

@@ -51,6 +51,8 @@ check_django: ## check if the django app is configured properly
 
 .PHONY: check_nginx
 check_nginx:  ## check if the nginx config for local development is configured properly
+	id -u
+	ls -lan /data/olympia/storage
 	mkdir -p /data/olympia/storage/shared_storage/uploads
 	echo "OK" > /data/olympia/storage/shared_storage/uploads/.check
 	@if [ "$$(curl -sf http://nginx/user-media/.check)" != "OK" ]; then echo "Requesting http://nginx/user-media/.check failed"; exit 1; fi

docker-compose.yml

@@ -32,7 +32,7 @@ x-olympia-mount: &olympia-mount
 # production: (data_olympia)_storage:/data/olympia/storage
 # development: (./)_storage:/data/olympia/storage
 x-storage-mount: &storage-mount
-  ${HOST_MOUNT_SOURCE:?}_storage:/data/olympia/storage
+  ${HOST_MOUNT_SOURCE:?}storage:/data/olympia/storage
 
 x-site-static-mount: &site-static-mount
   data_site_static:/data/olympia/site-static
@@ -153,7 +153,7 @@ services:
       # Disable performance schema for faster startup
       - --performance-schema=OFF
     healthcheck:
-      test: ["CMD-SHELL", "mysql -u root --silent --execute='SELECT 1;'"]
+      test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "--silent"]
       start_interval: 1s
       timeout: 2s
       start_period: 10s
@@ -229,7 +229,7 @@ volumes:
   # then we use the production volume mounts. Otherwise
   # it will map to the current directory ./<name>
   # (data_olympia)<name>:/<path>
-  data_olympia:
+  data_olympia_:
   data_olympia_storage:
   # Volume for rabbitmq/redis to avoid anonymous volumes
   data_rabbitmq:

docker/nginx/addons.conf

@@ -7,11 +7,11 @@ server {
 
     location /data/olympia/storage/ {
         internal;
-        alias /srv/storage/;
+        alias /data/olympia/storage/;
     }
 
     location /static/ {
-         alias /srv/static/;
+         alias /data/olympia/static/;
 
          # Fallback to the uwsgi server if the file is not found in the static files directory.
          # This will happen for vendor files from pytnon or npm dependencies that won't be available
@@ -20,7 +20,7 @@ server {
     }
 
     location /user-media/ {
-        alias /srv/storage/shared_storage/uploads/;
+        alias /data/olympia/storage/shared_storage/uploads/;
     }
 
     location ~ ^/api/ {

docs/topics/development/building_and_running_services.md

@@ -17,8 +17,7 @@ The Dockerfile for the **addons-server** project uses a multi-stage build to opt
    - **Mounting Dependencies**: The volume `./deps:/deps` mounts the dependencies directory, enabling better caching across builds and providing visibility for debugging directly on the host.
 
 4. **Environment Variables for OLYMPIA_USER**:
-   - **Development Setup**: The `HOST_UID` environment variable is set to the host user ID, ensuring that the container runs with the correct permissions.
-   - **CI Setup**: In CI environments, such as defined in `docker-compose.ci.yml`, the user ID is reset to the default 9500, and the Olympia mount is removed. This makes the container a closed system, mimicking production behavior closely.
+   - **Development Setup**: The `OLYMPIA_UID` .env variable is set to the host user ID, ensuring that the container runs with the correct permissions.
 
 ### Best Practices for the Dockerfile
 
@@ -146,7 +145,7 @@ We use docker compose under the hood to orchestrate container both locally and i
 The `docker-compose.yml` file defines the services, volumes, and networks required for the project.
 
 Our docker compose project is split into a root [docker-compose.yml](../../../docker-compose.yml) file and additional files for specific environments,
-such as [docker-compose.ci.yml](../../../docker-compose.ci.yml) for CI environments.
+such as [docker-compose.override.yml](../../../docker-compose.override.yml) for CI environments.
 
 ### Healthchecks
 
@@ -156,14 +155,13 @@ The health checks ensure the django wsgi server and celery worker node are runni
 ### Environment specific compose files
 
 - **Local Development**: The `docker-compose.yml` file is used for local development. It defines services like `web`, `db`, `redis`, and `elasticsearch`.
-- **CI Environment**: The `docker-compose.ci.yml` file is used for CI environments. It overrides the HOST_UID as well as removing volumes to make the container more production like.
 - **Private**: This file includes the customs service that is not open source and should therefore not be included by default.
 - **Override**: This file allows modifying the default configuration without changing the main `docker-compose.yml` file. This file is larglely obsolete and should not be used.
 
 To mount with a specific  set of docker compose files you can add the COMPOSE_FILE argument to make up. This will persist your setting to .env.
 
 ```sh
-make up COMPOSE_FILE=docker-compose.yml:docker-compose.ci.yml
+make up COMPOSE_FILE=docker-compose.yml:docker-compose.override.yml
 ```
 
 Files should be separated with a colon.

docs/topics/development/setup_and_configuration.md

@@ -187,7 +187,6 @@ Though it is **highly recommended to use the make commands** instead of directly
 ### Docker Compose Files
 
 - **[docker-compose.yml][docker-compose]**: The primary Docker Compose file defining services, networks, and volumes for local and CI environments.
-- **[docker-compose.ci.yml][docker-compose-ci]**: Overrides certain configurations for CI-specific needs, ensuring the environment is optimized for automated testing and builds.
 - **[docker-compose.private.yml][docker-compose-private]**: Runs addons-server with the _customs_ service that is only available to Mozilla employees
 
 Our docker compose files rely on substituted values, all of which are included in our .env file for direct CLI compatibility.
@@ -317,7 +316,6 @@ and docker-comose.yml file locally.
 To fix this error `rm -f .env` to remove your .env and `make up` to restart the containers.
 
 [docker-compose]: ../../../docker-compose.yml
-[docker-compose-ci]: ../../../docker-compose.ci.yml
 [docker-compose-private]: ../../../docker-compose.private.yml
 [docker-image-digest]: https://github.com/opencontainers/.github/blob/main/docs/docs/introduction/digests.md
 [addons-server-tags]: https://hub.docker.com/r/mozilla/addons-server/tags

scripts/setup.py

@@ -65,18 +65,22 @@ def get_docker_tag():
 
     return tag, version, digest
 
+
 def get_olympia_mount(docker_target):
     """
     When running on production targets, the user can specify the olympia mount
     to one of the valid values: development or production. In development, we
     hard code the values to ensure we have necessary files and permissions.
     """
-    olympia_mount = os.environ.get('OLYMPIA_MOUNT', docker_target)
+    olympia_mount = os.environ.get(
+        'OLYMPIA_MOUNT',
+        get_value('OLYMPIA_MOUNT', docker_target),
+    )
     olympia_mount_source = './'
 
     if docker_target == 'production' and olympia_mount == 'production':
         # This is the prefix of the volume name.
-        olympia_mount_source = 'data_olympia'
+        olympia_mount_source = 'data_olympia_'
 
     return olympia_mount, olympia_mount_source
 

src/olympia/amo/management/commands/initialize.py

@@ -13,6 +13,11 @@ class Command(BaseDataCommand):
     Ensures the database has the correct state.
     """
 
+    # We don't want to run system checks here, because this command
+    # can run before everything is ready.
+    # we run them at the end of the command.
+    requires_system_checks = []
+
     help = 'Creates, seeds, and indexes the database.'
 
     def add_arguments(self, parser):
@@ -74,3 +79,6 @@ def handle(self, *args, **options):
             services=['localdev_web', 'celery_worker', 'elastic', 'rabbitmq', 'signer'],
             attempts=10,
         )
+
+        # Finally, run the django checks to ensure everything is ok.
+        call_command('check')

src/olympia/amo/monitors.py

@@ -39,11 +39,14 @@ def localdev_web():
     and some json via the uwsgi http server.
     """
     status = ''
-    response = requests.get('http://127.0.0.1:8002/__version__')
-
-    if response.status_code != 200:
-        status = f'Failed to ping web with status code: {response.status_code}'
+    try:
+        response = requests.get('http://127.0.0.1:8002/__version__')
+        response.raise_for_status()
+    except Exception as e:
+        status = f'Failed to ping web: {e}'
         monitor_log.critical(status)
+        return status, None
+
     return status, None
 
 

src/olympia/amo/tests/test_commands.py

@@ -152,6 +152,7 @@ def test_generate_jsi18n_files():
 
 class BaseTestDataCommand(TestCase):
     class Commands:
+        check = mock.call('check')
         reset_db = mock.call('reset_db', '--no-utf8', '--noinput')
         monitors_olympia_database = mock.call('monitors', services=['olympia_database'])
         monitors_database = mock.call('monitors', services=['database'])
@@ -271,6 +272,7 @@ def test_handle_with_skip_data_initialize(self):
             [
                 self.mock_commands.monitors_olympia_database,
                 self.mock_commands.monitors,
+                self.mock_commands.check,
             ],
         )
 
@@ -287,6 +289,7 @@ def test_handle_with_load_argument_and_skip_data_initialize(self):
             [
                 self.mock_commands.monitors_olympia_database,
                 self.mock_commands.monitors,
+                self.mock_commands.check,
             ],
         )
 
@@ -305,6 +308,7 @@ def test_handle_with_clean_and_load_arguments(self):
                 self.mock_commands.data_seed,
                 self.mock_commands.monitors_database,
                 self.mock_commands.monitors,
+                self.mock_commands.check,
             ],
         )
 
@@ -322,6 +326,7 @@ def test_handle_with_clean_argument_no_local_admin(self):
                 self.mock_commands.data_seed,
                 self.mock_commands.monitors_database,
                 self.mock_commands.monitors,
+                self.mock_commands.check,
             ],
         )
 
@@ -341,6 +346,7 @@ def test_handle_without_clean_or_load_with_local_admin(self):
                 self.mock_commands.reindex_skip_if_exists,
                 self.mock_commands.monitors_database,
                 self.mock_commands.monitors,
+                self.mock_commands.check,
             ],
         )
 
@@ -358,6 +364,7 @@ def test_handle_without_clean_or_load_without_local_admin(self):
                 self.mock_commands.data_seed,
                 self.mock_commands.monitors_database,
                 self.mock_commands.monitors,
+                self.mock_commands.check,
             ],
         )
 
@@ -391,6 +398,7 @@ def test_handle_mysql_exception(self, mock_filter):
                 self.mock_commands.data_seed,
                 self.mock_commands.monitors_database,
                 self.mock_commands.monitors,
+                self.mock_commands.check,
             ],
         )
 

src/olympia/amo/tests/test_monitor.py

@@ -154,7 +154,7 @@ def test_localdev_web_fail(self):
             status=500,
         )
         status, _ = monitors.localdev_web()
-        assert status == 'Failed to ping web with status code: 500'
+        assert 'Failed to ping web' in status
 
     def test_localdev_web_success(self):
         responses.add(

tests/make/make.spec.js

@@ -101,9 +101,7 @@ describe('docker-compose.yml', () => {
         expect(service.volumes).toEqual(
           expect.arrayContaining([
             expect.objectContaining({
-              source: isProdMountTarget
-                ? 'data_olympia'
-                : expect.any(String),
+              source: isProdMountTarget ? 'data_olympia_' : expect.any(String),
               target: '/data/olympia',
             }),
             expect.objectContaining({
@@ -114,9 +112,10 @@ describe('docker-compose.yml', () => {
             }),
           ]),
         );
+        const { OLYMPIA_MOUNT, ...environmentOutput } = inputValues;
         expect(service.environment).toEqual(
           expect.objectContaining({
-            ...inputValues,
+            ...environmentOutput,
           }),
         );
         // We excpect not to pass the input values to the container
@@ -157,20 +156,18 @@ describe('docker-compose.yml', () => {
             source: 'data_nginx',
             target: '/etc/nginx/conf.d',
           }),
-          // // mapping for local host directory to /data/olympia
-          // expect.objectContaining({
-          //   source: isProdMountTarget
-          //     ? 'data_olymmpia'
-          //     : expect.any(String),
-          //   target: '/data/olympia',
-          // }),
-          // // mapping for local host directory to /data/olympia/storage
-          // expect.objectContaining({
-          //   source: isProdMountTarget
-          //     ? 'data_olympia_storage'
-          //     : expect.any(String),
-          //   target: '/data/olympia/storage',
-          // }),
+          // mapping for local host directory to /data/olympia
+          expect.objectContaining({
+            source: isProdMountTarget ? 'data_olympia_' : expect.any(String),
+            target: '/data/olympia',
+          }),
+          // mapping for local host directory to /data/olympia/storage
+          expect.objectContaining({
+            source: isProdMountTarget
+              ? 'data_olympia_storage'
+              : expect.any(String),
+            target: '/data/olympia/storage',
+          }),
         ]),
       );
     });

tests/make/test_setup.py

@@ -9,7 +9,14 @@ def override_env(**kwargs):
     return mock.patch.dict(os.environ, kwargs, clear=True)
 
 
-keys = ['COMPOSE_FILE', 'DOCKER_TAG', 'DOCKER_TARGET', 'HOST_UID', 'DEBUG']
+keys = [
+    'DOCKER_TAG',
+    'DOCKER_TARGET',
+    'HOST_UID',
+    'HOST_MOUNT',
+    'HOST_MOUNT_SOURCE',
+    'DEBUG',
+]
 
 
 class BaseTestClass(unittest.TestCase):
@@ -130,25 +137,6 @@ def test_default_env_file(self):
         self.assert_set_env_file_called_with(DOCKER_TARGET='production')
 
 
-@override_env()
-class TestComposeFile(BaseTestClass):
-    def test_default_compose_file(self):
-        main()
-        self.assert_set_env_file_called_with(COMPOSE_FILE='docker-compose.yml')
-
-    @override_env(DOCKER_TARGET='production')
-    def test_default_target_production(self):
-        main()
-        self.assert_set_env_file_called_with(
-            COMPOSE_FILE='docker-compose.yml:docker-compose.ci.yml'
-        )
-
-    @override_env(COMPOSE_FILE='test')
-    def test_compose_file_override(self):
-        main()
-        self.assert_set_env_file_called_with(COMPOSE_FILE='test')
-
-
 @override_env()
 class TestDebug(BaseTestClass):
     def test_default_debug(self):