From dd214d3e94b2170416161e2e53206b1501bc99da Mon Sep 17 00:00:00 2001 From: Kevin Meinhardt Date: Tue, 17 Dec 2024 19:25:14 +0100 Subject: [PATCH] TMP: fix --- .github/actions/run-docker/action.yml | 11 +------ .gitignore | 3 +- Makefile-docker | 2 ++ docker-compose.yml | 6 ++-- docker/nginx/addons.conf | 6 ++-- scripts/setup.py | 8 +++-- .../amo/management/commands/initialize.py | 8 +++++ src/olympia/amo/monitors.py | 11 ++++--- src/olympia/amo/tests/test_monitor.py | 2 +- storage/.gitignore | 0 tests/make/make.spec.js | 33 +++++++++---------- 11 files changed, 48 insertions(+), 42 deletions(-) create mode 100644 storage/.gitignore diff --git a/.github/actions/run-docker/action.yml b/.github/actions/run-docker/action.yml index fd8a318e5d5a..5a826110115a 100644 --- a/.github/actions/run-docker/action.yml +++ b/.github/actions/run-docker/action.yml @@ -27,11 +27,6 @@ inputs: runs: using: 'composite' steps: - - id: id - shell: bash - run: | - echo "id=$(id -u)" >> $GITHUB_OUTPUT - - name: Run Docker Container shell: bash run: | @@ -39,13 +34,9 @@ runs: make up \ DOCKER_VERSION="${{ inputs.version }}" \ DOCKER_DIGEST="${{ inputs.digest }}" \ - OLYMPIA_UID="${{ steps.id.outputs.id }}" \ + OLYMPIA_UID="$(id -u)" \ OLYMPIA_MOUNT="${{ inputs.mount }}" \ DATA_BACKUP_SKIP="${{ inputs.data_backup_skip }}" \ - # In CI, we should use the docker-compose wait flag to ensure \ - # healthchecks are passing before running any commands on the containers. \ - # This comes at a performance cost, but ensures containers are ready \ - # to accept commands before CI continues to execute. \ DOCKER_WAIT="true" diff --git a/.gitignore b/.gitignore index 1e5270636248..d185775e0294 100644 --- a/.gitignore +++ b/.gitignore @@ -47,7 +47,7 @@ src/olympia/discovery/strings.jinja2 static-build/* static/css/node_lib/* static/js/node_lib/* -storage +storage/* tmp/* # End of .gitignore. Please keep this in sync with the top section of .dockerignore @@ -56,3 +56,4 @@ tmp/* !docker-compose.ci.yml !docker-compose.private.yml !private/README.md +!storage/.gitignore diff --git a/Makefile-docker b/Makefile-docker index 00d3cee0ba71..61dab0fafb4f 100644 --- a/Makefile-docker +++ b/Makefile-docker @@ -51,6 +51,8 @@ check_django: ## check if the django app is configured properly .PHONY: check_nginx check_nginx: ## check if the nginx config for local development is configured properly + id -u + ls -lan /data/olympia/storage mkdir -p /data/olympia/storage/shared_storage/uploads echo "OK" > /data/olympia/storage/shared_storage/uploads/.check @if [ "$$(curl -sf http://nginx/user-media/.check)" != "OK" ]; then echo "Requesting http://nginx/user-media/.check failed"; exit 1; fi diff --git a/docker-compose.yml b/docker-compose.yml index 5be7b659778d..ee179cb77199 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,7 +32,7 @@ x-olympia-mount: &olympia-mount # production: (data_olympia)_storage:/data/olympia/storage # development: (./)_storage:/data/olympia/storage x-storage-mount: &storage-mount - ${HOST_MOUNT_SOURCE:?}_storage:/data/olympia/storage + ${HOST_MOUNT_SOURCE:?}storage:/data/olympia/storage x-site-static-mount: &site-static-mount data_site_static:/data/olympia/site-static @@ -153,7 +153,7 @@ services: # Disable performance schema for faster startup - --performance-schema=OFF healthcheck: - test: ["CMD-SHELL", "mysql -u root --silent --execute='SELECT 1;'"] + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "--silent"] start_interval: 1s timeout: 2s start_period: 10s @@ -229,7 +229,7 @@ volumes: # then we use the production volume mounts. Otherwise # it will map to the current directory ./ # (data_olympia):/ - data_olympia: + data_olympia_: data_olympia_storage: # Volume for rabbitmq/redis to avoid anonymous volumes data_rabbitmq: diff --git a/docker/nginx/addons.conf b/docker/nginx/addons.conf index 97eb1678f1d0..a6694e9bcdd7 100644 --- a/docker/nginx/addons.conf +++ b/docker/nginx/addons.conf @@ -7,11 +7,11 @@ server { location /data/olympia/storage/ { internal; - alias /srv/storage/; + alias /data/olympia/storage/; } location /static/ { - alias /srv/static/; + alias /data/olympia/static/; # Fallback to the uwsgi server if the file is not found in the static files directory. # This will happen for vendor files from pytnon or npm dependencies that won't be available @@ -20,7 +20,7 @@ server { } location /user-media/ { - alias /srv/storage/shared_storage/uploads/; + alias /data/olympia/storage/shared_storage/uploads/; } location ~ ^/api/ { diff --git a/scripts/setup.py b/scripts/setup.py index f7756ddf50b3..a57e41d12199 100755 --- a/scripts/setup.py +++ b/scripts/setup.py @@ -65,18 +65,22 @@ def get_docker_tag(): return tag, version, digest + def get_olympia_mount(docker_target): """ When running on production targets, the user can specify the olympia mount to one of the valid values: development or production. In development, we hard code the values to ensure we have necessary files and permissions. """ - olympia_mount = os.environ.get('OLYMPIA_MOUNT', docker_target) + olympia_mount = os.environ.get( + 'OLYMPIA_MOUNT', + get_value('OLYMPIA_MOUNT', docker_target), + ) olympia_mount_source = './' if docker_target == 'production' and olympia_mount == 'production': # This is the prefix of the volume name. - olympia_mount_source = 'data_olympia' + olympia_mount_source = 'data_olympia_' return olympia_mount, olympia_mount_source diff --git a/src/olympia/amo/management/commands/initialize.py b/src/olympia/amo/management/commands/initialize.py index 0f4bf276538d..f956b5a3b628 100644 --- a/src/olympia/amo/management/commands/initialize.py +++ b/src/olympia/amo/management/commands/initialize.py @@ -13,6 +13,11 @@ class Command(BaseDataCommand): Ensures the database has the correct state. """ + # We don't want to run system checks here, because this command + # can run before everything is ready. + # we run them at the end of the command. + requires_system_checks = [] + help = 'Creates, seeds, and indexes the database.' def add_arguments(self, parser): @@ -74,3 +79,6 @@ def handle(self, *args, **options): services=['localdev_web', 'celery_worker', 'elastic', 'rabbitmq', 'signer'], attempts=10, ) + + # Finally, run the django checks to ensure everything is ok. + call_command('check') diff --git a/src/olympia/amo/monitors.py b/src/olympia/amo/monitors.py index dfa9f2bbbcb2..f4036d6954bc 100644 --- a/src/olympia/amo/monitors.py +++ b/src/olympia/amo/monitors.py @@ -39,11 +39,14 @@ def localdev_web(): and some json via the uwsgi http server. """ status = '' - response = requests.get('http://127.0.0.1:8002/__version__') - - if response.status_code != 200: - status = f'Failed to ping web with status code: {response.status_code}' + try: + response = requests.get('http://127.0.0.1:8002/__version__') + response.raise_for_status() + except Exception as e: + status = f'Failed to ping web: {e}' monitor_log.critical(status) + return status, None + return status, None diff --git a/src/olympia/amo/tests/test_monitor.py b/src/olympia/amo/tests/test_monitor.py index 853d91c43700..510d55185bfd 100644 --- a/src/olympia/amo/tests/test_monitor.py +++ b/src/olympia/amo/tests/test_monitor.py @@ -154,7 +154,7 @@ def test_localdev_web_fail(self): status=500, ) status, _ = monitors.localdev_web() - assert status == 'Failed to ping web with status code: 500' + assert 'Failed to ping web' in status def test_localdev_web_success(self): responses.add( diff --git a/storage/.gitignore b/storage/.gitignore new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/make/make.spec.js b/tests/make/make.spec.js index 87fd52e9f279..47c996cfec60 100644 --- a/tests/make/make.spec.js +++ b/tests/make/make.spec.js @@ -101,9 +101,7 @@ describe('docker-compose.yml', () => { expect(service.volumes).toEqual( expect.arrayContaining([ expect.objectContaining({ - source: isProdMountTarget - ? 'data_olympia' - : expect.any(String), + source: isProdMountTarget ? 'data_olympia_' : expect.any(String), target: '/data/olympia', }), expect.objectContaining({ @@ -114,9 +112,10 @@ describe('docker-compose.yml', () => { }), ]), ); + const { OLYMPIA_MOUNT, ...environmentOutput } = inputValues; expect(service.environment).toEqual( expect.objectContaining({ - ...inputValues, + ...environmentOutput, }), ); // We excpect not to pass the input values to the container @@ -157,20 +156,18 @@ describe('docker-compose.yml', () => { source: 'data_nginx', target: '/etc/nginx/conf.d', }), - // // mapping for local host directory to /data/olympia - // expect.objectContaining({ - // source: isProdMountTarget - // ? 'data_olymmpia' - // : expect.any(String), - // target: '/data/olympia', - // }), - // // mapping for local host directory to /data/olympia/storage - // expect.objectContaining({ - // source: isProdMountTarget - // ? 'data_olympia_storage' - // : expect.any(String), - // target: '/data/olympia/storage', - // }), + // mapping for local host directory to /data/olympia + expect.objectContaining({ + source: isProdMountTarget ? 'data_olympia_' : expect.any(String), + target: '/data/olympia', + }), + // mapping for local host directory to /data/olympia/storage + expect.objectContaining({ + source: isProdMountTarget + ? 'data_olympia_storage' + : expect.any(String), + target: '/data/olympia/storage', + }), ]), ); });