From f8c95041a7d014ce8ae5063c880b2300c7e5fa92 Mon Sep 17 00:00:00 2001 From: Kevin Meinhardt Date: Sun, 22 Dec 2024 14:53:29 +0100 Subject: [PATCH] Revert "Remove django_node_assets" This reverts commit ede7c2e3e4d3584eb4baa5d02da5d5d70fbe41b4. --- .dockerignore | 3 ++ Dockerfile | 3 ++ Makefile-docker | 1 + docker-compose.yml | 9 ++++- docker/nginx/addons.conf | 16 +++++---- package.json | 9 ++++- requirements/prod.txt | 6 ++++ settings.py | 6 ++++ src/olympia/core/apps.py | 24 +++++++++++++ .../devhub/templates/devhub/index.html | 4 +++ src/olympia/lib/settings_base.py | 22 +++++++++++- static/css/index.css | 3 ++ static/css/index.css.js | 1 + static/js/blue.css | 3 ++ static/js/blue.js | 3 ++ static/js/common/index.js | 15 ++++++++ static/js/index.js | 1 + vite.config.js | 35 +++++++++++++++++++ 18 files changed, 155 insertions(+), 9 deletions(-) create mode 100644 static/css/index.css create mode 100644 static/css/index.css.js create mode 100644 static/js/blue.css create mode 100644 static/js/blue.js create mode 100644 static/js/common/index.js create mode 100644 static/js/index.js create mode 100644 vite.config.js diff --git a/.dockerignore b/.dockerignore index e7b5b4c3d5aa..7e02b72b9ff7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -61,3 +61,6 @@ docker-bake.hcl docker-compose*.yml Dockerfile* Makefile-os + +# Ignore all .git directories in any subdirectory +.git/ diff --git a/Dockerfile b/Dockerfile index 3f2a1b8942c6..f135aaa4bcd5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -175,6 +175,9 @@ RUN \ --mount=type=bind,src=src,target=${HOME}/src \ --mount=type=bind,src=Makefile-docker,target=${HOME}/Makefile-docker \ --mount=type=bind,src=manage.py,target=${HOME}/manage.py \ + --mount=type=bind,src=package.json,target=${HOME}/package.json \ + --mount=type=bind,src=package-lock.json,target=${HOME}/package-lock.json \ + --mount=type=bind,src=vite.config.js,target=${HOME}/vite.config.js \ < settings_local.py DJANGO_SETTINGS_MODULE="settings_local" make -f Makefile-docker update_assets diff --git a/Makefile-docker b/Makefile-docker index c9c6ad3e93b9..f49be53e86b7 100644 --- a/Makefile-docker +++ b/Makefile-docker @@ -71,6 +71,7 @@ data_load: .PHONY: update_assets update_assets: + npm $(NPM_ARGS) run build # Copy files required in compress_assets to the static folder # If changing this here, make sure to adapt tests in amo/test_commands.py $(PYTHON_COMMAND) manage.py compress_assets diff --git a/docker-compose.yml b/docker-compose.yml index 4e847cacc989..e1f71ab458c8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,6 +49,14 @@ services: - ${HOST_MOUNT_SOURCE:?}deps:/deps - data_site_static:/data/olympia/site-static - ${HOST_MOUNT_SOURCE:?}storage:/data/olympia/storage + + static: + <<: *olympia + volumes: + - ${HOST_MOUNT_SOURCE:?}:/data/olympia + - ${HOST_MOUNT_SOURCE:?}deps:/deps + command: npm run dev -d + worker: <<: *olympia command: [ @@ -103,7 +111,6 @@ services: image: nginx volumes: - data_nginx:/etc/nginx/conf.d - - ${HOST_MOUNT_SOURCE:?}:/srv - data_site_static:/srv/site-static - ${HOST_MOUNT_SOURCE:?}storage:/srv/storage ports: diff --git a/docker/nginx/addons.conf b/docker/nginx/addons.conf index 97eb1678f1d0..cbc7e9f047ff 100644 --- a/docker/nginx/addons.conf +++ b/docker/nginx/addons.conf @@ -10,13 +10,10 @@ server { alias /srv/storage/; } + # Try 3 locations in order: site-static, vite assets (@static), and olympia (@olympia) + # TODO: this is not resolving correctly location /static/ { - alias /srv/static/; - - # Fallback to the uwsgi server if the file is not found in the static files directory. - # This will happen for vendor files from pytnon or npm dependencies that won't be available - # in the static files directory. - error_page 404 = @olympia; + try_files $uri @static @olympia; } location /user-media/ { @@ -39,6 +36,11 @@ server { try_files $uri @frontendamo; } + location @static { + proxy_pass http://static:5173; + add_header X-Served-By "static" always; + } + location @olympia { uwsgi_pass web; include uwsgi_params; @@ -50,6 +52,8 @@ server { uwsgi_param X-Real-IP $remote_addr; uwsgi_param X-Forwarded-For $proxy_add_x_forwarded_for; uwsgi_param X-Forwarded-Protocol ssl; + + add_header X-Served-By "olympia" always; } location @frontendamo { diff --git a/package.json b/package.json index b42457f7fffe..4189eebbd12a 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,12 @@ "engines": { "node": ">= 18.18" }, + "type": "module", + "scripts": { + "dev": "vite -d", + "build": "vite build", + "preview": "vite preview" + }, "dependencies": { "@claviska/jquery-minicolors": "2.3.6", "addons-linter": "7.6.0", @@ -31,6 +37,7 @@ "jest-environment-jsdom": "^29.7.0", "lodash": "^4.17.21", "prettier": "^3.3.3", - "terser": "^5.36.0" + "terser": "^5.36.0", + "vite": "^6.0.3" } } diff --git a/requirements/prod.txt b/requirements/prod.txt index 61e88074f950..14e6c65dfc43 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -1205,3 +1205,9 @@ watchdog[watchmedo]==3.0.0 \ --hash=sha256:c9d8c8ec7efb887333cf71e328e39cffbf771d8f8f95d308ea4125bf5f90ba64 \ --hash=sha256:d00e6be486affb5781468457b21a6cbe848c33ef43f9ea4a73b4882e5f188a44 \ --hash=sha256:d429c2430c93b7903914e4db9a966c7f2b068dd2ebdd2fa9b9ce094c7d459f33 +django-vite==3.0.5 \ + --hash=sha256:049b74f38c999cbfcf0e2c21b254c2e059bb97bfd7e4049caf2d0f9fba0b482f \ + --hash=sha256:431c1212e7627adc20666d150578f1a8983f043e90f3905778fb3c5c0ffe6963 +django-node-assets==0.9.14 \ + --hash=sha256:80cbe3d10521808309712b2aa5ef6d69799bbcafef844cf7f223d3c93f405768 \ + --hash=sha256:d5b5c472136084d533268f52ab77897327863a102e25c81f484aae85eb806987 diff --git a/settings.py b/settings.py index ad6b13112338..dd5d36e8deeb 100644 --- a/settings.py +++ b/settings.py @@ -9,6 +9,7 @@ import os from urllib.parse import urlparse +from olympia.core.utils import get_version_json from olympia.lib.settings_base import * # noqa @@ -198,3 +199,8 @@ def insert_debug_toolbar_middleware(middlewares): } ENABLE_ADMIN_MLBF_UPLOAD = True + +print('DJANGO_VITE banana', get_version_json().get('target')) + +# Override the dev mode from the base settings depending on if we are in a production image +DJANGO_VITE["default"]["dev_mode"] = get_version_json().get('target') != 'production' diff --git a/src/olympia/core/apps.py b/src/olympia/core/apps.py index 879b30868248..0f24c23e4410 100644 --- a/src/olympia/core/apps.py +++ b/src/olympia/core/apps.py @@ -1,3 +1,4 @@ +import json import logging import os import subprocess @@ -120,6 +121,29 @@ def static_check(app_configs, **kwargs): ) ) + if not os.path.exists(settings.STATIC_BUILD_MANIFEST_PATH): + errors.append( + Error( + f'Static build manifest file does not exist: {settings.STATIC_BUILD_MANIFEST_PATH}', + id='setup.E003', + ) + ) + else: + with open(settings.DJANGO_VITE['default']['manifest_path'], 'r') as f: + manifest = json.load(f) + + for name, asset in manifest.items(): + # Assets compiled by vite are in the static root directory + # after running collectstatic. So we should look there. + path = os.path.join(settings.STATIC_ROOT, asset['file']) + if not os.path.exists(path): + errors.append( + Error( + f'Static asset {name} does not exist at expected path: {path}', + id='setup.E003', + ) + ) + return errors diff --git a/src/olympia/devhub/templates/devhub/index.html b/src/olympia/devhub/templates/devhub/index.html index 115dd23750f6..327b6b6e8dfa 100644 --- a/src/olympia/devhub/templates/devhub/index.html +++ b/src/olympia/devhub/templates/devhub/index.html @@ -3,6 +3,10 @@ + + {{ vite_hmr_client() }} + # TODO: cannot resolve the asset in prod mode + {{ vite_asset('js/blue.js') }} {% if not settings.ENGAGE_ROBOTS %} diff --git a/src/olympia/lib/settings_base.py b/src/olympia/lib/settings_base.py index b06b5ced1319..09673c47061b 100644 --- a/src/olympia/lib/settings_base.py +++ b/src/olympia/lib/settings_base.py @@ -376,6 +376,10 @@ def get_db_config(environ_var, atomic_requests=True): path('src/olympia/templates'), ), 'OPTIONS': { + 'globals': { + 'vite_hmr_client': 'django_vite.templatetags.django_vite.vite_hmr_client', + 'vite_asset': 'django_vite.templatetags.django_vite.vite_asset', + }, # http://jinja.pocoo.org/docs/dev/extensions/#newstyle-gettext 'newstyle_gettext': True, # Match our regular .html and .txt file endings except @@ -562,6 +566,8 @@ def get_db_config(environ_var, atomic_requests=True): 'rangefilter', 'django_recaptcha', 'drf_yasg', + 'django_vite', + 'django_node_assets', # Django contrib apps 'django.contrib.admin', 'django.contrib.auth', @@ -1324,13 +1330,17 @@ def read_only_mode(env): STATICFILES_FINDERS = ( 'django.contrib.staticfiles.finders.FileSystemFinder', 'django.contrib.staticfiles.finders.AppDirectoriesFinder', + 'django_node_assets.finders.NodeModulesFinder', ) NODE_MODULES_ROOT = path('node_modules') NODE_PACKAGE_JSON = path('package.json') NODE_PACKAGE_MANAGER_INSTALL_OPTIONS = ['--dry-run'] -STATIC_BUILD_PATH = os.path.join('/', 'data', 'olympia', 'static-build') +STATIC_BUILD_PATH = path('static-build') +# The manifest file is created in static-build but copied into the static root +# so we should expect to find it at //manifest.json +STATIC_BUILD_MANIFEST_PATH = path(STATIC_BUILD_PATH, 'manifest.json') STATICFILES_DIRS = ( path('static'), @@ -1617,3 +1627,13 @@ def read_only_mode(env): TESTING_ENV = False ENABLE_ADMIN_MLBF_UPLOAD = False + +# TODO: we need to make this work for production environments as well. +DJANGO_VITE = { + "default": { + # Always use prod mode in the base settings. We can override this in settings.py + "dev_mode": False, + # "static_url_prefix": "", + "manifest_path": STATIC_BUILD_MANIFEST_PATH, + } +} diff --git a/static/css/index.css b/static/css/index.css new file mode 100644 index 000000000000..d8ccc9d491b6 --- /dev/null +++ b/static/css/index.css @@ -0,0 +1,3 @@ +* { + color: red; +} diff --git a/static/css/index.css.js b/static/css/index.css.js new file mode 100644 index 000000000000..6a9a4b132858 --- /dev/null +++ b/static/css/index.css.js @@ -0,0 +1 @@ +import './index.css'; diff --git a/static/js/blue.css b/static/js/blue.css new file mode 100644 index 000000000000..9309367c7ad4 --- /dev/null +++ b/static/js/blue.css @@ -0,0 +1,3 @@ +* { + color: blue !important; +} diff --git a/static/js/blue.js b/static/js/blue.js new file mode 100644 index 000000000000..39889292b8c1 --- /dev/null +++ b/static/js/blue.js @@ -0,0 +1,3 @@ +import './blue.css'; + +alert('blue'); diff --git a/static/js/common/index.js b/static/js/common/index.js new file mode 100644 index 000000000000..d69bfcfd92c2 --- /dev/null +++ b/static/js/common/index.js @@ -0,0 +1,15 @@ +import 'underscore/underscore.js'; +import '../zamboni/init.js'; +import '../zamboni/capabilities.js'; +import '../lib/format.js'; +import 'jquery.cookie/jquery.cookie.js'; +import '../zamboni/storage.js'; +import '../common/keys.js'; +import '../zamboni/helpers.js'; +import '../zamboni/global.js'; +import '../zamboni/l10n.js'; +// Unicode letters for our makeslug function +import '../zamboni/unicode.js'; +// Login tweaks +import '../zamboni/users.js'; +import '../common/lang_switcher.js'; diff --git a/static/js/index.js b/static/js/index.js new file mode 100644 index 000000000000..3451e9b7ed77 --- /dev/null +++ b/static/js/index.js @@ -0,0 +1 @@ +console.log('Hello World'); diff --git a/vite.config.js b/vite.config.js new file mode 100644 index 000000000000..d0c5855e6e68 --- /dev/null +++ b/vite.config.js @@ -0,0 +1,35 @@ +import { defineConfig } from 'vite'; +import { resolve, join } from 'path'; + +// TODO: vite is clearing the cwd on every build. that is wrong and annoying. +// also it's having trouble matching the manifest.json here and the one in the settings_base.py file. +export default defineConfig((_) => { + + const INPUT_DIR = './static'; + const OUTPUT_DIR = './static-build'; + + return { + root: resolve(INPUT_DIR), + base: '/static/', + server: { + host: true, + port: 5173, + }, + build: { + manifest: 'manifest.json', + emptyOutDir: false, + copyPublicDir: false, + outDir: resolve(OUTPUT_DIR), + rollupOptions: { + input: { + 'common': join(INPUT_DIR, 'js/common/index.js'), + 'blue_js': join(INPUT_DIR, 'js/blue.js'), + }, + }, + }, + }; +}); + +/* +{"Timestamp": 1734876156142280960, "Type": "django.request", "Logger": "http_app_addons", "Hostname": "4bd613fee1b8", "EnvVersion": "2.0", "Severity": 3, "Pid": 35, "Fields": {"status_code": 500, "request": "", "uid": "", "remoteAddressChain": "", "msg": "Internal Server Error: /en-US/developers/", "error": "DjangoViteAssetNotFoundError('Cannot find js/blue.js for app=default in Vite manifest at /data/olympia/src/olympia/../../static-build/manifest.json')", "traceback": "Uncaught exception:\n File \"/deps/lib/python3.12/site-packages/django/core/handlers/exception.py\", line 55, in inner\n response = get_response(request)\n ^^^^^^^^^^^^^^^^^^^^^\n File \"/deps/lib/python3.12/site-packages/django/core/handlers/base.py\", line 220, in _get_response\n response = response.render()\n ^^^^^^^^^^^^^^^^^\n File \"/deps/lib/python3.12/site-packages/sentry_sdk/integrations/django/views.py\", line 38, in sentry_patched_render\n return old_render(self)\n ^^^^^^^^^^^^^^^^\n File \"/deps/lib/python3.12/site-packages/django/template/response.py\", line 114, in render\n self.content = self.rendered_content\n ^^^^^^^^^^^^^^^^^^^^^\n File \"/deps/lib/python3.12/site-packages/sentry_sdk/integrations/django/templates.py\", line 75, in rendered_content\n return real_rendered_content.fget(self)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/deps/lib/python3.12/site-packages/django/template/response.py\", line 92, in rendered_content\n return template.render(context, self._request)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/deps/lib/python3.12/site-packages/django_jinja/backend.py\", line 59, in render\n return mark_safe(self._process_template(self.template.render, context, request))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/deps/lib/python3.12/site-packages/django_jinja/backend.py\", line 105, in _process_template\n return handler(context)\n ^^^^^^^^^^^^^^^^\n File \"/deps/lib/python3.12/site-packages/jinja2/environment.py\", line 1304, in render\n self.environment.handle_exception()\n File \"/deps/lib/python3.12/site-packages/jinja2/environment.py\", line 939, in handle_exception\n raise rewrite_traceback_stack(source=source)\n File \"/data/olympia/src/olympia/devhub/templates/devhub/index.html\", line 8, in top-level template code\n {{ vite_asset('js/blue.js') }}\n ^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/deps/lib/python3.12/site-packages/django/utils/safestring.py\", line 53, in wrapper\n return safety_marker(func(*args, **kwargs))\n ^^^^^^^^^^^^^^^^^^^^^\n File \"/deps/lib/python3.12/site-packages/django_vite/templatetags/django_vite.py\", line 67, in vite_asset\n return DjangoViteAssetLoader.instance().generate_vite_asset(path, app, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/deps/lib/python3.12/site-packages/django_vite/core/asset_loader.py\", line 802, in generate_vite_asset\n return app_client.generate_vite_asset(path, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/deps/lib/python3.12/site-packages/django_vite/core/asset_loader.py\", line 312, in generate_vite_asset\n manifest_entry = self.manifest.get(path)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/deps/lib/python3.12/site-packages/django_vite/core/asset_loader.py\", line 201, in get\n raise DjangoViteAssetNotFoundError(\n\nDjangoViteAssetNotFoundError('Cannot find js/blue.js for app=default in Vite manifest at /data/olympia/src/olympia/../../static-build/manifest.json')\n"}, "severity": 500} +*/