Merge pull request #1044 from lsst-sqre/tickets/DM-44567

DM-44567: Update dependencies, adjust for new tox-docker

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.5
+    rev: v0.4.6
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

changelog.d/20240528_150942_rra_DM_44567.md

@@ -0,0 +1,3 @@
+### Other changes
+
+- Honor the `POSTGRES_5432_TCP_PORT`, `POSTGRES_HOST`, `REDIS_6379_TCP_PORT`, and `REDIS_HOST` environment variables if they are set and override the configured database URL and Redis URL with them. This is required to work with the latest version of tox-docker for testing and development. These environment variables are not used inside a Phalanx deployment.

docs/dev/release.rst

@@ -21,8 +21,8 @@ See :ref:`backport-release` to patch an earlier major-minor version.
 
 Release tags are semantic version identifiers following the :pep:`440` specification.
 
-1. Change log and documentation
--------------------------------
+1. Change log and dependencies
+------------------------------
 
 Change log messages for each release are accumulated using scriv_ (see :ref:`dev-change-log`).
 When it comes time to make the release, there should be a collection of change log fragments in :file:`changelog.d`.
@@ -39,6 +39,8 @@ Then, run ``scriv collect --version <version>`` specifying the version number yo
 This will delete the fragment files and collect them into :file:`CHANGELOG.md` under an entry for the new release.
 Review that entry and edit it as needed (proofread, change the order to put more important things first, etc.).
 
+Update dependencies by running :command:`make update-deps`.
+
 Finally, create a PR from those changes and merge it before continuing with the release process.
 
 2. Tag the release

requirements/main.in

@@ -35,8 +35,3 @@ structlog
 # Uncomment this, change the branch, comment out safir above, and run make
 # update-deps-no-hashes to test against an unreleased version of Safir.
 # safir[db,kubernetes] @ git+https://github.com/lsst-sqre/safir@main
-
-# docker 6.1.3 doesn't work with requests 2.32.0. This only affects
-# tox-docker, but all of the dependency files have to be consistent and
-# there is a transitive dependency on requests.
-requests<2.32.0

requirements/tox.in

@@ -12,6 +12,3 @@
 tox
 tox-docker
 tox-uv
-
-# docker 6.1.3 doesn't work with requests 2.31.0.
-requests<2.32.0

requirements/tox.txt

@@ -120,9 +120,9 @@ distlib==0.3.8 \
     --hash=sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784 \
     --hash=sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64
     # via virtualenv
-docker==6.1.3 \
-    --hash=sha256:aa6d17830045ba5ef0168d5eaa34d37beeb113948c413affe1d5991fc11f9a20 \
-    --hash=sha256:aecd2277b8bf8e506e484f6ab7aec39abe0038e29fa4a6d3ba86c3fe01844ed9
+docker==7.1.0 \
+    --hash=sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c \
+    --hash=sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0
     # via tox-docker
 filelock==3.14.0 \
     --hash=sha256:43339835842f110ca7ae60f1e1c160714c5a6afd15a2873419ab185334975c0f \
@@ -142,10 +142,8 @@ packaging==24.0 \
     --hash=sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9
     # via
     #   -c requirements/dev.txt
-    #   docker
     #   pyproject-api
     #   tox
-    #   tox-docker
     #   tox-uv
 platformdirs==4.2.2 \
     --hash=sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee \
@@ -164,13 +162,12 @@ pyproject-api==1.6.1 \
     --hash=sha256:1817dc018adc0d1ff9ca1ed8c60e1623d5aaca40814b953af14a9cf9a5cae538 \
     --hash=sha256:4c0116d60476b0786c88692cf4e325a9814965e2469c5998b830bba16b183675
     # via tox
-requests==2.31.0 \
-    --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \
-    --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1
+requests==2.32.2 \
+    --hash=sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289 \
+    --hash=sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c
     # via
     #   -c requirements/dev.txt
     #   -c requirements/main.txt
-    #   -r requirements/tox.in
     #   docker
 tox==4.15.0 \
     --hash=sha256:300055f335d855b2ab1b12c5802de7f62a36d4fd53f30bd2835f6a201dda46ea \
@@ -179,9 +176,9 @@ tox==4.15.0 \
     #   -r requirements/tox.in
     #   tox-docker
     #   tox-uv
-tox-docker==4.1.0 \
-    --hash=sha256:0317e692dc80f2197eaf9c905dcb8d1d1f9d5bf2686ecfd83c22a1da9d23fb24 \
-    --hash=sha256:444c72192a2443d2b4db5766545d4413ea683cc488523d770e2e216f15fa3086
+tox-docker==5.0.0 \
+    --hash=sha256:3cbfe70713910b9dbb9f74e20a5c42f19da42d375bac7d6a3ccc11aed0738d7a \
+    --hash=sha256:7dd7293ea286dff422eab6c23a70652147b275eb436c02e97a90ac59ba41f8a8
     # via -r requirements/tox.in
 tox-uv==1.8.2 \
     --hash=sha256:264b829d0e2d117dec27c3b83d90f83ac674f7b52a367d5449fe5073e4508c23 \
@@ -195,30 +192,26 @@ urllib3==2.2.1 \
     #   -c requirements/main.txt
     #   docker
     #   requests
-uv==0.2.3 \
-    --hash=sha256:12c25976c0ab62a9d0333afcaf80033dc04695907932a227b5950c9bf716ddee \
-    --hash=sha256:23e3321c9e4d006e6454ae3375c3e6ec71ba0cc9c73394bb4777f1ae8f2ceb2d \
-    --hash=sha256:304fbcd9272c7e81f62a46b9254daf68f7622e38b2a1e450f27f3a7cc9d70fb8 \
-    --hash=sha256:3227894999ae465b4f3b88b60a94500f85f9b479e39b7de1bd1c32db19245252 \
-    --hash=sha256:3b9b077561704f8d0733dc8ea46c084b6113be7618d88b2fadbe111f3986dac1 \
-    --hash=sha256:4a2c04f146c351ef534052472794c68ebd528239c622764c1fe762a6ae8e6e1f \
-    --hash=sha256:5867b54fad22eb8d9052d7c69192844b551fafa5ce6d951d5fd467059cb35b3d \
-    --hash=sha256:6e57a7f92462cd06f0f43112cebc98406eb1a1640345040108c8d7de5b9556ad \
-    --hash=sha256:6ef658ea907963e497c2d6a0a5502eb13796b97b843571d4acd3c8e1f6507d4a \
-    --hash=sha256:7116eac4eb0e7329ce74d95cb7fada29f0623a39ca34357c374877bb00b098f0 \
-    --hash=sha256:79a92bb307d0b62f859a5705880b1433e552db3631405f19bc8862638a54c71f \
-    --hash=sha256:8081eb30b5cde7661eade4d87c80e8876a3b23635e0823e727d98d1b0ce4d883 \
-    --hash=sha256:8108cf952d84bc25062de6402ff5db0485aedeedb2bd87f5e764534ae9ce07e4 \
-    --hash=sha256:882f3b55d298045d7f7b3073f54aa87aafbad10bae4e1a0fdd14807b15a918ca \
-    --hash=sha256:d87ebe10c030ede3598ce560f35a54fe1e14515018fb222707576e5c80d0b6fb \
-    --hash=sha256:de1bc4defcbce6077475f11d21437ac9469b0b641814690cdacf11d36d38aa2c \
-    --hash=sha256:f24700aa936814a682b49f033e64c4e61f360407c4176a4476fcc8ff82868b15
+uv==0.2.5 \
+    --hash=sha256:041b1ee240fc36606dcea2a68369c7db29eb0ea75dc15d33f74ed6485a7d3aa2 \
+    --hash=sha256:14f5ca36f8873d272a715250b3d602815924742a5f184e1400826fb967fd371e \
+    --hash=sha256:1e0b790a2367c8ebdd1a3eb2aca87ae393c5536021fbc37f97c36e5454cb2875 \
+    --hash=sha256:1ec01d0505e2d56cebad02b7565db5b1e61ad97c4eeca751c4816bfe8261ea28 \
+    --hash=sha256:541c8ee1369232c7be8bde2165175b08ddfa1cae6d0d31ffca9993fa83b64634 \
+    --hash=sha256:5b2f56c0219c9ee37da702d824ae3bffd7d023184824e6fdaacac8b435624784 \
+    --hash=sha256:5ffb593256d69623071c248348e6fe101aae806944da68e3b597c377444e022f \
+    --hash=sha256:650f81439c4f65e86fdba84f2cbb6700a074b95d70c5cc3d38e8ce2b87e43a45 \
+    --hash=sha256:6b977b3d83b7bb48ccc0ef568f1be73f45ff0bc53ff9ba1e33b4c883027feec9 \
+    --hash=sha256:6cedb8bfabd59ab56e781297dc6c66a298cfa912da16a4da9439274681c39622 \
+    --hash=sha256:7a4e793419993967183633afb2cfd9c4a7c4c5134f01af8c8aa25bf9520ff1db \
+    --hash=sha256:7fb8c35694de30186140adddc8498222c8728b00edde25ddc7c93c3dee69644c \
+    --hash=sha256:a55c5b1bf948d439a40f4d959be853f7441923c6b324e4d423116fda2cb277da \
+    --hash=sha256:b64372e90c6e921efbcca82113bc6a8f86a5a6ba8695f8c1d2dd215b7989b2a3 \
+    --hash=sha256:d7f24e33253e0b60faf4dd76849f0e0e710f30229f205eaaa94660b8626d6636 \
+    --hash=sha256:d879d0c436ad9289ff1ad39112a0b53bf5b99ec69d3d300e55d18864b4ed3291 \
+    --hash=sha256:ff1c8fe7d54dd3051bab529f715e562994d19e038822ce99acd903cbbf8d0cd0
     # via tox-uv
 virtualenv==20.26.2 \
     --hash=sha256:82bf0f4eebbb78d36ddaee0283d43fe5736b53880b8a8cdcd37390a07ac3741c \
     --hash=sha256:a624db5e94f01ad993d476b9ee5346fdf7b9de43ccaee0e0197012dc838a0e9b
     # via tox
-websocket-client==1.8.0 \
-    --hash=sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526 \
-    --hash=sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da
-    # via docker

src/gafaelfawr/config.py

@@ -19,12 +19,14 @@
 
 from __future__ import annotations
 
+import os
 import re
 from collections import defaultdict
 from datetime import timedelta
 from ipaddress import IPv4Network, IPv6Network
 from pathlib import Path
 from typing import Any, Self
+from urllib.parse import urlparse, urlunparse
 from uuid import UUID
 
 import yaml
@@ -965,6 +967,38 @@ def _validate_database_url(cls, v: PostgresDsnString) -> PostgresDsnString:
         if not v.startswith(("postgresql:", "postgresql+asyncpg:")):
             msg = "Use asyncpg as the PostgreSQL library or leave unspecified"
             raise ValueError(msg)
+
+        # When run via tox and tox-docker, the PostgreSQL hostname and port
+        # will be randomly selected and exposed only in environment
+        # variables. We have to patch that into the database URL at runtime
+        # since tox doesn't have a way of substituting it into the environment
+        # (see https://github.com/tox-dev/tox-docker/issues/55).
+        if port := os.getenv("POSTGRES_5432_TCP_PORT"):
+            url = urlparse(v)
+            hostname = os.getenv("POSTGRES_HOST", url.hostname)
+            if url.password:
+                auth = f"{url.username}@{url.password}@"
+            elif url.username:
+                auth = f"{url.username}@"
+            else:
+                auth = ""
+            return urlunparse(url._replace(netloc=f"{auth}{hostname}:{port}"))
+
+        return v
+
+    @field_validator("redis_url")
+    @classmethod
+    def _validate_redis_url(cls, v: RedisDsnString) -> RedisDsnString:
+        # When run via tox and tox-docker, the Redis port will be randomly
+        # selected and exposed only in the REDIS_6379_TCP environment
+        # variable. We have to patch that into the Redis URL at runtime since
+        # tox doesn't have a way of substituting it into the environment (see
+        # https://github.com/tox-dev/tox-docker/issues/55).
+        if port := os.getenv("REDIS_6379_TCP_PORT"):
+            url = urlparse(v)
+            hostname = os.getenv("REDIS_HOST", url.hostname)
+            return urlunparse(url._replace(netloc=f"{hostname}:{port}"))
+
         return v
 
     @field_validator("initial_admins")

tests/conftest.py

@@ -35,7 +35,7 @@
 
 from .pages.tokens import TokensPage
 from .support.config import config_path, configure
-from .support.constants import TEST_DATABASE_URL, TEST_HOSTNAME
+from .support.constants import TEST_HOSTNAME
 from .support.database import clear_alembic_version
 from .support.firestore import MockFirestore, patch_firestore
 from .support.ldap import MockLDAP, patch_ldap
@@ -158,7 +158,7 @@ def set_version(connection: Connection) -> None:
 
 
 @pytest_asyncio.fixture
-def engine() -> AsyncEngine:
+def engine(config: Config) -> AsyncEngine:
     """Create a database engine for testing.
 
     Previously, this fixture was session-scoped so that all tests could share
@@ -168,7 +168,9 @@ def engine() -> AsyncEngine:
     pytest-asyncio was upgraded from 0.21.1 to 0.23.2 and the maintenance
     burden doesn't seem worth it.
     """
-    return create_database_engine(TEST_DATABASE_URL, None)
+    return create_database_engine(
+        config.database_url, config.database_password.get_secret_value()
+    )
 
 
 @pytest_asyncio.fixture

tests/main_test.py

@@ -11,13 +11,14 @@
 from gafaelfawr.exceptions import DatabaseSchemaError
 from gafaelfawr.main import create_app
 
-from .support.constants import TEST_DATABASE_URL
 from .support.database import create_old_database, drop_database
 
 
 @pytest.mark.asyncio
 async def test_out_of_date_schema(config: Config) -> None:
-    engine = create_database_engine(TEST_DATABASE_URL, None)
+    engine = create_database_engine(
+        config.database_url, config.database_password.get_secret_value()
+    )
     await drop_database(engine)
     await create_old_database(config, engine, "9.6.1")
 

tests/support/constants.py

@@ -2,14 +2,6 @@
 
 from gafaelfawr.keypair import RSAKeyPair
 
-TEST_DATABASE_URL = (
-    "postgresql://gafaelfawr:[email protected]/gafaelfawr"
-)
-"""The URL used for the test database.
-
-This must match the ``tox.ini`` configuration for the PostgreSQL container.
-"""
-
 TEST_HOSTNAME = "example.com"
 """The hostname used in ASGI requests to the application."""
 

tox.ini

@@ -4,13 +4,10 @@ isolated_build = True
 
 [docker:postgres]
 image = postgres:latest
-ports =
-    5432:5432/tcp
 environment =
     POSTGRES_PASSWORD=INSECURE-PASSWORD
     POSTGRES_USER=gafaelfawr
     POSTGRES_DB=gafaelfawr
-    PGPORT=5432
 # The healthcheck ensures that tox-docker won't run tests until the
 # container is up and the command finishes with exit code 0 (success)
 healthcheck_cmd = PGPASSWORD=$POSTGRES_PASSWORD psql \
@@ -26,8 +23,6 @@ volumes =
 
 [docker:redis]
 image = redis:latest
-ports =
-    6379:6379/tcp
 healthcheck_cmd =
     redis-cli ping
 healthcheck_timeout = 1
@@ -42,9 +37,9 @@ deps =
     -r{toxinidir}/requirements/dev.txt
 setenv =
     GAFAELFAWR_ALEMBIC_CONFIG_PATH = {toxinidir}/alembic.ini
-    GAFAELFAWR_DATABASE_URL = postgresql://gafaelfawr@127.0.0.1/gafaelfawr
+    GAFAELFAWR_DATABASE_URL = postgresql://gafaelfawr@localhost/gafaelfawr
     GAFAELFAWR_DATABASE_PASSWORD = INSECURE-PASSWORD
-    GAFAELFAWR_REDIS_URL = redis://localhost:6379/0
+    GAFAELFAWR_REDIS_URL = redis://localhost:{env:REDIS_6379_TCP}/0
     GAFAELFAWR_UI_PATH = {toxinidir}/ui/public
 
 [testenv:alembic]
@@ -55,7 +50,7 @@ setenv =
     GAFAELFAWR_CONFIG_PATH = {toxinidir}/alembic/gafaelfawr.yaml
     GAFAELFAWR_DATABASE_URL = postgresql://gafaelfawr@localhost/gafaelfawr
     GAFAELFAWR_DATABASE_PASSWORD = INSECURE
-    GAFAELFAWR_REDIS_URL = redis://localhost:6379/0
+    GAFAELFAWR_REDIS_URL = redis://localhost/0
 
 [testenv:coverage-report]
 description = Compile coverage from each test run.
@@ -91,7 +86,7 @@ setenv =
     GAFAELFAWR_CONFIG_PATH = {toxinidir}/alembic/gafaelfawr.yaml
     GAFAELFAWR_DATABASE_URL = postgresql://gafaelfawr@localhost/gafaelfawr
     GAFAELFAWR_DATABASE_PASSWORD = INSECURE
-    GAFAELFAWR_REDIS_URL = redis://localhost:6379/0
+    GAFAELFAWR_REDIS_URL = redis://localhost/0
     GAFAELFAWR_REDIS_PASSWORD = TOTALLY-INSECURE-test-password
 
 [testenv:lint]
@@ -126,9 +121,9 @@ commands =
     pytest -vv --cov=gafaelfawr --cov-branch --cov-report= {posargs}
 setenv =
     GAFAELFAWR_ALEMBIC_CONFIG_PATH = {toxinidir}/alembic.ini
-    GAFAELFAWR_DATABASE_URL = postgresql://gafaelfawr@127.0.0.1/gafaelfawr
+    GAFAELFAWR_DATABASE_URL = postgresql://gafaelfawr@localhost/gafaelfawr
     GAFAELFAWR_DATABASE_PASSWORD = INSECURE-PASSWORD
-    GAFAELFAWR_REDIS_URL = redis://localhost:6379/0
+    GAFAELFAWR_REDIS_URL = redis://localhost/0
     GAFAELFAWR_UI_PATH = {toxinidir}/ui/public
     TEST_KUBERNETES = 1