Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ci: warm-up docker images #2099

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ docs = [
]
test = [
"build",
"filelock",
"jinja2",
"pytest-timeout",
"pytest-xdist",
Expand Down
81 changes: 81 additions & 0 deletions test/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,10 @@
from typing import Generator

import pytest
from filelock import FileLock

from cibuildwheel.architecture import Architecture
from cibuildwheel.options import CommandLineArguments, Options
from cibuildwheel.util import detect_ci_provider, find_uv

from .utils import EMULATED_ARCHS, platform
Expand All @@ -28,6 +31,84 @@ def pytest_addoption(parser: pytest.Parser) -> None:
)


def docker_warmup(request: pytest.FixtureRequest) -> None:
machine = request.config.getoption("--run-emulation", default=None)
if machine is None:
archs = {arch.value for arch in Architecture.auto_archs("linux")}
elif machine == "all":
archs = set(EMULATED_ARCHS)
else:
archs = {machine}

# Only include architectures where there are missing pre-installed interpreters
archs &= {"x86_64", "i686", "aarch64"}
if not archs:
return

options = Options(
platform="linux",
command_line_arguments=CommandLineArguments.defaults(),
env={},
defaults=True,
)
build_options = options.build_options(None)
assert build_options.manylinux_images is not None
assert build_options.musllinux_images is not None
images = [build_options.manylinux_images[arch] for arch in archs] + [
build_options.musllinux_images[arch] for arch in archs
]
# exclude GraalPy as it's not a target for cibuildwheel
command = (
"manylinux-interpreters ensure $(manylinux-interpreters list 2>/dev/null | grep -v graalpy) &&"
"cpython3.13 -m pip download -d /tmp setuptools wheel pytest"
)
for image in images:
container_id = subprocess.run(
["docker", "create", image, "bash", "-c", command],
text=True,
check=True,
stdout=subprocess.PIPE,
).stdout.strip()
try:
subprocess.run(["docker", "start", container_id], check=True, stdout=subprocess.DEVNULL)
exit_code = subprocess.run(
["docker", "wait", container_id], text=True, check=True, stdout=subprocess.PIPE
).stdout.strip()
assert exit_code == "0"
subprocess.run(
["docker", "commit", container_id, image], check=True, stdout=subprocess.DEVNULL
)
finally:
subprocess.run(["docker", "rm", container_id], check=True, stdout=subprocess.DEVNULL)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Might be polite to run this in a finally: block

Copy link
Member Author

@mayeut mayeut Nov 30, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done in the commit that enforce the warm-up does not fail
As it's only done in CI, being polite is not required but when considering failure now, it's easy enough to change this (would require nesting 2 try/except/finally in the commit before that)



@pytest.fixture(scope="session", autouse=True)
def docker_warmup_fixture(
request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory, worker_id: str
) -> None:
# if we're in CI testing linux, let's warm-up docker images
if detect_ci_provider() is None or platform != "linux":
return None
if request.config.getoption("--run-emulation", default=None) is not None:
# emulation tests only run one test in CI, caching the image only slows down the test
return None

if worker_id == "master":
# not executing with multiple workers
# it might be unsafe to write to tmp_path_factory.getbasetemp().parent
return docker_warmup(request)

# get the temp directory shared by all workers
root_tmp_dir = tmp_path_factory.getbasetemp().parent

fn = root_tmp_dir / "warmup.done"
with FileLock(str(fn) + ".lock"):
if not fn.is_file():
docker_warmup(request)
fn.write_text("done")
return None


@pytest.fixture(params=["pip", "build"])
def build_frontend_env_nouv(request: pytest.FixtureRequest) -> dict[str, str]:
frontend = request.param
Expand Down
Loading