From 673d763ccef70230801c94675206b1d80118626f Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Wed, 17 Jul 2024 18:18:38 -0400 Subject: [PATCH 01/31] Revert "Remove Hatch" This reverts commit 962b238f6afde16ce552b9c631ca81e2c9ded7f6. --- src/builder/app/__init__.py | 3 ++ src/builder/app/build_app.py | 56 ++++++++++++++++++++++++++++++++ src/builder/app/hooks_app.py | 11 +++++++ src/builder/app/pyproject.toml | 22 +++++++++++++ src/builder/zip/__init__.py | 3 ++ src/builder/zip/build_zip.py | 59 ++++++++++++++++++++++++++++++++++ src/builder/zip/hooks_zip.py | 11 +++++++ src/builder/zip/pyproject.toml | 22 +++++++++++++ 8 files changed, 187 insertions(+) create mode 100644 src/builder/app/__init__.py create mode 100644 src/builder/app/build_app.py create mode 100644 src/builder/app/hooks_app.py create mode 100644 src/builder/app/pyproject.toml create mode 100644 src/builder/zip/__init__.py create mode 100644 src/builder/zip/build_zip.py create mode 100644 src/builder/zip/hooks_zip.py create mode 100644 src/builder/zip/pyproject.toml diff --git a/src/builder/app/__init__.py b/src/builder/app/__init__.py new file mode 100644 index 0000000..4662c4f --- /dev/null +++ b/src/builder/app/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers +# +# SPDX-License-Identifier: LGPL-3.0-only diff --git a/src/builder/app/build_app.py b/src/builder/app/build_app.py new file mode 100644 index 0000000..22675ed --- /dev/null +++ b/src/builder/app/build_app.py @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers +# +# SPDX-License-Identifier: LGPL-3.0-only + +import os +from io import BytesIO +from pathlib import Path +from typing import Any, Callable, Iterable +from zipapp import create_archive + +from hatchling.builders.plugin.interface import BuilderInterface + + +class ZipAppBuilder(BuilderInterface): + PLUGIN_NAME = "zipapp" + + def get_version_api(self) -> dict[str, Callable[..., str]]: + return {"standard": self.build_standard} + + def clean(self, directory: str, versions: Iterable[str]) -> None: + try: + os.remove(Path(directory, 'jlmkr')) + except: + pass + + def build_standard(self, directory: str, **build_data: Any) -> str: + + # generate zipapp source archive + pyzbuffer = BytesIO() + create_archive('src/jlmkr', target=pyzbuffer, + interpreter='=PLACEHOLDER=', +# main='donor.jlmkr:main', + compressed=True) + zipdata = pyzbuffer.getvalue() #.removeprefix(b"#!=PLACEHOLDER=\n") + + # output with preamble + outpath = Path(directory, 'jlmkr') + with open(outpath, 'wb') as f: + f.write(preamble(self.metadata.version).encode()) + f.write(zipdata) + os.chmod(outpath, 0o755) + return os.fspath(outpath) + + +# 10 lines will conveniently match the default of head(1) +def preamble(version): return f'''#!/usr/bin/env python3 + +jlmkr {version} + +Persistent Linux 'jails' on TrueNAS SCALE to install software (k3s, docker, portainer, podman, etc.) with full access to all files via bind mounts. + +SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers +SPDX-License-Identifier: LGPL-3.0-only + +-=-=-=- this is a zip file -=-=-=- what follows is binary -=-=-=- +''' diff --git a/src/builder/app/hooks_app.py b/src/builder/app/hooks_app.py new file mode 100644 index 0000000..beb4253 --- /dev/null +++ b/src/builder/app/hooks_app.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers +# +# SPDX-License-Identifier: LGPL-3.0-only + +from hatchling.plugin import hookimpl + +from build_app import ZipAppBuilder + +@hookimpl +def hatch_register_builder(): + return ZipAppBuilder diff --git a/src/builder/app/pyproject.toml b/src/builder/app/pyproject.toml new file mode 100644 index 0000000..8af91bc --- /dev/null +++ b/src/builder/app/pyproject.toml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers +# +# SPDX-License-Identifier: LGPL-3.0-only + +[build-system] +build-backend = "hatchling.build" +requires = ["hatchling"] + +[project] +name = "hatch-zipapp" +version = "0.0.dev0" +classifiers = [ + "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)", + "Private :: Do Not Upload", +] +dependencies = ["hatchling"] + +[project.entry-points.hatch] +zipapp = "hooks_app" + +[tool.hatch.build.targets.wheel] +packages = ["."] diff --git a/src/builder/zip/__init__.py b/src/builder/zip/__init__.py new file mode 100644 index 0000000..4662c4f --- /dev/null +++ b/src/builder/zip/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers +# +# SPDX-License-Identifier: LGPL-3.0-only diff --git a/src/builder/zip/build_zip.py b/src/builder/zip/build_zip.py new file mode 100644 index 0000000..53caf1f --- /dev/null +++ b/src/builder/zip/build_zip.py @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers +# +# SPDX-License-Identifier: LGPL-3.0-only + +# hat tip: + +import os +from hatchling.builders.config import BuilderConfig +from hatchling.builders.plugin.interface import BuilderInterface +from hatchling.builders.utils import normalize_relative_path +from pathlib import Path +from typing import Any, Callable, Iterable +from zipfile import ZipFile, ZIP_DEFLATED + + +class AppZipBuilderConfig(BuilderConfig): pass + +class AppZipBuilder(BuilderInterface): + PLUGIN_NAME = "appzip" + + @classmethod + def get_config_class(cls): + return AppZipBuilderConfig + + def get_version_api(self) -> dict[str, Callable[..., str]]: + return {'standard': self.build_standard} + + def clean(self, directory: str, versions: Iterable[str]) -> None: + for filename in os.listdir(directory): + if filename.startswith('jlmkr-') and filename.endswith('.zip'): + os.remove(Path(directory, filename)) + + def build_standard(self, directory: str, **build_data: Any) -> str: + outpath = Path(directory, f'jlmkr-{self.metadata.version}.zip') + with ZipFile(outpath, 'w') as zip: + zip.write(Path(directory, 'jlmkr'), 'jlmkr') + force_map = build_data['force_include'] + for included_file in self.recurse_forced_files(force_map): + zip.write( + included_file.relative_path, + included_file.distribution_path, + ZIP_DEFLATED) + return os.fspath(outpath) + + def get_default_build_data(self) -> dict[str, Any]: + build_data: dict[str, Any] = super().get_default_build_data() + + extra_files = [] + if self.metadata.core.readme_path: + extra_files.append(self.metadata.core.readme_path) + if self.metadata.core.license_files: + extra_files.extend(self.metadata.core.license_files) + + force_include = build_data.setdefault("force_include", {}) + for fn in map(normalize_relative_path, extra_files): + force_include[os.path.join(self.root, fn)] = Path(fn).name + build_data['force_include'] = force_include + + return build_data diff --git a/src/builder/zip/hooks_zip.py b/src/builder/zip/hooks_zip.py new file mode 100644 index 0000000..5bd8a60 --- /dev/null +++ b/src/builder/zip/hooks_zip.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers +# +# SPDX-License-Identifier: LGPL-3.0-only + +from hatchling.plugin import hookimpl + +from build_zip import AppZipBuilder + +@hookimpl +def hatch_register_builder(): + return AppZipBuilder diff --git a/src/builder/zip/pyproject.toml b/src/builder/zip/pyproject.toml new file mode 100644 index 0000000..1aa6674 --- /dev/null +++ b/src/builder/zip/pyproject.toml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers +# +# SPDX-License-Identifier: LGPL-3.0-only + +[build-system] +build-backend = "hatchling.build" +requires = ["hatchling"] + +[project] +name = "hatch-appzip" +version = "0.0.dev0" +classifiers = [ + "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)", + "Private :: Do Not Upload", +] +dependencies = ["hatchling"] + +[project.entry-points.hatch] +appzip = "hooks_zip" + +[tool.hatch.build.targets.wheel] +packages = ["."] From 350992bc26d83861e096feb573c96cd65e3b2bd1 Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Wed, 17 Jul 2024 23:18:16 -0400 Subject: [PATCH 02/31] Substitute third-party general build script plugin --- .gitignore | 5 +- build.py | 68 ++++++++++++++++++++++++ pyproject.toml | 96 ++++++++++++++++++++++++++++++++++ src/builder/app/__init__.py | 3 -- src/builder/app/build_app.py | 56 -------------------- src/builder/app/hooks_app.py | 11 ---- src/builder/app/pyproject.toml | 22 -------- src/builder/zip/__init__.py | 3 -- src/builder/zip/build_zip.py | 59 --------------------- src/builder/zip/hooks_zip.py | 11 ---- src/builder/zip/pyproject.toml | 22 -------- src/jlmkr/__main__.py | 6 +-- 12 files changed, 169 insertions(+), 193 deletions(-) create mode 100644 build.py create mode 100644 pyproject.toml delete mode 100644 src/builder/app/__init__.py delete mode 100644 src/builder/app/build_app.py delete mode 100644 src/builder/app/hooks_app.py delete mode 100644 src/builder/app/pyproject.toml delete mode 100644 src/builder/zip/__init__.py delete mode 100644 src/builder/zip/build_zip.py delete mode 100644 src/builder/zip/hooks_zip.py delete mode 100644 src/builder/zip/pyproject.toml diff --git a/.gitignore b/.gitignore index e3d38ff..1cf3e4d 100644 --- a/.gitignore +++ b/.gitignore @@ -20,5 +20,6 @@ dist/ __pycache__/ *.py[cod] -.pytest_cache/ -.ruff_cache/ +/.pytest_cache/ +/.ruff_cache/ +/.coverage diff --git a/build.py b/build.py new file mode 100644 index 0000000..9b68ac3 --- /dev/null +++ b/build.py @@ -0,0 +1,68 @@ +# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers +# +# SPDX-License-Identifier: LGPL-3.0-only + +# hat tip: + +import os +from io import BytesIO +from pathlib import Path +from typing import Any, Callable, Iterable +from zipapp import create_archive +from zipfile import ZipFile, ZIP_DEFLATED + +from src.jlmkr.__main__ import __version__ as VERSION +PROJECT_PATH = Path.cwd() +SRC_PATH = Path('./src/jlmkr') +DIST_PATH = Path('./dist') + +# 10 lines will conveniently match the default of head(1) +PREAMBLE = f'''#!/usr/bin/env python3 + +jlmkr {VERSION} + +Persistent Linux 'jails' on TrueNAS SCALE to install software (k3s, docker, portainer, podman, etc.) with full access to all files via bind mounts. + +SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers +SPDX-License-Identifier: LGPL-3.0-only + +-=-=-=- this is a zip file -=-=-=- what follows is binary -=-=-=- +''' + + +def build_tool() -> Path: + + # generate zipapp source archive + pyzbuffer = BytesIO() + create_archive(SRC_PATH, target=pyzbuffer, + interpreter='=PLACEHOLDER=', + compressed=True) + zipdata = pyzbuffer.getvalue().removeprefix(b"#!=PLACEHOLDER=\n") + + # output with preamble + tool_path = DIST_PATH.joinpath('jlmkr') + with open(tool_path, 'wb') as f: + f.write(PREAMBLE.encode()) + f.write(zipdata) + os.chmod(tool_path, 0o755) + return tool_path + + +def build_zip() -> Path: + zip_path = DIST_PATH.joinpath(f'jlmkr-{VERSION}.zip') + tool_path = DIST_PATH.joinpath('jlmkr') + attachments = ['README.md', 'LICENSE'] + + # include the tool as-is (with the uncompressed preamble), + # then compress and attach other hangers-on + with ZipFile(zip_path, 'w') as zip: + zip.write(tool_path) + for attachment in attachments: + path = PROJECT_PATH.joinpath(attachment) + zip.write(path, compress_type=ZIP_DEFLATED) + return zip_path + + +if __name__ == '__main__': + print(build_tool()) + print(build_zip()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..00bcb6c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,96 @@ +# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers +# +# SPDX-License-Identifier: LGPL-3.0-only + +[project] +name = "jlmkr" +description = "Build and manage jails on TrueNAS SCALE" +dynamic = ["version"] +readme = "README.md" +classifiers = [ + "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)", + "Private :: Do Not Upload", +] +authors = [ + {name = "Jip-Hop"}, + {name = "Lockszmith-GH"}, + {name = "jonct"}, +] +maintainers = [ + {name = "Jip-Hop"}, +] +requires-python = ">=3.8" + +[project.urls] +GitHub = "https://github.com/Jip-Hop/jailmaker" + +[build-system] +requires = ["hatchling", "hatch-build-scripts"] +build-backend = "hatchling.build" + +[tool.hatch.version] +path = "src/jlmkr/__main__.py" # or source = "vcs" + +[project.scripts] +jlmkr = "jlmkr:__main__" + + +#### BUILD + + +[[tool.hatch.build.hooks.build-scripts.scripts]] +out_dir = "dist" +commands = ['python ./build.py'] +artifacts = [ + 'dist/jlmkr', + 'dist/jlmkr-*.zip' +] +clean_artifacts = true +clean_out_dir = true + + +#### TEST + + +[[tool.hatch.envs.hatch-test.matrix]] +python = [ + "3.11", # TrueNAS SCALE 24.04 Dragonfish +] + +[tool.pytest.ini_options] +pythonpath = "src" +addopts = [ + "--import-mode=importlib", +] + + +#### ANALYZE + + +[tool.hatch.envs.types] +extra-dependencies = [ + "mypy>=1.0.0", +] + +[tool.hatch.envs.types.scripts] +check = "mypy --install-types --non-interactive {args:src/jlmkr tests}" + + +#### COVERAGE + + +[tool.coverage.run] +source_pkgs = ["jlmkr", "tests"] +branch = true +parallel = true + +[tool.coverage.paths] +jlmkr = ["src/jlmkr"] +tests = ["tests"] + +[tool.coverage.report] +exclude_lines = [ + "no cov", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] diff --git a/src/builder/app/__init__.py b/src/builder/app/__init__.py deleted file mode 100644 index 4662c4f..0000000 --- a/src/builder/app/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers -# -# SPDX-License-Identifier: LGPL-3.0-only diff --git a/src/builder/app/build_app.py b/src/builder/app/build_app.py deleted file mode 100644 index 22675ed..0000000 --- a/src/builder/app/build_app.py +++ /dev/null @@ -1,56 +0,0 @@ -# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers -# -# SPDX-License-Identifier: LGPL-3.0-only - -import os -from io import BytesIO -from pathlib import Path -from typing import Any, Callable, Iterable -from zipapp import create_archive - -from hatchling.builders.plugin.interface import BuilderInterface - - -class ZipAppBuilder(BuilderInterface): - PLUGIN_NAME = "zipapp" - - def get_version_api(self) -> dict[str, Callable[..., str]]: - return {"standard": self.build_standard} - - def clean(self, directory: str, versions: Iterable[str]) -> None: - try: - os.remove(Path(directory, 'jlmkr')) - except: - pass - - def build_standard(self, directory: str, **build_data: Any) -> str: - - # generate zipapp source archive - pyzbuffer = BytesIO() - create_archive('src/jlmkr', target=pyzbuffer, - interpreter='=PLACEHOLDER=', -# main='donor.jlmkr:main', - compressed=True) - zipdata = pyzbuffer.getvalue() #.removeprefix(b"#!=PLACEHOLDER=\n") - - # output with preamble - outpath = Path(directory, 'jlmkr') - with open(outpath, 'wb') as f: - f.write(preamble(self.metadata.version).encode()) - f.write(zipdata) - os.chmod(outpath, 0o755) - return os.fspath(outpath) - - -# 10 lines will conveniently match the default of head(1) -def preamble(version): return f'''#!/usr/bin/env python3 - -jlmkr {version} - -Persistent Linux 'jails' on TrueNAS SCALE to install software (k3s, docker, portainer, podman, etc.) with full access to all files via bind mounts. - -SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers -SPDX-License-Identifier: LGPL-3.0-only - --=-=-=- this is a zip file -=-=-=- what follows is binary -=-=-=- -''' diff --git a/src/builder/app/hooks_app.py b/src/builder/app/hooks_app.py deleted file mode 100644 index beb4253..0000000 --- a/src/builder/app/hooks_app.py +++ /dev/null @@ -1,11 +0,0 @@ -# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers -# -# SPDX-License-Identifier: LGPL-3.0-only - -from hatchling.plugin import hookimpl - -from build_app import ZipAppBuilder - -@hookimpl -def hatch_register_builder(): - return ZipAppBuilder diff --git a/src/builder/app/pyproject.toml b/src/builder/app/pyproject.toml deleted file mode 100644 index 8af91bc..0000000 --- a/src/builder/app/pyproject.toml +++ /dev/null @@ -1,22 +0,0 @@ -# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers -# -# SPDX-License-Identifier: LGPL-3.0-only - -[build-system] -build-backend = "hatchling.build" -requires = ["hatchling"] - -[project] -name = "hatch-zipapp" -version = "0.0.dev0" -classifiers = [ - "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)", - "Private :: Do Not Upload", -] -dependencies = ["hatchling"] - -[project.entry-points.hatch] -zipapp = "hooks_app" - -[tool.hatch.build.targets.wheel] -packages = ["."] diff --git a/src/builder/zip/__init__.py b/src/builder/zip/__init__.py deleted file mode 100644 index 4662c4f..0000000 --- a/src/builder/zip/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers -# -# SPDX-License-Identifier: LGPL-3.0-only diff --git a/src/builder/zip/build_zip.py b/src/builder/zip/build_zip.py deleted file mode 100644 index 53caf1f..0000000 --- a/src/builder/zip/build_zip.py +++ /dev/null @@ -1,59 +0,0 @@ -# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers -# -# SPDX-License-Identifier: LGPL-3.0-only - -# hat tip: - -import os -from hatchling.builders.config import BuilderConfig -from hatchling.builders.plugin.interface import BuilderInterface -from hatchling.builders.utils import normalize_relative_path -from pathlib import Path -from typing import Any, Callable, Iterable -from zipfile import ZipFile, ZIP_DEFLATED - - -class AppZipBuilderConfig(BuilderConfig): pass - -class AppZipBuilder(BuilderInterface): - PLUGIN_NAME = "appzip" - - @classmethod - def get_config_class(cls): - return AppZipBuilderConfig - - def get_version_api(self) -> dict[str, Callable[..., str]]: - return {'standard': self.build_standard} - - def clean(self, directory: str, versions: Iterable[str]) -> None: - for filename in os.listdir(directory): - if filename.startswith('jlmkr-') and filename.endswith('.zip'): - os.remove(Path(directory, filename)) - - def build_standard(self, directory: str, **build_data: Any) -> str: - outpath = Path(directory, f'jlmkr-{self.metadata.version}.zip') - with ZipFile(outpath, 'w') as zip: - zip.write(Path(directory, 'jlmkr'), 'jlmkr') - force_map = build_data['force_include'] - for included_file in self.recurse_forced_files(force_map): - zip.write( - included_file.relative_path, - included_file.distribution_path, - ZIP_DEFLATED) - return os.fspath(outpath) - - def get_default_build_data(self) -> dict[str, Any]: - build_data: dict[str, Any] = super().get_default_build_data() - - extra_files = [] - if self.metadata.core.readme_path: - extra_files.append(self.metadata.core.readme_path) - if self.metadata.core.license_files: - extra_files.extend(self.metadata.core.license_files) - - force_include = build_data.setdefault("force_include", {}) - for fn in map(normalize_relative_path, extra_files): - force_include[os.path.join(self.root, fn)] = Path(fn).name - build_data['force_include'] = force_include - - return build_data diff --git a/src/builder/zip/hooks_zip.py b/src/builder/zip/hooks_zip.py deleted file mode 100644 index 5bd8a60..0000000 --- a/src/builder/zip/hooks_zip.py +++ /dev/null @@ -1,11 +0,0 @@ -# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers -# -# SPDX-License-Identifier: LGPL-3.0-only - -from hatchling.plugin import hookimpl - -from build_zip import AppZipBuilder - -@hookimpl -def hatch_register_builder(): - return AppZipBuilder diff --git a/src/builder/zip/pyproject.toml b/src/builder/zip/pyproject.toml deleted file mode 100644 index 1aa6674..0000000 --- a/src/builder/zip/pyproject.toml +++ /dev/null @@ -1,22 +0,0 @@ -# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers -# -# SPDX-License-Identifier: LGPL-3.0-only - -[build-system] -build-backend = "hatchling.build" -requires = ["hatchling"] - -[project] -name = "hatch-appzip" -version = "0.0.dev0" -classifiers = [ - "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)", - "Private :: Do Not Upload", -] -dependencies = ["hatchling"] - -[project.entry-points.hatch] -appzip = "hooks_zip" - -[tool.hatch.build.targets.wheel] -packages = ["."] diff --git a/src/jlmkr/__main__.py b/src/jlmkr/__main__.py index 2b263d5..429c21e 100644 --- a/src/jlmkr/__main__.py +++ b/src/jlmkr/__main__.py @@ -15,11 +15,9 @@ IT COMES WITHOUT WARRANTY AND IS NOT SUPPORTED BY IXSYSTEMS.""" -import sys - -from cli import main - if __name__ == "__main__": + import sys + from cli import main try: sys.exit(main()) except KeyboardInterrupt: From 65dd6e92cc67236e30dc3b2699bfde1b4fd6dcff Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Wed, 17 Jul 2024 23:19:17 -0400 Subject: [PATCH 03/31] Stand up some rudimentary unit tests --- tests/test_paths.py | 9 +++++++++ tests/test_utils_editor.py | 15 +++++++++++++++ tests/test_utils_files.py | 9 +++++++++ 3 files changed, 33 insertions(+) create mode 100644 tests/test_paths.py create mode 100644 tests/test_utils_editor.py create mode 100644 tests/test_utils_files.py diff --git a/tests/test_paths.py b/tests/test_paths.py new file mode 100644 index 0000000..4aaa2b4 --- /dev/null +++ b/tests/test_paths.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers +# +# SPDX-License-Identifier: LGPL-3.0-only + +from jlmkr import paths + + +def test_script_name(): + assert paths.SHORTNAME == 'jlmkr' diff --git a/tests/test_utils_editor.py b/tests/test_utils_editor.py new file mode 100644 index 0000000..2012bd9 --- /dev/null +++ b/tests/test_utils_editor.py @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers +# +# SPDX-License-Identifier: LGPL-3.0-only + +from jlmkr.utils import editor + +from pathlib import Path + + +def test_editor_executable(): + e = editor.get_text_editor() + path = Path(e).resolve() + print(path) + assert path.is_file() + assert path.stat().st_mode & 0o7777 == 0o755 diff --git a/tests/test_utils_files.py b/tests/test_utils_files.py new file mode 100644 index 0000000..0b043ad --- /dev/null +++ b/tests/test_utils_files.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers +# +# SPDX-License-Identifier: LGPL-3.0-only + +from jlmkr.utils.files import get_mount_point + + +def test_mount_point(): + assert get_mount_point('/usr/local/share/truenas/eula.html') == '/usr' From 48acc57c766eabe7cca13b06300a9ef746b14f8c Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Wed, 17 Jul 2024 23:19:57 -0400 Subject: [PATCH 04/31] See what GitHub Actions thinks of build-and-test --- .github/workflows/test.yml | 50 +++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 77ec144..5ab578d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,5 +1,3 @@ -# This is a basic workflow to help you get started with Actions - name: CI # Controls when the workflow will run @@ -13,9 +11,53 @@ on: # Allows you to run this workflow manually from the Actions tab workflow_dispatch: -# A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - # This workflow contains a single job called "build" + + ########################################################## + ## Build software for distribution; perform code-level QA + build: + name: Build jlmkr tool + runs-on: ubuntu-24.04 + strategy: + matrix: + python-version: + - "3.11" # TrueNAS SCALE 24.04 Dragonfish + steps: + # hat tip: + + - uses: actions/setup-python@v5 + with: + python-version-file: 'pyproject.toml' + cache: 'pip' + + - name: Install Hatch + run: pip install hatch + + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Build distribution + run: hatch build --ext + + - name: Run style check + run: hatch fmt --check + + - name: Run unit tests + run: hatch test --cover + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + path: + - dist/jlmkr + - dist/jlmkr-*.zip + if-no-files-found: error + + + ########################################################## + ## Perform rubber-meets-road integration testing test: # The type of runner that the job will run on runs-on: ubuntu-24.04 From e0155566c410fc5fa3047d12714258097e931de1 Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Wed, 17 Jul 2024 23:23:43 -0400 Subject: [PATCH 05/31] Curse at GitHub inconsistency --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5ab578d..856bb2e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,9 +50,9 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 with: - path: - - dist/jlmkr - - dist/jlmkr-*.zip + path: | + dist/jlmkr + dist/jlmkr-*.zip if-no-files-found: error From be99ce22765418874669383a11bd949a8c881dd6 Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Wed, 17 Jul 2024 23:37:19 -0400 Subject: [PATCH 06/31] Ask for it by name --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 00bcb6c..6f4855c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ authors = [ maintainers = [ {name = "Jip-Hop"}, ] -requires-python = ">=3.8" +requires-python = "3.11" [project.urls] GitHub = "https://github.com/Jip-Hop/jailmaker" From 0cfad8a5402f1037b0d85d2760bd97ac9bff3bc1 Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Wed, 17 Jul 2024 23:38:45 -0400 Subject: [PATCH 07/31] Be less dummy --- .github/workflows/test.yml | 10 +++++----- pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 856bb2e..e0d570a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,6 +25,11 @@ jobs: steps: # hat tip: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 with: python-version-file: 'pyproject.toml' @@ -33,11 +38,6 @@ jobs: - name: Install Hatch run: pip install hatch - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Build distribution run: hatch build --ext diff --git a/pyproject.toml b/pyproject.toml index 6f4855c..e8ae6bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ authors = [ maintainers = [ {name = "Jip-Hop"}, ] -requires-python = "3.11" +requires-python = ">=3.11" [project.urls] GitHub = "https://github.com/Jip-Hop/jailmaker" From ae1278e270b88d181280aa4b1d063b221eb6c274 Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Wed, 17 Jul 2024 23:42:59 -0400 Subject: [PATCH 08/31] Tolerate a little schlubbiness --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e0d570a..28cd405 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: run: hatch build --ext - name: Run style check - run: hatch fmt --check + run: hatch fmt --check || echo IGNORING ADVISORIES - name: Run unit tests run: hatch test --cover From 24011b42b344bdc643d2be548f3ea7380acf4a58 Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Wed, 17 Jul 2024 23:46:57 -0400 Subject: [PATCH 09/31] Disable TrueNAS-specific test for CI environment --- tests/test_utils_files.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_utils_files.py b/tests/test_utils_files.py index 0b043ad..62fcb82 100644 --- a/tests/test_utils_files.py +++ b/tests/test_utils_files.py @@ -6,4 +6,5 @@ def test_mount_point(): + return # oops good choice for TrueNAS; poor choice for GitHub Runner assert get_mount_point('/usr/local/share/truenas/eula.html') == '/usr' From 3bd3327fd9d5872c7b6401f7b555870b0bf69d78 Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Wed, 17 Jul 2024 23:50:41 -0400 Subject: [PATCH 10/31] Deconflict build artifact --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 28cd405..46a60ab 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,8 +51,8 @@ jobs: uses: actions/upload-artifact@v4 with: path: | - dist/jlmkr dist/jlmkr-*.zip + # dist/jlmkr conflicts with other job if-no-files-found: error From 26f025e2c4d4e0f440a6150d9bc6fec2b61e9f6a Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Wed, 17 Jul 2024 23:55:03 -0400 Subject: [PATCH 11/31] Switch loyalties --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 46a60ab..b94d9e5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,8 +51,8 @@ jobs: uses: actions/upload-artifact@v4 with: path: | + dist/jlmkr dist/jlmkr-*.zip - # dist/jlmkr conflicts with other job if-no-files-found: error @@ -133,9 +133,9 @@ jobs: run: | python3 -m zipapp src/jlmkr -p "/usr/bin/env python3" -o jlmkr - - uses: actions/upload-artifact@v4 - with: - path: jlmkr +# - uses: actions/upload-artifact@v4 +# with: +# path: jlmkr # Run multiple commands using the runners shell - name: Run the test script From 3cc95f3c0d8a61e92cf1251a93c251fb2fae9c89 Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Wed, 17 Jul 2024 23:59:56 -0400 Subject: [PATCH 12/31] Tidy up comments --- .github/workflows/test.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b94d9e5..4f4b67d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,7 +30,8 @@ jobs: with: fetch-depth: 0 - - uses: actions/setup-python@v5 + - name: Set up Python + uses: actions/setup-python@v5 with: python-version-file: 'pyproject.toml' cache: 'pip' @@ -133,10 +134,6 @@ jobs: run: | python3 -m zipapp src/jlmkr -p "/usr/bin/env python3" -o jlmkr -# - uses: actions/upload-artifact@v4 -# with: -# path: jlmkr - # Run multiple commands using the runners shell - name: Run the test script env: From acea4345f83a0f3b19d811f237fc40950ca5ff13 Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Thu, 8 Aug 2024 03:50:35 -0400 Subject: [PATCH 13/31] Take another run at edit/build/testing in place --- .github/workflows/test.yml | 3 +- DEVELOPER.md | 46 ++++++++++++++++++++++++ src/jlmkr/actions/create.py | 12 +++---- src/jlmkr/paths.py | 70 ++++++++++++++++++++++++++++++------- src/jlmkr/utils/dataset.py | 6 ++-- 5 files changed, 114 insertions(+), 23 deletions(-) create mode 100644 DEVELOPER.md diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4f4b67d..8cc4e46 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -131,8 +131,7 @@ jobs: # END - name: Build - run: | - python3 -m zipapp src/jlmkr -p "/usr/bin/env python3" -o jlmkr + run: python3 build.py && ln dist/jlmkr . # Run multiple commands using the runners shell - name: Run the test script diff --git a/DEVELOPER.md b/DEVELOPER.md new file mode 100644 index 0000000..2da7b21 --- /dev/null +++ b/DEVELOPER.md @@ -0,0 +1,46 @@ +# Developer notes + +## Building Jailmaker + +No external dependencies are needed to perform a simple build from the project directory. + + python3 build.py + +Anything beyond this is *completely optional…* + +## Development mode + +Jailmaker's user-facing design and important safety features tend to get in the way of rapid development. To run directly from the editable source code, create an external configuration file. + + mkdir -p ~/.local/share + cat <~/.local/share/jailmaker.conf + [DEFAULT] + ignore_owner = 1 + jailmaker_dir = /mnt/pool/jailmaker + EOF + +If present, this file will override traditional self-detection of the Jailmaker directory. + +## Code quality tooling + +Additional tools for testing, coverage, and code quality review are available through [Hatch][1]. Install them in a self-contained, disposable virtual environment. + + python3 -m venv --without-pip .venv + curl -OL https://bootstrap.pypa.io/pip/pip.pyz + .venv/bin/python3 pip.pyz install pip hatch + rm pip.pyz + +Activate a session inside the virtual environment. (For more information see the `venv` [tutorial][2].) + + source .venv/bin/activate + +Use `hatch` to build, test, lint, etc. + + hatch build + +## Integration testing + +See [`test/README.md`](./test/README.md). + +[1]: https://hatch.pypa.io/ +[2]: https://docs.python.org/3/tutorial/venv.html diff --git a/src/jlmkr/actions/create.py b/src/jlmkr/actions/create.py index 635bbda..fc72981 100644 --- a/src/jlmkr/actions/create.py +++ b/src/jlmkr/actions/create.py @@ -10,7 +10,7 @@ from textwrap import dedent from data import DISCLAIMER -from paths import COMMAND_NAME, JAILS_DIR_PATH, SCRIPT_DIR_PATH, SCRIPT_NAME +from paths import COMMAND_NAME, JAILS_DIR_PATH, JAILMAKER_DIR_PATH, SCRIPT_NAME from utils.chroot import Chroot from utils.config_parser import DEFAULT_CONFIG, KeyValueParser from utils.console import BOLD, NORMAL, YELLOW, eprint @@ -33,19 +33,19 @@ def create_jail(**kwargs): print(DISCLAIMER) - if os.path.basename(SCRIPT_DIR_PATH) != "jailmaker": + if os.path.basename(JAILMAKER_DIR_PATH) != "jailmaker": eprint( dedent( f""" {COMMAND_NAME} needs to create files. Currently it can not decide if it is safe to create files in: - {SCRIPT_DIR_PATH} + {JAILMAKER_DIR_PATH} Please create a dedicated dataset called "jailmaker", store {SCRIPT_NAME} there and try again.""" ) ) return 1 - if not PurePath(get_mount_point(SCRIPT_DIR_PATH)).is_relative_to("/mnt"): + if not PurePath(get_mount_point(JAILMAKER_DIR_PATH)).is_relative_to("/mnt"): print( dedent( f""" @@ -54,7 +54,7 @@ def create_jail(**kwargs): {SCRIPT_NAME} should be on a dataset mounted under /mnt (it currently is not). Storing it on the boot-pool means losing all jails when updating TrueNAS. Jails will be stored under: - {SCRIPT_DIR_PATH} + {JAILMAKER_DIR_PATH} """ ) ) @@ -118,7 +118,7 @@ def create_jail(**kwargs): try: # Create the dir or dataset where to store the jails if not os.path.exists(JAILS_DIR_PATH): - if get_zfs_dataset(SCRIPT_DIR_PATH): + if get_zfs_dataset(JAILMAKER_DIR_PATH): # Creating "jails" dataset if "jailmaker" is a ZFS Dataset create_zfs_dataset(JAILS_DIR_PATH) else: diff --git a/src/jlmkr/paths.py b/src/jlmkr/paths.py index fbe98ad..3fd938b 100755 --- a/src/jlmkr/paths.py +++ b/src/jlmkr/paths.py @@ -2,20 +2,66 @@ # # SPDX-License-Identifier: LGPL-3.0-only -import os.path +import os +import sys -# When running as a zipapp, the script file is a parent -ZIPAPP_PATH = os.path.realpath(__file__) -while not os.path.exists(ZIPAPP_PATH): - ZIPAPP_PATH = os.path.dirname(ZIPAPP_PATH) +from configparser import ConfigParser +from pathlib import Path +from utils.console import fail -SCRIPT_PATH = os.path.realpath(ZIPAPP_PATH) -SCRIPT_NAME = os.path.basename(SCRIPT_PATH) -SCRIPT_DIR_PATH = os.path.dirname(SCRIPT_PATH) -COMMAND_NAME = os.path.basename(ZIPAPP_PATH) -JAILS_DIR_PATH = os.path.join(SCRIPT_DIR_PATH, "jails") -JAIL_CONFIG_NAME = "config" -JAIL_ROOTFS_NAME = "rootfs" +def _get_selected_jailmaker_directory() -> Path: + ''' + Determine the user's affirmative choice of parent jailmaker directory + ''' + # first choice: global --dir/-D argument + #TODO + + # next: JAILMAKER_DIR environment variable + envname = 'JAILMAKER_DIR' + if envname in os.environ: + return Path(os.environ[envname]) + + # next: ~/.local/share/jailmaker.conf + secname = 'DEFAULT' + cfgname = 'jailmaker_dir' + username = '' + if os.getuid() == 0 and 'SUDO_USER' in os.environ: + username = os.environ['SUDO_USER'] + cfgpath = Path(f'~{username}/.local/share/jailmaker.conf').expanduser() + cfg = ConfigParser() + cfg.read(cfgpath) + if 'ignore_owner' in cfg[secname]: + os.environ['JLMKR_DEBUG'] = cfg[secname]['ignore_owner'] + if cfgname in cfg[secname]: + return Path(cfg[secname][cfgname]) + + # next: current directory iff it's named jailmaker + script = Path(sys.argv[0]).resolve(True) + if script.parent.name == 'jailmaker': + return script.parent + + fail("Please specify a jailmaker directory path (JAILMAKER_DIR)") + + +def get_tool_path_on_disk() -> Path: + ''' + Determine the script's location on disk + ''' + # When running as a zipapp, the script file is an ancestor + path = Path(__file__).resolve(strict=False) + while path and not path.is_file(): + path = path.parent + return path + +SCRIPT_PATH = get_tool_path_on_disk() +SCRIPT_NAME = SCRIPT_PATH.name +COMMAND_NAME = SCRIPT_NAME SHORTNAME = "jlmkr" + +JAILMAKER_DIR_PATH = _get_selected_jailmaker_directory() + +JAILS_DIR_PATH = JAILMAKER_DIR_PATH.joinpath("jails") +JAIL_CONFIG_NAME = "config" +JAIL_ROOTFS_NAME = "rootfs" diff --git a/src/jlmkr/utils/dataset.py b/src/jlmkr/utils/dataset.py index 5afee9c..8a50bef 100644 --- a/src/jlmkr/utils/dataset.py +++ b/src/jlmkr/utils/dataset.py @@ -6,13 +6,13 @@ import subprocess from pathlib import PurePath -from paths import SCRIPT_DIR_PATH +from paths import JAILMAKER_DIR_PATH from utils.console import eprint, fail def _get_relative_path_in_jailmaker_dir(absolute_path): - return PurePath(absolute_path).relative_to(SCRIPT_DIR_PATH) + return PurePath(absolute_path).relative_to(JAILMAKER_DIR_PATH) def get_zfs_dataset(path): @@ -37,7 +37,7 @@ def get_zfs_base_path(): """ Get ZFS dataset path for jailmaker directory. """ - zfs_base_path = get_zfs_dataset(SCRIPT_DIR_PATH) + zfs_base_path = get_zfs_dataset(JAILMAKER_DIR_PATH) if not zfs_base_path: fail("Failed to get dataset path for jailmaker directory.") From eae49def77bf02fd500de7d7cd77a8217af5b545 Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Thu, 8 Aug 2024 04:12:55 -0400 Subject: [PATCH 14/31] Insert the dot of destiny --- src/jlmkr/paths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jlmkr/paths.py b/src/jlmkr/paths.py index 3fd938b..07cea86 100755 --- a/src/jlmkr/paths.py +++ b/src/jlmkr/paths.py @@ -7,7 +7,7 @@ from configparser import ConfigParser from pathlib import Path -from utils.console import fail +from .utils.console import fail def _get_selected_jailmaker_directory() -> Path: From 721ab5df1200f2b940ea8ac4dfe9a1595cbf5688 Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Thu, 8 Aug 2024 04:24:36 -0400 Subject: [PATCH 15/31] Ensure dist directory during build --- build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build.py b/build.py index 9b68ac3..2827fcc 100644 --- a/build.py +++ b/build.py @@ -64,5 +64,6 @@ def build_zip() -> Path: if __name__ == '__main__': + DIST_PATH.mkdir(exist_ok=True) print(build_tool()) print(build_zip()) From 8d2d9a014a85a2233ca1254f2a8a505a69a07d31 Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Thu, 8 Aug 2024 20:28:19 -0400 Subject: [PATCH 16/31] Move project scripts into a subdirectory --- DEVELOPER.md | 2 +- pyproject.toml | 2 +- scripts/build.py | 72 ++++++++++++++++++++++++++++++++++++++++++++++ src/jlmkr/paths.py | 2 +- 4 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 scripts/build.py diff --git a/DEVELOPER.md b/DEVELOPER.md index 2da7b21..d9d6850 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -4,7 +4,7 @@ No external dependencies are needed to perform a simple build from the project directory. - python3 build.py + python3 -m scripts.build Anything beyond this is *completely optional…* diff --git a/pyproject.toml b/pyproject.toml index e8ae6bf..67a01cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ jlmkr = "jlmkr:__main__" [[tool.hatch.build.hooks.build-scripts.scripts]] out_dir = "dist" -commands = ['python ./build.py'] +commands = ['python -m scripts.build'] artifacts = [ 'dist/jlmkr', 'dist/jlmkr-*.zip' diff --git a/scripts/build.py b/scripts/build.py new file mode 100644 index 0000000..3a4b37d --- /dev/null +++ b/scripts/build.py @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers +# +# SPDX-License-Identifier: LGPL-3.0-only + +# hat tip: + +import importlib +import os +import sys +from io import BytesIO +from pathlib import Path +from typing import Any, Callable, Iterable +from zipapp import create_archive +from zipfile import ZipFile, ZIP_DEFLATED + +from src.jlmkr.__main__ import __version__ as VERSION + +PROJECT_PATH = Path.cwd() +SRC_PATH = Path('./src/jlmkr') +DIST_PATH = Path('./dist') + +# 10 lines will conveniently match the default of head(1) +PREAMBLE = f'''#!/usr/bin/env python3 + +jlmkr {VERSION} + +Persistent Linux 'jails' on TrueNAS SCALE to install software (k3s, docker, portainer, podman, etc.) with full access to all files via bind mounts. + +SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers +SPDX-License-Identifier: LGPL-3.0-only + +-=-=-=- this is a zip file -=-=-=- what follows is binary -=-=-=- +''' + + +def build_tool() -> Path: + + # generate zipapp source archive + pyzbuffer = BytesIO() + create_archive(SRC_PATH, target=pyzbuffer, + interpreter='=PLACEHOLDER=', + compressed=True) + zipdata = pyzbuffer.getvalue().removeprefix(b"#!=PLACEHOLDER=\n") + + # output with preamble + tool_path = DIST_PATH.joinpath('jlmkr') + with open(tool_path, 'wb') as f: + f.write(PREAMBLE.encode()) + f.write(zipdata) + os.chmod(tool_path, 0o755) + return tool_path + + +def build_zip() -> Path: + zip_path = DIST_PATH.joinpath(f'jlmkr-{VERSION}.zip') + tool_path = DIST_PATH.joinpath('jlmkr') + attachments = ['README.md', 'LICENSE'] + + # include the tool as-is (with the uncompressed preamble), + # then compress and attach other hangers-on + with ZipFile(zip_path, 'w') as zip: + zip.write(tool_path) + for attachment in attachments: + path = PROJECT_PATH.joinpath(attachment) + zip.write(path, compress_type=ZIP_DEFLATED) + return zip_path + + +if __name__ == '__main__': + DIST_PATH.mkdir(exist_ok=True) + print(build_tool()) + print(build_zip()) diff --git a/src/jlmkr/paths.py b/src/jlmkr/paths.py index 07cea86..3fd938b 100755 --- a/src/jlmkr/paths.py +++ b/src/jlmkr/paths.py @@ -7,7 +7,7 @@ from configparser import ConfigParser from pathlib import Path -from .utils.console import fail +from utils.console import fail def _get_selected_jailmaker_directory() -> Path: From ab2d03ebcec01b9c4d1f8344f0387e9c9bd217ef Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Thu, 8 Aug 2024 20:28:51 -0400 Subject: [PATCH 17/31] Merge GitHub actions --- .github/workflows/test.yml | 57 +++++++++++++++----------------------- 1 file changed, 22 insertions(+), 35 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8cc4e46..45947a4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,9 +13,7 @@ on: jobs: - ########################################################## - ## Build software for distribution; perform code-level QA - build: + build-and-test: name: Build jlmkr tool runs-on: ubuntu-24.04 strategy: @@ -25,6 +23,9 @@ jobs: steps: # hat tip: + ########################################################## + ## Build software for distribution; perform code-level QA + - name: Checkout code uses: actions/checkout@v4 with: @@ -48,36 +49,14 @@ jobs: - name: Run unit tests run: hatch test --cover - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - path: | - dist/jlmkr - dist/jlmkr-*.zip - if-no-files-found: error - + ########################################################## + ## Build software for distribution; perform code-level QA - ########################################################## - ## Perform rubber-meets-road integration testing - test: - # The type of runner that the job will run on - runs-on: ubuntu-24.04 - defaults: - run: - shell: bash - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.11' # TrueNAS SCALE 24.04 Dragonfish - - name: Tune GitHub-hosted runner network - uses: smorimoto/tune-github-hosted-runner-network@v1 + - uses: smorimoto/tune-github-hosted-runner-network@v1 # Create a network namespace in the GitHub-hosted runner VM, # simulating a primary bridge network on TrueNAS SCALE - - name: Install packages and setup networking + - name: Prepare for integration tests run: | sudo -s < Date: Thu, 8 Aug 2024 20:33:48 -0400 Subject: [PATCH 18/31] =?UTF-8?q?Tune=20the=20runner=E2=80=99s=20network?= =?UTF-8?q?=20before=20code=20checkout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 45947a4..46dc308 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,6 +23,9 @@ jobs: steps: # hat tip: + - name: Tune GitHub network + uses: smorimoto/tune-github-hosted-runner-network@v1 + ########################################################## ## Build software for distribution; perform code-level QA @@ -52,8 +55,6 @@ jobs: ########################################################## ## Build software for distribution; perform code-level QA - - uses: smorimoto/tune-github-hosted-runner-network@v1 - # Create a network namespace in the GitHub-hosted runner VM, # simulating a primary bridge network on TrueNAS SCALE - name: Prepare for integration tests From 2cde9184fa554f8ff9e7eaf1649f43c525b798d5 Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Thu, 8 Aug 2024 20:56:55 -0400 Subject: [PATCH 19/31] Resolve unit test module search path --- build.py | 69 -------------------------------------- pyproject.toml | 2 +- tests/test_paths.py | 2 +- tests/test_utils_editor.py | 2 +- tests/test_utils_files.py | 2 +- 5 files changed, 4 insertions(+), 73 deletions(-) delete mode 100644 build.py diff --git a/build.py b/build.py deleted file mode 100644 index 2827fcc..0000000 --- a/build.py +++ /dev/null @@ -1,69 +0,0 @@ -# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers -# -# SPDX-License-Identifier: LGPL-3.0-only - -# hat tip: - -import os -from io import BytesIO -from pathlib import Path -from typing import Any, Callable, Iterable -from zipapp import create_archive -from zipfile import ZipFile, ZIP_DEFLATED - -from src.jlmkr.__main__ import __version__ as VERSION -PROJECT_PATH = Path.cwd() -SRC_PATH = Path('./src/jlmkr') -DIST_PATH = Path('./dist') - -# 10 lines will conveniently match the default of head(1) -PREAMBLE = f'''#!/usr/bin/env python3 - -jlmkr {VERSION} - -Persistent Linux 'jails' on TrueNAS SCALE to install software (k3s, docker, portainer, podman, etc.) with full access to all files via bind mounts. - -SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers -SPDX-License-Identifier: LGPL-3.0-only - --=-=-=- this is a zip file -=-=-=- what follows is binary -=-=-=- -''' - - -def build_tool() -> Path: - - # generate zipapp source archive - pyzbuffer = BytesIO() - create_archive(SRC_PATH, target=pyzbuffer, - interpreter='=PLACEHOLDER=', - compressed=True) - zipdata = pyzbuffer.getvalue().removeprefix(b"#!=PLACEHOLDER=\n") - - # output with preamble - tool_path = DIST_PATH.joinpath('jlmkr') - with open(tool_path, 'wb') as f: - f.write(PREAMBLE.encode()) - f.write(zipdata) - os.chmod(tool_path, 0o755) - return tool_path - - -def build_zip() -> Path: - zip_path = DIST_PATH.joinpath(f'jlmkr-{VERSION}.zip') - tool_path = DIST_PATH.joinpath('jlmkr') - attachments = ['README.md', 'LICENSE'] - - # include the tool as-is (with the uncompressed preamble), - # then compress and attach other hangers-on - with ZipFile(zip_path, 'w') as zip: - zip.write(tool_path) - for attachment in attachments: - path = PROJECT_PATH.joinpath(attachment) - zip.write(path, compress_type=ZIP_DEFLATED) - return zip_path - - -if __name__ == '__main__': - DIST_PATH.mkdir(exist_ok=True) - print(build_tool()) - print(build_zip()) diff --git a/pyproject.toml b/pyproject.toml index 67a01cc..dde173a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,7 @@ python = [ ] [tool.pytest.ini_options] -pythonpath = "src" +pythonpath = "src/jlmkr" addopts = [ "--import-mode=importlib", ] diff --git a/tests/test_paths.py b/tests/test_paths.py index 4aaa2b4..8f71756 100644 --- a/tests/test_paths.py +++ b/tests/test_paths.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: LGPL-3.0-only -from jlmkr import paths +import paths def test_script_name(): diff --git a/tests/test_utils_editor.py b/tests/test_utils_editor.py index 2012bd9..fe344bb 100644 --- a/tests/test_utils_editor.py +++ b/tests/test_utils_editor.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: LGPL-3.0-only -from jlmkr.utils import editor +from utils import editor from pathlib import Path diff --git a/tests/test_utils_files.py b/tests/test_utils_files.py index 62fcb82..c0d94a3 100644 --- a/tests/test_utils_files.py +++ b/tests/test_utils_files.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: LGPL-3.0-only -from jlmkr.utils.files import get_mount_point +from utils.files import get_mount_point def test_mount_point(): From b53956f295bccf2671598dee359dab578a0480ff Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Thu, 8 Aug 2024 20:57:24 -0400 Subject: [PATCH 20/31] Build distribution without going through Hatch --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 46dc308..4b64a73 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,7 +44,7 @@ jobs: run: pip install hatch - name: Build distribution - run: hatch build --ext + run: python3 -m scripts.build - name: Run style check run: hatch fmt --check || echo IGNORING ADVISORIES From 938e76f45a8cab4ac090cc13aa004df57bf4ed28 Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Thu, 8 Aug 2024 21:16:42 -0400 Subject: [PATCH 21/31] Ensure jlmkr can run from an alternate directory --- src/jlmkr/paths.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/jlmkr/paths.py b/src/jlmkr/paths.py index 3fd938b..ba3f8e9 100755 --- a/src/jlmkr/paths.py +++ b/src/jlmkr/paths.py @@ -36,11 +36,16 @@ def _get_selected_jailmaker_directory() -> Path: if cfgname in cfg[secname]: return Path(cfg[secname][cfgname]) - # next: current directory iff it's named jailmaker - script = Path(sys.argv[0]).resolve(True) + # next: parent directory of script iff it's named jailmaker + script = get_tool_path_on_disk() if script.parent.name == 'jailmaker': return script.parent + # next: current directory iff it's named jailmaker + cwd = Path.cwd() + if cwd.name == 'jailmaker': + return cwd + fail("Please specify a jailmaker directory path (JAILMAKER_DIR)") From ec72f610c124c9f19d59afa1c4a7f050b7c56186 Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Thu, 8 Aug 2024 21:24:20 -0400 Subject: [PATCH 22/31] Upload artifacts before bolting them down --- .github/workflows/test.yml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4b64a73..589b378 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -52,6 +52,14 @@ jobs: - name: Run unit tests run: hatch test --cover + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + path: | + dist/jlmkr + dist/jlmkr-*.zip + if-no-files-found: error + ########################################################## ## Build software for distribution; perform code-level QA @@ -118,14 +126,3 @@ jobs: sudo chown 0:0 jlmkr ./test/test-jlmkr sudo bash ./test/test-jlmkr sudo ./test/test.sh - - ########################################################## - ## Release build artifacts - - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - path: | - dist/jlmkr - dist/jlmkr-*.zip - if-no-files-found: error From 1fab73aa3dc0dd7924b770b502bff6cd2ab113c3 Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Thu, 8 Aug 2024 21:25:53 -0400 Subject: [PATCH 23/31] Add testing to the developer notes --- DEVELOPER.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/DEVELOPER.md b/DEVELOPER.md index d9d6850..9d85b94 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -38,6 +38,12 @@ Use `hatch` to build, test, lint, etc. hatch build + hatch fmt --check + + hatch test --cover + +*Please don't forget to run tests before submitting a pull request!* + ## Integration testing See [`test/README.md`](./test/README.md). From fb8b8ff50a6c2a3f354ef2131ce37c40db026ac1 Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Thu, 8 Aug 2024 21:45:32 -0400 Subject: [PATCH 24/31] Hunt trailing whitespace --- src/jlmkr/data.py | 2 +- src/jlmkr/paths.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/jlmkr/data.py b/src/jlmkr/data.py index 135fa76..87fcb01 100644 --- a/src/jlmkr/data.py +++ b/src/jlmkr/data.py @@ -36,7 +36,7 @@ # echo 1 > /proc/sys/net/bridge/bridge-nf-call-ip6tables # Specify command/script to run on the HOST after starting the jail -# For example to attach to multiple bridge interfaces +# For example to attach to multiple bridge interfaces # when using --network-veth-extra=ve-myjail-1:veth1 post_start_hook= # post_start_hook=#!/usr/bin/bash diff --git a/src/jlmkr/paths.py b/src/jlmkr/paths.py index ba3f8e9..309382d 100755 --- a/src/jlmkr/paths.py +++ b/src/jlmkr/paths.py @@ -16,12 +16,12 @@ def _get_selected_jailmaker_directory() -> Path: ''' # first choice: global --dir/-D argument #TODO - + # next: JAILMAKER_DIR environment variable envname = 'JAILMAKER_DIR' if envname in os.environ: return Path(os.environ[envname]) - + # next: ~/.local/share/jailmaker.conf secname = 'DEFAULT' cfgname = 'jailmaker_dir' @@ -35,17 +35,17 @@ def _get_selected_jailmaker_directory() -> Path: os.environ['JLMKR_DEBUG'] = cfg[secname]['ignore_owner'] if cfgname in cfg[secname]: return Path(cfg[secname][cfgname]) - + # next: parent directory of script iff it's named jailmaker script = get_tool_path_on_disk() if script.parent.name == 'jailmaker': return script.parent - + # next: current directory iff it's named jailmaker cwd = Path.cwd() if cwd.name == 'jailmaker': return cwd - + fail("Please specify a jailmaker directory path (JAILMAKER_DIR)") From 97d5873acfd8ced6f209379472a485dca6530f21 Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Thu, 8 Aug 2024 21:46:52 -0400 Subject: [PATCH 25/31] Strike unused import --- scripts/build.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/build.py b/scripts/build.py index 3a4b37d..4033a56 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -4,7 +4,6 @@ # hat tip: -import importlib import os import sys from io import BytesIO From 0fb034f7d9871e635d19be6b99ad3727ad820f0c Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Fri, 9 Aug 2024 17:54:28 -0400 Subject: [PATCH 26/31] Add integration test runner using temp ZFS pools --- .github/workflows/test.yml | 5 ++ scripts/test.py | 102 +++++++++++++++++++++++++++++++++++++ scripts/test_blank.sh | 23 +++++++++ 3 files changed, 130 insertions(+) create mode 100644 scripts/test.py create mode 100644 scripts/test_blank.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 589b378..c49b9ea 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -126,3 +126,8 @@ jobs: sudo chown 0:0 jlmkr ./test/test-jlmkr sudo bash ./test/test-jlmkr sudo ./test/test.sh + + - name: Run secondary integration tests + env: + PYTHONUNBUFFERED: 1 + run: python3 -m scripts.test diff --git a/scripts/test.py b/scripts/test.py new file mode 100644 index 0000000..aea26c8 --- /dev/null +++ b/scripts/test.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2024 Jip-Hop and the Jailmakers +# +# SPDX-License-Identifier: LGPL-3.0-only + +import os +import sys + +from contextlib import AbstractContextManager, chdir +from pathlib import Path +from subprocess import run, PIPE, STDOUT, CalledProcessError +from tempfile import NamedTemporaryFile, TemporaryDirectory + +SUDO = '/usr/bin/sudo' +ZPOOL = '/sbin/zpool' +ZFS = '/sbin/zfs' + + +class TemporaryPool(AbstractContextManager): + vdev: NamedTemporaryFile + mountpoint: Path + name: str + + def __init__(self, bytesize: int): + self.vdisk = NamedTemporaryFile() + self.vdisk.truncate(bytesize) + self._tmpdir = TemporaryDirectory() + self.mountpoint = Path(self._tmpdir.name) + self.name = self.mountpoint.name + + def __enter__(self): + run([SUDO, ZPOOL, 'create', + '-m', self.mountpoint, + self.name, self.vdisk.name], check=False) + return self + + def __exit__(self, exc_type, exc_value, traceback): + run([SUDO, ZPOOL, 'destroy', self.name], check=False) + + +class JailmakerDataset(AbstractContextManager): + name: str + path: Path + + def __init__(self, pool: TemporaryPool): + self.name = f'{pool.name}/jailmaker' + self.path = pool.mountpoint.joinpath('jailmaker') + + def __enter__(self): + run([SUDO, ZFS, 'create', self.name], check=False) + run([SUDO, '/usr/bin/cp', 'dist/jlmkr', self.path], check=False) + return self + + def __exit__(self, exc_type, exc_value, traceback): + run([SUDO, ZFS, 'destroy', '-r', self.name], check=False) + + +def run_test(path): + with ( + TemporaryPool(512*1024*1024) as pool, + JailmakerDataset(pool) as dataset, + ): + print(f'▶️ {path.name} ...') + environment = { + **os.environ, + 'JAILMAKER_DIR': dataset.path, + 'TEMPLATES': Path('templates').resolve(True), + } + result = run([path], env=environment, + capture_output=False, #stdout=PIPE, stderr=STDOUT, + cwd=dataset.path, check=False) + if result.returncode == 0: + print(f'✅ {path.name} completed OK') + else: + if result.stdout: + print(result.stdout.decode()) + print(f'❌ {path.name} returned {result.returncode}') + result.check_returncode() + + +if __name__ == '__main__': + jlmkr = Path('dist/jlmkr') + if not jlmkr.is_file(): + raise(Exception('build jlmkr first')) + testdir = Path(__file__).parent + + succeeded = 0 + failed = 0 + for testscript in testdir.iterdir(): + if testscript.name.startswith('test_'): + try: + run_test(testscript) + succeeded += 1 + except CalledProcessError: + failed += 1 + print() + if failed: + print(f'❌ {failed} of {failed+succeeded} tests FAILED') + sys.exit(-1) + else: + print(f'✅ All {succeeded} tests OK') + sys.exit(0) diff --git a/scripts/test_blank.sh b/scripts/test_blank.sh new file mode 100644 index 0000000..b1502aa --- /dev/null +++ b/scripts/test_blank.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +# $JAILMAKER_DIR -> Jailmaker directory (also current working dir) +# $TEMPLATES -> path to project templates directory + +# Side-effects are a problem. We can't sandbox network settings, +# etc. and we can only clean up ZFS if no jails are left running. + + +sudo -E ./jlmkr create blankjail + +sudo -E ./jlmkr list + +sudo -E ./jlmkr start blankjail + +sudo -E ./jlmkr list + +sudo -E ./jlmkr stop blankjail + +echo blankjail | sudo -E ./jlmkr remove blankjail + +sudo -E ./jlmkr list From 79683af31d5bfb8aff81728d9ca7f7291dda634d Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Fri, 9 Aug 2024 18:00:56 -0400 Subject: [PATCH 27/31] Install zfsutils-linux on GitHub runner --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c49b9ea..c249e3d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -74,7 +74,7 @@ jobs: echo 'nameserver 1.1.1.1' > /etc/resolv.conf apt-get update - apt-get install -qq -y systemd-container + apt-get install -qq -y systemd-container zfsutils-linux cat </etc/systemd/network/10-br1.network [Match] From 8441793f9440c1a7aa67c9294e74bd088cbb432f Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Fri, 9 Aug 2024 18:06:33 -0400 Subject: [PATCH 28/31] Probe the GitHub Runner environment --- .github/workflows/test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c249e3d..d24fcc0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -106,6 +106,8 @@ jobs: cat /etc/os-release python3 --version ip addr + ls -laFh scripts + which bash # # TODO: create zpool with virtual disks, create jailmaker dataset and test jlmkr.py from there # # https://medium.com/@abaddonsd/zfs-usage-with-virtual-disks-62898064a29b @@ -124,8 +126,8 @@ jobs: run: | sudo ln dist/jlmkr . sudo chown 0:0 jlmkr ./test/test-jlmkr - sudo bash ./test/test-jlmkr - sudo ./test/test.sh +# sudo bash ./test/test-jlmkr +# sudo ./test/test.sh - name: Run secondary integration tests env: From 977f9f78fac5be2e1ed58d9639331ba8658aba1a Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Fri, 9 Aug 2024 18:12:41 -0400 Subject: [PATCH 29/31] Load ZFS; mark test scripts executable --- .github/workflows/test.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d24fcc0..1b75fe8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -98,6 +98,8 @@ jobs: iptables -I DOCKER-USER -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT iptables -I DOCKER-USER -i br1 -o eth0 -j ACCEPT + + modprobe zfs END - name: Inspect the runtime environment @@ -106,8 +108,7 @@ jobs: cat /etc/os-release python3 --version ip addr - ls -laFh scripts - which bash + zfs version # # TODO: create zpool with virtual disks, create jailmaker dataset and test jlmkr.py from there # # https://medium.com/@abaddonsd/zfs-usage-with-virtual-disks-62898064a29b @@ -126,10 +127,12 @@ jobs: run: | sudo ln dist/jlmkr . sudo chown 0:0 jlmkr ./test/test-jlmkr -# sudo bash ./test/test-jlmkr -# sudo ./test/test.sh + sudo bash ./test/test-jlmkr + sudo ./test/test.sh - name: Run secondary integration tests env: PYTHONUNBUFFERED: 1 - run: python3 -m scripts.test + run: | + chmod +x scripts/* + python3 -m scripts.test From 5ec22c14c10da167acf075a1007b537d99d7e5c4 Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Sat, 10 Aug 2024 01:40:44 -0400 Subject: [PATCH 30/31] Sweep abandoned jails from temporary test pools --- scripts/test.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scripts/test.py b/scripts/test.py index aea26c8..f5f9061 100644 --- a/scripts/test.py +++ b/scripts/test.py @@ -10,10 +10,12 @@ from pathlib import Path from subprocess import run, PIPE, STDOUT, CalledProcessError from tempfile import NamedTemporaryFile, TemporaryDirectory +from time import sleep SUDO = '/usr/bin/sudo' ZPOOL = '/sbin/zpool' ZFS = '/sbin/zfs' +MACHINECTL = '/usr/bin/machinectl' class TemporaryPool(AbstractContextManager): @@ -52,8 +54,30 @@ def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): + for straggler in self.find_running_stragglers(): + print(f'⚠️ Terminating {straggler}', file=sys.stderr) + run([SUDO, MACHINECTL, 'terminate', straggler], check=False) + for tries in range(10): + attempt = run([MACHINECTL, 'show', straggler], + capture_output=True, check=False) + if not attempt.stdout: + break + sleep(0.5) run([SUDO, ZFS, 'destroy', '-r', self.name], check=False) + def find_running_stragglers(self): + machinelist = run([MACHINECTL, 'list', '--no-legend'], + capture_output=True, check=False) + for listing in machinelist.stdout.decode().splitlines(): + machinename = listing.partition(' ')[0] + lookup = run([MACHINECTL, 'show', + '-p', 'RootDirectory', '--value', + machinename], + capture_output=True, check=False) + machinepath = lookup.stdout.decode() + if machinepath.startswith(str(self.path)): + yield machinename + def run_test(path): with ( From 5c8b41061f6d8c39ed7141b16815645bff3f284d Mon Sep 17 00:00:00 2001 From: jonct <2807816+jonct@users.noreply.github.com> Date: Sat, 10 Aug 2024 01:43:46 -0400 Subject: [PATCH 31/31] Document lifted restriction on test stragglers --- scripts/test_blank.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/test_blank.sh b/scripts/test_blank.sh index b1502aa..ffe853d 100644 --- a/scripts/test_blank.sh +++ b/scripts/test_blank.sh @@ -4,8 +4,7 @@ set -euo pipefail # $JAILMAKER_DIR -> Jailmaker directory (also current working dir) # $TEMPLATES -> path to project templates directory -# Side-effects are a problem. We can't sandbox network settings, -# etc. and we can only clean up ZFS if no jails are left running. +# Side-effects are a problem. We can't sandbox network settings, etc. sudo -E ./jlmkr create blankjail