Merge pull request #848 from projectsyn/feat/reclass-rs

Use `reclass-rs` in Commodore Docker image

.github/workflows/build-virtualenv-caches.yml

@@ -65,6 +65,25 @@ jobs:
       run: echo "PYVER=$(echo ${{ matrix.python-version}} |cut -d. -f1,2)" >> $GITHUB_ENV
     - run: |
         make testenv_py${PYVER}
+  build-test-reclass-rs-virtualenvs:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+    - uses: Gr1N/setup-poetry@v8
+    - name: Install tox
+      run: |
+        pip install tox
+    - uses: actions/cache@v3
+      with:
+        path: |
+          .tox
+          ~/.cache/pypoetry/virtualenvs
+        key: ${{ runner.os }}-test-py3.11-reclass-rs-${{ hashFiles('poetry.lock', 'tox.ini') }}
+    - run: |
+        make testenv_py3.11_reclass_rs
 
   build-bench-virtualenvs:
     runs-on: ubuntu-latest

.github/workflows/test.yml

@@ -74,6 +74,32 @@ jobs:
       run: echo "PYVER=$(echo ${{ matrix.python-version}} |cut -d. -f1,2)" >> $GITHUB_ENV
     - name: Run tests on Python ${{ matrix.python-version }}
       run: make test_py${PYVER}
+  tests_reclass_rs:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+    - uses: Gr1N/setup-poetry@v8
+    - name: Install tox
+      run: |
+        pip install tox
+    - uses: actions/cache@v3
+      with:
+        path: |
+          .tox
+          ~/.cache/pypoetry/virtualenvs
+        key: ${{ runner.os }}-test-py3.11-reclass-rs-${{ hashFiles('poetry.lock', 'tox.ini') }}
+    - name: Install jsonnet-bundler
+      run: |
+        mkdir -p /opt/bin && curl -sLo /opt/bin/jb \
+          https://github.com/jsonnet-bundler/jsonnet-bundler/releases/download/v0.4.0/jb-linux-amd64 \
+          && chmod +x /opt/bin/jb
+    - name: Update PATH
+      run: echo "/opt/bin" >> $GITHUB_PATH
+    - name: Run tests on Python 3.11 with reclass-rs
+      run: make test_py3.11_reclass_rs
   benchs:
     runs-on: ubuntu-latest
     strategy:

Dockerfile

@@ -20,7 +20,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY pyproject.toml poetry.lock ./
 
 RUN poetry config virtualenvs.create false \
- && poetry install --no-dev --no-root
+ && poetry install --no-dev --no-root \
+ && curl -L \
+ https://raw.githubusercontent.com/projectsyn/reclass-rs/main/hack/kapitan_0.32_reclass_rs.patch \
+ | patch -p1 -d /usr/local/lib/python3.11/site-packages/kapitan
 
 COPY . ./
 

docs/modules/ROOT/pages/explanation/running-commodore.adoc

@@ -89,6 +89,30 @@ brew install libmagic
 commodore --version
 ----
 
+. Finally, patch Kapitan to use `reclass-rs` over the bundled Python implementation.
+This significantly improves compilation times of Commodore cluster catalogs.
++
+[source,bash]
+----
+KAPITAN_PATH=$(python -c 'import kapitan; print(kapitan.__path__[0])')
+curl -L \
+  https://raw.githubusercontent.com/projectsyn/reclass-rs/main/hack/kapitan_0.32_reclass_rs.patch \
+  | patch -p1 -d "${KAPITAN_PATH}"
+----
++
+[NOTE]
+====
+The `reclass-rs` Python package is installed automatically as a dependency of Commodore.
+For officially supported platforms (Linux, macOS), prebuilt wheels are available on PyPI.
+====
++
+[TIP]
+====
+This step is optional, but we highly recommend using `reclass-rs` over Kapitan's bundled Reclass implementation.
+
+We're actively working with Kapitan to upstream support for `reclass-rs`, so this step may disappear again in future versions of this page.
+====
+
 === Usage
 
 If you've installed Commodore into a virtualenv, you either need to activate the virtualenv whenever you want to run Commodore, or you can create a symlink to the `commodore` entrypoint script in a directory which is part of your `$PATH`.

pyproject.toml

@@ -43,6 +43,7 @@ cruft = "2.15.0"
 oauthlib = "3.2.2"
 pyjwt = "2.8.0"
 PyGithub = "1.59.1"
+reclass-rs = "0.1.0"
 
 [tool.poetry.dev-dependencies]
 tox = "3.28.0"

tools/patch-kapitan.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -eo pipefail
+
+KAPITAN_DIRECTORY=$1
+echo "Patching Kapitan in $KAPITAN_DIRECTORY"
+if [ -f ${KAPITAN_DIRECTORY}/.patched ]; then
+  echo "Already patched"
+  exit 0
+fi
+
+curl -L https://raw.githubusercontent.com/projectsyn/reclass-rs/main/hack/kapitan_0.32_reclass_rs.patch | patch -p1 -d $KAPITAN_DIRECTORY
+touch ${KAPITAN_DIRECTORY}/.patched

tox.ini

@@ -7,6 +7,7 @@ envlist =
     mypy
     black
     py3{8,9,10,11}{,-bench}
+    py311-reclass-rs
 
 [testenv]
 description = Unit tests and doctests
@@ -20,6 +21,7 @@ deps =
     !bench: pytest-xdist
     bench: pytest-benchmark
 commands = \
+    reclass-rs: {toxinidir}/tools/patch-kapitan.sh {envsitepackagesdir}/kapitan
     pytest {toxinidir}/tests \
     bench: -m bench \
     !bench: -m "not bench and not integration" -n auto --dist worksteal \
@@ -28,6 +30,10 @@ passenv =
     SSH_AUTH_SOCK
     HOME
     PATH
+allowlist_externals =
+    curl
+    patch
+    {toxinidir}/tools/patch-kapitan.sh
 
 [testenv:bandit]
 description = PyCQA security linter

tox.mk

@@ -39,7 +39,7 @@ lintenv_mypy:
 lintenv_black:
 	$(TOX_COMMAND) -e black --notest
 
-.PHONY: test_py3.8 test_py3.9 test_py3.10 test_py3.11
+.PHONY: test_py3.8 test_py3.9 test_py3.10 test_py3.11 test_py3.11_reclass_rs
 
 test_py3.8:
 	$(TOX_COMMAND) -e py38
@@ -53,7 +53,11 @@ test_py3.10:
 test_py3.11:
 	$(TOX_COMMAND) -e py311
 
-.PHONY: testenv_py3.8 testenv_py3.9 testenv_py3.10 testenv_py3.11
+test_py3.11_reclass_rs:
+	$(TOX_COMMAND) -e py311-reclass-rs
+
+.PHONY: testenv_py3.8 testenv_py3.9 testenv_py3.10 testenv_py3.11 testenv_py3.11_reclass_rs
+
 
 testenv_py3.8:
 	$(TOX_COMMAND) -e py38 --notest
@@ -67,6 +71,9 @@ testenv_py3.10:
 testenv_py3.11:
 	$(TOX_COMMAND) -e py311 --notest
 
+testenv_py3.11_reclass_rs:
+	$(TOX_COMMAND) -e py311 --notest
+
 .PHONY: bench_py3.8 bench_py3.9 bench_py3.10 bench_py3.11
 
 bench_py3.8: