From d52785caf10857b8aaa962986139181e3e6c51e4 Mon Sep 17 00:00:00 2001 From: Alexander Artemenko Date: Fri, 23 Feb 2024 05:22:33 +0000 Subject: [PATCH] Added cache support inside the action itself. Now it can run up to 10 times faster! --- action.yml | 193 +++++++++++++++++++++++++++++++++++++++++-------- changelog.lisp | 8 ++ docs.lisp | 79 +++++--------------- 3 files changed, 191 insertions(+), 89 deletions(-) diff --git a/action.yml b/action.yml index 4bc8bbe..58c274f 100644 --- a/action.yml +++ b/action.yml @@ -1,44 +1,98 @@ name: 'Setup Common Lisp' +author: Alexander Artemenko +description: This action setup Roswell and a Common Lisp implementation plus Qlot for managing virtual environments. inputs: roswell-version: description: 'Roswell version to install. If not specified, the latest working version will be used; if "latest", the latest version is used' required: false default: v23.10.14.114 + asdf-system: description: 'ASDF system to install' required: false + asdf-version: description: 'ASDF version to install. If not specified, the latest working version will be used; if "latest", the latest version is used' required: false default: 3.3.5.3 + qlot-version: description: 'Qlot version to install. If not specified, the latest working version will be used; if "latest", the latest version is used' required: false default: 0.11.5 + qlfile-template: description: "Djula template for qlfile. All environment variables are available in it's context" required: false + cache: + description: 'If true (default), then cache will be created to speedup repeated action runs.' + required: false + default: true + + # GitHub does not support anchors in the action + # and returns error like this: + # + # Anchors are not currently supported. Remove the anchor 'roswell-cache-paths' + # + # That is why I use "input" variable to not repeat this list in two places + roswell-cache-paths: + description: "Internal var. Don't use it." + required: false + default: | + ~/.quicklisp-client-fix + ~/.roswell + /usr/local/etc/roswell + /usr/local/bin/ros + /usr/local/Cellar/roswell + + qlot-cache-paths: + description: "Internal var. Don't use it." + required: false + default: | + path: | + qlfile + qlfile.lock + ~/.cache/common-lisp/ + .qlot + runs: using: composite steps: - # Using branch v2.14.0 - - uses: msys2/setup-msys2@d40200dc2db4c351366b048a9565ad82919e1c24 + - name: Calculate variables + id: locals + shell: bash + run: | + if [[ '${{ inputs.roswell-version }}' == 'latest' ]] + then + echo "windows-package-name=mingw-w64-x86_64-roswell" >> $GITHUB_OUTPUT + else + # Strip v prefix from version number + ROS_VERSION=$(echo ${{ inputs.roswell-version }} | sed 's/v//') + echo "windows-package-name=mingw-w64-x86_64-roswell=$ROS_VERSION" >> $GITHUB_OUTPUT + fi + + # echo 'roswell-cache-paths=~/.quicklisp-client-fix\n~/.roswell\n/usr/local/etc/roswell\n/usr/local/bin/ros\n/usr/local/Cellar/roswell' >> $GITHUB_OUTPUT + + echo "current-month=$(date -u '+%Y-%m')" >> $GITHUB_OUTPUT + + - if: runner.os == 'Windows' + uses: msys2/setup-msys2@cc11e9188b693c2b100158c3322424c4cc1dadea #v2.22.0 with: - # Roswell was added to msys2 just _recently_, so the following makes - # sure packages metadata is up to date. Otherwise... - # - # $ pacman -S mingw-w64-x86_64-roswell - # error: target not found: mingw-w64-x86_64-roswell - # Error: Process completed with exit code 1 - update: true # Msys2 has its own PATH, and the following setting enables standard # PATH manipulation expressions like the one shown below, to succeed: # # $ echo /usr/local/bin >> $GITHUB_PATH path-type: inherit platform-check-severity: warn + # Installing ASDF requires `make`, so let's make sure it's + # available + install: >- + make + ${{ steps.locals.outputs.windows-package-name }} + cache: ${{ inputs.cache }} + - name: Create lispsh shell: bash run: | @@ -63,7 +117,7 @@ runs: # below, and have some of them with `shell: bash`, and others with # `shell: msys2 {0}`. if [[ "$RUNNER_OS" == "Windows" ]]; then - powershell New-Item -ItemType SymbolicLink \ + powershell New-Item -ItemType SymbolicLink -Force \ -Path "D:/a/_temp/setup-msys2/lispsh.cmd" \ -Target "D:/a/_temp/setup-msys2/msys2.cmd" else @@ -121,17 +175,56 @@ runs: fi echo $HOME/.roswell/bin >> $GITHUB_PATH echo ::endgroup:: - - name: Current Env - shell: bash + + # TODO: comment for prod + # - name: Current Env + # shell: bash + # run: | + # echo ::group::Environment + # echo "Current dir:" + # pwd + + # echo "Environment Variables:" + # env | sort -u + # echo ::endgroup:: + + # On Windows we dont have such problems with permission. + # Also we don't have sudo there, so just skip this step + # on this platform: + - if: inputs.cache == 'true' && runner.os != 'Windows' + name: Grant All Perms to Make Roswell Cache Restoring Possible + shell: lispsh -eo pipefail {0} run: | - echo ::group::Environment - echo "Current dir:" - pwd + sudo mkdir -p /usr/local/etc/roswell + sudo chown "${USER}" /usr/local/etc/roswell + # Here the ros binary will be restored: + sudo chown "${USER}" /usr/local/bin - echo "Environment Variables:" - env | sort -u - echo ::endgroup:: - - name: Install Roswell + - if: inputs.cache == 'true' + name: Restore Roswell From Cache + id: roswell-cache-restore + uses: actions/cache/restore@v4 + with: + path: ${{ inputs.roswell-cache-paths }} + key: roswell-${{ inputs.roswell-version }}-${{ steps.locals.outputs.current-month }}-${{ env.cache-name }}-${{ runner.os }}-${{ env.LISP }} + + - if: inputs.cache == 'true' && steps.roswell-cache-restore.outputs.cache-hit == 'true' + name: Restore Path To Cached Files + shell: lispsh -eo pipefail {0} + run: | + echo $HOME/.roswell/bin >> $GITHUB_PATH + echo .qlot/bin >> $GITHUB_PATH + + if [[ "$RUNNER_OS" == "Windows" ]]; then + echo $HOME/.roswell/lisp/quicklisp/bin >> $GITHUB_PATH + fi + + + # Start the piece which results should be cached + + # On Windows we install roswell using Pacman package manager and don't need this step + - if: (inputs.cache == 'false' || steps.roswell-cache-restore.outputs.cache-hit != 'true') && runner.os != 'Windows' + name: Install Roswell shell: lispsh -eo pipefail {0} run: | echo ::group::Installing Roswell dependencies @@ -143,11 +236,7 @@ runs: if [[ "$RUNNER_OS" == "macOS" ]]; then brew install automake autoconf curl fi - if [[ "$RUNNER_OS" == "Windows" ]]; then - # Installing ASDF requires `make`, so let's make sure it's - # available - msys2.cmd -c "pacman --noconfirm -S --needed --overwrite '*' make" - fi + echo ::endgroup:: if [[ "${{ inputs.roswell-version }}" != "latest" ]]; then @@ -159,14 +248,18 @@ runs: fi echo ::endgroup:: - - name: Upgrade Quicklisp dists + + - if: inputs.cache == 'false' || steps.roswell-cache-restore.outputs.cache-hit != 'true' + name: Upgrade Quicklisp dists shell: lispsh -eo pipefail {0} run: | # The parent workflow might have caching enabled for Roswell and all # the other Lisp files in general, so it's better to tell Quicklisp # to update all its dists. ros -e "(ql:update-all-dists :prompt nil)" - - name: Install Quicklisp patch for package-inferred systems + + - if: inputs.cache == 'false' || steps.roswell-cache-restore.outputs.cache-hit != 'true' + name: Install Quicklisp patch for package-inferred systems shell: lispsh -eo pipefail {0} run: | git clone \ @@ -194,7 +287,9 @@ runs: (t (format t "Quicklisp fix was not found at ~S.~%" fix-filename))))) EOF - - name: Upgrade ASDF to the Latest Version + + - if: inputs.cache == 'false' || steps.roswell-cache-restore.outputs.cache-hit != 'true' + name: Upgrade ASDF to the Latest Version shell: lispsh -eo pipefail {0} run: | if [[ "${{ inputs.asdf-version }}" != "latest" ]]; then @@ -205,7 +300,9 @@ runs: ros install asdf fi echo ::endgroup:: - - name: Install Qlot + + - if: inputs.cache == 'false' || steps.roswell-cache-restore.outputs.cache-hit != 'true' + name: Install Qlot shell: lispsh -eo pipefail {0} run: | if [[ "${{ inputs.qlot-version }}" != "latest" ]]; then @@ -217,7 +314,32 @@ runs: fi echo .qlot/bin >> $GITHUB_PATH echo ::endgroup:: - - name: Create Qlot Environment + + - if: inputs.cache == 'true' && steps.roswell-cache-restore.outputs.cache-hit != 'true' + name: Cache Roswell Setup + id: roswell-cache-save + uses: actions/cache/save@v4 + with: + path: ${{ inputs.roswell-cache-paths }} + key: ${{ steps.roswell-cache-restore.outputs.cache-primary-key }} + + + - if: inputs.cache == 'true' + name: Restore Qlot Environment + id: qlot-cache-restore + uses: actions/cache/restore@v4 + with: + path: ${{ inputs.qlot-cache-paths }} + key: qlot-${{ steps.locals.outputs.current-month }}-${{ env.cache-name }}-${{ runner.os }}-${{ env.QUICKLISP_DIST }}-${{ env.LISP }}-${{ hashFiles('qlfile', 'qlfile.lock', '*.asd') }} + + - if: inputs.cache == 'true' && steps.qlot-cache-restore.outputs.cache-hit == 'true' + name: Restore Path To .qlot/bin + shell: lispsh -eo pipefail {0} + run: | + echo .qlot/bin >> $GITHUB_PATH + + - if: inputs.cache == 'false' || steps.qlot-cache-restore.outputs.cache-hit != 'true' + name: Create Qlot Environment shell: lispsh -eo pipefail {0} run: | echo ::group::Create Qlot Environment @@ -246,7 +368,8 @@ runs: # This step will install system and # all possible roswell scripts, if the system # has them in the roswell/ subdirectory: - - name: Install ASDF System + - if: inputs.cache == 'false' || steps.qlot-cache-restore.outputs.cache-hit != 'true' + name: Install ASDF System shell: lispsh -eo pipefail {0} run: | echo ::group::Install ASDF System @@ -258,6 +381,16 @@ runs: fi echo ::endgroup:: + - if: inputs.cache == 'true' && steps.qlot-cache-restore.outputs.cache-hit != 'true' + name: Cache Qlot Environment + id: qlot-cache-save + uses: actions/cache/save@v4 + with: + path: ${{ inputs.qlot-cache-paths }} + key: ${{ steps.qlot-cache-restore.outputs.cache-primary-key }} + + # End of the cached piece + - name: Check it is possible to run desired lisp implementation shell: lispsh -eo pipefail {0} # Call ${{ github.action_path }}test.ros does not work on windows diff --git a/changelog.lisp b/changelog.lisp index ebe40a7..79368f1 100644 --- a/changelog.lisp +++ b/changelog.lisp @@ -8,6 +8,14 @@ (defchangelog (:ignore-words ("ASDF" "PATH" "HOME")) + (4.0.0 2024-02-24 + " +# Changed + +* Internal cache mechanism was added. Now action caches Roswell and Qlot files to speed up reruns. For example, without cache action could be executed about 4 minutes, and with cache it runs only 20 seconds on Ubuntu or 1 minute on Windows. +* A new input variable `cache` was added to control caching beheviour. It is `true` by default, but you can switch it to `false` to turn caching off. + +") (3.2.0 2023-01-30 " # Changed diff --git a/docs.lisp b/docs.lisp index 3af818d..d3e589c 100644 --- a/docs.lisp +++ b/docs.lisp @@ -71,6 +71,8 @@ and [Qlot](https://github.com/fukamachi/qlot) inside the Github CI. `qlfile`, see \"Overriding qlfile\" section. * And finally, it can install a specified ASDF system and all it's dependencies. But this step is optional. +* Installed Roswell, `.qlot` and `~/.cache/common-lisp/` files are cached to speed up + repeated builds. ") @@ -91,7 +93,7 @@ when trying to call `ros` or `qlot` scripts: | **Implementation** | **Supported** | |--------------------|------------------------------------------------------| | abcl-bin | ✅ | -| allegro | ✅ | +| allegro | ✅ (sometimes fails because of \"expired-license\") | | ccl-bin | ✅ | | clasp | [❌](https://github.com/40ants/setup-lisp/issues/16) | | clasp-bin | ✅ | @@ -134,8 +136,8 @@ jobs: LISP: ${{ matrix.lisp }} steps: - - uses: actions/checkout@v2 - - uses: 40ants/setup-lisp@v2 + - uses: actions/checkout@v4 + - uses: 40ants/setup-lisp@v4 with: asdf-system: cl-info - uses: 40ants/run-tests@v2 @@ -146,7 +148,7 @@ jobs: The part, corresponding to an action call is: ```yaml -- uses: 40ants/setup-lisp@v2 +- uses: 40ants/setup-lisp@v4 with: asdf-system: cl-info ``` @@ -175,7 +177,7 @@ working with this action. However, should you need to use a different version instead, you can specify that via the `roswell-version` argument: ``` -- uses: 40ants/setup-lisp@v2 +- uses: 40ants/setup-lisp@v4 with: roswell-version: v21.10.14.111 ``` @@ -189,7 +191,7 @@ working with this action. However, should you need to use a different version instead, you can specify that via the `asdf-version` argument: ``` -- uses: 40ants/setup-lisp@v2 +- uses: 40ants/setup-lisp@v4 with: asdf-version: 3.3.5.3 ``` @@ -203,7 +205,7 @@ working with this action. However, should you need to use a different version instead, you can specify that via the `qlot-version` argument: ``` -- uses: 40ants/setup-lisp@v2 +- uses: 40ants/setup-lisp@v4 with: qlot-version: 0.11.5 ``` @@ -240,8 +242,8 @@ env: QUICKLISP_DIST: ${{ matrix.quicklisp-dist }} steps: - - uses: actions/checkout@v2 - - uses: 40ants/setup-lisp@v2 + - uses: actions/checkout@v4 + - uses: 40ants/setup-lisp@v4 with: asdf-system: cl-info qlfile-template: | @@ -271,58 +273,17 @@ Usually installing Roswell, a lisp implementation and dependencies take from 2 to 10 minutes. Multiply this to the number of matrix combinations and you'll get signifficant time. -To speed up build, you can use caching using a standad GitHub action `actions/cache@v2`. +Starting from version `4.0.0`, this action cares about caching itself +and you don't need to wrap it with `actions/cache`. This behaviour +of enabled by default. Without cache action could be executed about +4 minutes, and with cache it runs only 20 seconds on Ubuntu or 1 minute on Windows. -To make caching work, add such sections into your workflow file: +A new input variable `cache` was added to control caching beheviour. +It is `true` by default, but you can switch it to `false` to turn caching off. -```yaml -- name: Grant All Perms to Make Cache Restoring Possible - run: | - sudo mkdir -p /usr/local/etc/roswell - sudo chown \"${USER}\" /usr/local/etc/roswell - # Here the ros binary will be restored: - sudo chown \"${USER}\" /usr/local/bin -- name: Get Current Month - id: current-month - run: | - echo \"::set-output name=value::$(date -u \"+%Y-%m\")\" -- name: Cache Roswell Setup - id: cache - uses: actions/cache@v2 - env: - cache-name: cache-roswell - with: - path: | - /usr/local/bin/ros - ~/.cache/common-lisp/ - ~/.roswell - /usr/local/etc/roswell - .qlot - key: \"${{ steps.current-month.outputs.value }}-${{ env.cache-name }}-${{ runner.os }}-${{ hashFiles('qlfile.lock') }}\" -- name: Restore Path To Cached Files - run: | - echo $HOME/.roswell/bin >> $GITHUB_PATH - echo .qlot/bin >> $GITHUB_PATH - if: steps.cache.outputs.cache-hit == 'true' -- uses: 40ants/setup-lisp@v2 - if: steps.cache.outputs.cache-hit != 'true' -``` - -There are two important lines here. - -- The last line `if: steps.cache.outputs.cache-hit != 'true'` skips - running lisp installation, it it was take from the cache. -- The `key` value: - - ``` - key: \"${{ steps.current-month.outputs.value }}-${{ env.cache-name }}-${{ runner.os }}-${{ hashFiles('qlfile.lock') }}\" - ``` - - It controls when your cache will be matched. If you are using `matrix`, put all it's components - into the key. - - I also added a current month there, to make sure cache will be renewed at least monthly. - This way a new Roswell, Qlot and ASDF will be used in a build. +The current month is used as part of the cache key, to refresh caches every month. +This way a new Roswell, Qlot and ASDF will be used in a build. Also, you can set +`env.cache-name` variable to some value, to force rebuild with a fresh cache. ")