test_changed_notebooks #165
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflows executes new or modified example notebooks. | |
| name: test_changed_notebooks | |
| defaults: | |
| run: | |
| shell: bash # To override PowerShell on Windows | |
| on: | |
| # Trigger the workflow on push or PR to any branch | |
| push: | |
| paths: | |
| - '**/*.ipynb' | |
| pull_request: | |
| paths: | |
| - '**/*.ipynb' | |
| # don't trigger for draft PRs | |
| types: [ opened, synchronize, reopened, ready_for_review ] | |
| # Trigger the workflow on manual dispatch | |
| workflow_dispatch: | |
| jobs: | |
| select-runner: | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| runner: ${{ steps.select_runner.outputs.runner }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Select runner | |
| id: select_runner | |
| run: | | |
| if [ "${{ github.repository }}" = "NeuroDesk/example-notebooks" ]; then | |
| echo "runner=self-hosted" >> $GITHUB_OUTPUT | |
| else | |
| echo "runner=ubuntu-22.04" >> $GITHUB_OUTPUT | |
| fi | |
| get-notebooks: | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| notebook_list: ${{ steps.list_changed_notebooks.outputs.notebook_list }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Find all notebooks with changes | |
| id: find_changed_notebooks | |
| uses: tj-actions/[email protected] | |
| with: | |
| path: "./books" | |
| files: | | |
| **/*.ipynb | |
| *.md | |
| dir_names_exclude_current_dir: "true" | |
| - name: Filter down to changed notebooks | |
| id: list_changed_notebooks | |
| run: | | |
| echo ${{ steps.find_changed_notebooks.outputs.all_changed_and_modified_files }} | |
| changed_notebooks=$(echo "${{ steps.find_changed_notebooks.outputs.all_changed_and_modified_files }}") | |
| notebook_list='[' | |
| for NOTEBOOK in $(echo "${changed_notebooks}"); do | |
| notebook_list+="\"${NOTEBOOK}\"," | |
| done | |
| notebook_list=$(sed '$s/,$//' <<< $notebook_list) | |
| notebook_list+=']' | |
| echo "notebook_list=${notebook_list}" | |
| echo "notebook_list=${notebook_list}" >> $GITHUB_OUTPUT | |
| test-notebooks: | |
| needs: [ select-runner, get-notebooks ] | |
| if: ${{ needs.get-notebooks.outputs.notebook_list != '[]' }} | |
| runs-on: ${{ needs.select-runner.outputs.runner }} | |
| timeout-minutes: 1400 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| notebooks: ${{ fromJson(needs.get-notebooks.outputs.notebook_list) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Test singularity and cvmfs before pushing to neurocommand | |
| run: | |
| /bin/bash .github/workflows/setup_cvmfs.sh | |
| - name: Install dependencies | |
| shell: bash | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install jupyter-book ghp-import graphviz ipyniivue osfclient osfclient | |
| - name: Run ${{ matrix.notebooks }} | |
| shell: bash | |
| timeout-minutes: 1400 | |
| working-directory: ./books | |
| run: | | |
| echo ${{ github.workspace }} | |
| echo $GITHUB_WORKSPACE | |
| export PATH=$HOME/.local/bin:$PATH | |
| echo "PATH=$PATH" | |
| export LD_PRELOAD= | |
| echo "LD_PRELOAD=$LD_PRELOAD" | |
| export APPTAINER_BINDPATH=${{ github.workspace }},/cvmfs | |
| echo "APPTAINER_BINDPATH=$APPTAINER_BINDPATH" | |
| export LMOD_CMD=/usr/share/lmod/lmod/libexec/lmod | |
| export MPLCONFIGDIR=${{ github.workspace }}/matplotlib-mpldir | |
| export MODULEPATH=$(find /cvmfs/neurodesk.ardc.edu.au/neurodesk-modules/ -maxdepth 1 -mindepth 1 -type d -exec realpath {} \; | tr '\n' ':') | |
| echo "MODULEPATH=$MODULEPATH" | |
| echo "NOTEBOOK=$(basename -s .ipynb ${{ matrix.notebooks }})" | |
| echo "DIRNAME=$(dirname ${{ matrix.notebooks }})" | |
| echo "NOTEBOOK=$(basename -s .ipynb ${{ matrix.notebooks }})" >> $GITHUB_ENV | |
| echo "DIRNAME=$(dirname ${{ matrix.notebooks }})" >> $GITHUB_ENV | |
| jb build ${{ matrix.notebooks }} | |
| - name: Restructure | |
| shell: bash | |
| run: | | |
| tree ./books/_build/**/**/html | |
| mkdir -p ./books/_build/**/**/html/$DIRNAME | |
| if [ "$NOTEBOOK" == "intro.md" ]; then | |
| echo "The directories are the same. Skipping the next step." | |
| else | |
| cp -R ./books/_build/**/**/html/_static ./books/_build/**/**/html/$DIRNAME | |
| fi | |
| if [ -d ./books/_build/**/**/html/_images ]; then | |
| cp -R ./books/_build/**/**/html/_images ./books/_build/**/**/html/$DIRNAME | |
| fi | |
| if [ "$NOTEBOOK" == "intro.md" ]; then | |
| echo "The files are the same. Skipping the next step." | |
| else | |
| cp ./books/_build/**/**/html/$NOTEBOOK.html ./books/_build/**/**/html/$DIRNAME | |
| fi | |
| - name: Push to GitHub Pages | |
| uses: peaceiris/[email protected] | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| publish_dir: ./books/_build/**/**/html | |
| keep_files: true | |
| cname: www.neurodesk.org | |
| - name: Clean build dir | |
| run: | | |
| rm -rf /home/ubuntu/actions_github_pages_* | |
| build-structure: | |
| if: always() | |
| needs: [ test-notebooks ] | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install dependencies | |
| shell: bash | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install jupyter-book ghp-import graphviz ipyniivue | |
| - name: Run all | |
| shell: bash | |
| working-directory: ./books | |
| run: | | |
| export PATH=$HOME/.local/bin:$PATH | |
| /bin/bash ../.github/workflows/write-toc-entry.sh | |
| sed -i 's/execute_notebooks: .*/execute_notebooks: off/' _config.yml | |
| cat _config.yml | |
| jb build . | |
| - name: Push to GitHub Pages | |
| uses: peaceiris/[email protected] | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| publish_dir: ./books/_build/html | |
| keep_files: true | |
| cname: www.neurodesk.org |