cache pip dependencies in CI; pin setuptool[_scm] dependencies to fix build; codecov action bump #357
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: tests+pypi | |
| defaults: | |
| run: | |
| shell: bash | |
| # arbitrarily selected runs | |
| env: | |
| animation_run: '3.10/ubuntu-latest/0/openmpi' | |
| codecov_run: '3.10/ubuntu-latest/1/openmpi' | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| branches: [ main ] | |
| schedule: | |
| - cron: '0 13 * * 4' | |
| release: | |
| types: [published] | |
| jobs: | |
| precommit: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up Python | |
| uses: actions/setup-python@v1 | |
| with: | |
| python-version: '3.10' | |
| - name: Linting | |
| run: | | |
| pip install pre-commit | |
| pre-commit clean | |
| pre-commit autoupdate | |
| pre-commit run --all-files | |
| pdoc: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.10' | |
| - uses: mpi4py/setup-mpi@v1 | |
| - name: Build | |
| run: | | |
| pip install pdoc3 | |
| pip install -e . | |
| python -We -m pdoc --html PyMPDATA_MPI | |
| - name: Deploy | |
| if: github.ref == 'refs/heads/main' | |
| uses: JamesIves/[email protected] | |
| with: | |
| BRANCH: pdoc | |
| FOLDER: html/PyMPDATA_MPI | |
| CLEAN: true | |
| zenodo_json: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: notiz-dev/github-action-json-property@release | |
| with: | |
| path: '.zenodo.json' | |
| prop_path: 'creators' | |
| pylint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: mpi4py/setup-mpi@v1 | |
| - name: Set up Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.10' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e .[tests] | |
| pip install pylint pytest matplotlib | |
| - name: Analysing the code with pylint | |
| run: | | |
| pylint --unsafe-load-any-extension=y --disable=fixme $(git ls-files '*.py') | |
| tests_setup: | |
| strategy: | |
| matrix: | |
| platform: [ubuntu-latest, macos-latest] | |
| mpi: [ 'mpich', 'openmpi', 'intelmpi'] | |
| python-version: ["3.10"] | |
| exclude: | |
| # as of time of writing, mpi4py/setup-mpi does not support it | |
| - platform: macos-latest | |
| mpi: intelmpi | |
| # issues with: *** The MPI_Comm_rank() function was called before MPI_INIT was invoked. | |
| - platform: ubuntu-latest | |
| mpi: intelmpi | |
| # https://github.com/Homebrew/homebrew-core/issues/26974 | |
| - platform: macos-latest | |
| mpi: mpich | |
| runs-on: ${{ matrix.platform }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-python@v1 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - run: echo pip_user_site=$(python -c "import sysconfig; print(sysconfig.get_path('purelib'))") >> $GITHUB_ENV | |
| - id: cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.pip_user_site }} | |
| key: ${{ matrix.platform }}-${{ matrix.mpi }}-${{ matrix.python-version }} | |
| - if: steps.cache.outputs.cache-hit != 'true' | |
| uses: mpi4py/setup-mpi@v1 | |
| with: | |
| mpi: ${{ matrix.mpi }} | |
| - if: steps.cache.outputs.cache-hit != 'true' && matrix.mpi == 'mpich' | |
| run: echo _ch="ch" >> $GITHUB_ENV | |
| - if: steps.cache.outputs.cache-hit != 'true' && startsWith(matrix.platform, 'ubuntu-') | |
| run: | | |
| sudo apt-get update && sudo apt-get install -y libhdf5-mpi$_ch-dev pkg-config | |
| lscpu | |
| - if: steps.cache.outputs.cache-hit != 'true' && startsWith(matrix.platform, 'ubuntu-') && matrix.mpi == 'mpich' | |
| run: | | |
| echo HDF5_LIBDIR=/usr/lib/x86_64-linux-gnu/hdf5/mpich >> $GITHUB_ENV | |
| echo HDF5_INCLUDEDIR=/usr/include/hdf5/mpich >> $GITHUB_ENV | |
| - if: steps.cache.outputs.cache-hit != 'true' && startsWith(matrix.platform, 'macos-') | |
| run: | | |
| brew install hdf5-mpi && echo HDF5_DIR=/opt/homebrew >> $GITHUB_ENV | |
| sysctl -a | grep cpu | grep hw | |
| - if: steps.cache.outputs.cache-hit != 'true' | |
| run: | | |
| HDF5_MPI="ON" CC=mpicc pip install --no-binary=h5py h5py==3.10.0 | |
| pip install -e .[tests] | |
| - run: pip show numpy | |
| - id: cache-save | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: ${{ env.pip_user_site }} | |
| key: ${{ matrix.platform }}-${{ matrix.mpi }}-${{ matrix.python-version }} | |
| tests: | |
| needs: [zenodo_json, pylint, pdoc, precommit, tests_setup] | |
| strategy: | |
| matrix: | |
| platform: [ubuntu-latest, macos-latest] | |
| mpi: [ 'mpich', 'openmpi', 'intelmpi'] | |
| python-version: ["3.10"] | |
| disable-jit: [1, 0] | |
| mpi-np: [1, 2, 3] | |
| exclude: | |
| # as of time of writing, mpi4py/setup-mpi does not support it | |
| - platform: macos-latest | |
| mpi: intelmpi | |
| # issues with: *** The MPI_Comm_rank() function was called before MPI_INIT was invoked. | |
| - platform: ubuntu-latest | |
| mpi: intelmpi | |
| # https://github.com/Homebrew/homebrew-core/issues/26974 | |
| - platform: macos-latest | |
| mpi: mpich | |
| fail-fast: false | |
| runs-on: ${{ matrix.platform }} | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-python@v1 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - run: echo pip_user_site=$(python -c "import sysconfig; print(sysconfig.get_path('purelib'))") >> $GITHUB_ENV | |
| - uses: mpi4py/setup-mpi@v1 | |
| with: | |
| mpi: ${{ matrix.mpi }} | |
| - if: matrix.mpi == 'mpich' | |
| run: echo _ch="ch" >> $GITHUB_ENV | |
| - if: startsWith(matrix.platform, 'ubuntu-') | |
| run: sudo apt-get update && sudo apt-get install -y libhdf5-mpi$_ch-dev pkg-config | |
| - if: startsWith(matrix.platform, 'macos-') | |
| run: brew install hdf5-mpi && echo HDF5_DIR=/opt/homebrew >> $GITHUB_ENV | |
| - if: startsWith(matrix.platform, 'ubuntu-') && matrix.mpi == 'mpich' | |
| run: | | |
| echo HDF5_LIBDIR=/usr/lib/x86_64-linux-gnu/hdf5/mpich >> $GITHUB_ENV | |
| echo HDF5_INCLUDEDIR=/usr/include/hdf5/mpich >> $GITHUB_ENV | |
| #- run: pip install -e .[tests] | |
| - id: cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ${{ env.pip_user_site }} | |
| key: ${{ matrix.platform }}-${{ matrix.mpi }}-${{ matrix.python-version }} | |
| - run: python -We -c "import PyMPDATA_MPI" | |
| - if: matrix.mpi == 'openmpi' | |
| run: echo _mpiexec_args="--oversubscribe" >> $GITHUB_ENV | |
| - name: "mpiexec pytest" | |
| env: | |
| NUMBA_DISABLE_JIT: ${{ matrix.disable-jit }} | |
| run: | | |
| pip show pytest | |
| if [ "$animation_run" == "${{ matrix.python-version }}/${{ matrix.platform }}/${{ matrix.disable-jit }}/${{ matrix.mpi }}" ]; then | |
| export CI_PLOTS_PATH=plots/$animation_run | |
| mkdir -p $CI_PLOTS_PATH | |
| fi | |
| if [ "${{ matrix.mpi-np }}" == "1" ] && [ "$codecov_run" == "${{ matrix.python-version }}/${{ matrix.platform }}/${{ matrix.disable-jit }}/${{ matrix.mpi }}" ]; then | |
| echo "CODECOV_RUN=1" >> $GITHUB_ENV | |
| export COV_ARGS="--cov=./ --cov-report=xml" | |
| pip install pytest-cov | |
| fi | |
| pytest -s -vv -We tests/local | |
| NUMBA_NUM_THREADS=3 mpiexec $_mpiexec_args -n ${{ matrix.mpi-np }} pytest $COV_ARGS --timeout=600 --timeout_method=thread -s -vv -We tests/local; | |
| - uses: actions/upload-artifact@v2 | |
| with: | |
| name: plots | |
| path: plots | |
| - if: env.CODECOV_RUN == '1' | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| fail_ci_if_error: true | |
| verbose: true | |
| package: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 # https://github.com/pypa/setuptools_scm/issues/480 | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: "3.10" | |
| - run: pip install twine build | |
| - run: | | |
| unset CI | |
| python -m build 2>&1 | tee build.log | |
| exit `fgrep -i warning build.log | wc -l` | |
| - run: twine check --strict dist/* | |
| - uses: actions/upload-artifact@v2 | |
| with: | |
| name: dist | |
| path: dist | |
| dist: | |
| runs-on: ubuntu-latest | |
| needs: [package,tests] | |
| steps: | |
| - uses: actions/download-artifact@v2 | |
| with: | |
| name: dist | |
| path: dist | |
| - if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| uses: pypa/gh-action-pypi-publish@unstable/v1 | |
| with: | |
| password: ${{ secrets.TEST_PYPI_API_TOKEN }} | |
| repository_url: https://test.pypi.org/legacy/ | |
| - if: startsWith(github.ref, 'refs/tags') | |
| uses: pypa/gh-action-pypi-publish@unstable/v1 | |
| with: | |
| password: ${{ secrets.PYPI_API_TOKEN }} | |
| devops_tests: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 # https://github.com/pypa/setuptools_scm/issues/480 | |
| - run: pip install -r tests/devops_tests/requirements.txt | |
| - env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: pytest --durations=10 -v -s -We -p no:unraisableexception tests/devops_tests | |
| tip-release-uploads: | |
| runs-on: ubuntu-latest | |
| needs: [tests, devops_tests] | |
| steps: | |
| - uses: actions/download-artifact@v2 | |
| with: | |
| name: plots | |
| path: plots | |
| - run: | | |
| sudo apt install -y librsvg2-bin | |
| for scenario in ./plots/${{ env.animation_run }}/*; do | |
| for test_plots in $scenario/*; do | |
| if [ ! $(find "$test_plots" -maxdepth 0 -empty | wc -l) -eq 1 ]; then | |
| for file in $test_plots/*.svg; do | |
| rsvg-convert $file -o tmp.png | |
| convert tmp.png -flatten -trim +repage -bordercolor white -border 10 $file.png | |
| done; | |
| rm tmp.png | |
| convert $test_plots/*.png plots/`basename $test_plots`-`basename $scenario`-anim.gif | |
| fi | |
| done; | |
| done; | |
| - uses: actions/upload-artifact@v2 | |
| with: | |
| name: anims | |
| path: plots/*-anim.gif | |
| - if: github.ref == 'refs/heads/main' | |
| uses: eine/tip@master | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| files: plots/*-anim.gif | |
| tag: 'latest-generated-plots' |