CI/CD data processing pipeline #1876
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD data processing pipeline | |
| on: | |
| push: | |
| paths: | |
| - "backend/src/main/resources/db/migration/**" | |
| - ".github/workflows/cicd-database.yml" | |
| - ".github/workflows/cicd-datapipeline.yml" | |
| - "datascience/**" | |
| - "infra/**" | |
| - "Makefile" | |
| schedule: | |
| - cron: "38 11 */3 * *" | |
| workflow_dispatch: | |
| jobs: | |
| build: | |
| name: Build & test docker image | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Get last release version | |
| id: lastrelease | |
| uses: pozetroninc/github-action-get-latest-release@master | |
| with: | |
| repository: mtes-mct/monitorenv | |
| - name: Set ENV_PROFILE as PROD when it is a release | |
| if: startsWith(github.ref, 'refs/tags/v1') || startsWith(github.ref, 'refs/heads/v1') || startsWith(github.ref, 'refs/tags/v2') || startsWith(github.ref, 'refs/heads/v2') | |
| run: echo "ENV_PROFILE=prod" >> $GITHUB_ENV | |
| - name: Set VERSION | |
| run: | | |
| if [ "${ENV_PROFILE}" != "prod" ]; then\ | |
| if [ "${{github.ref}}" == "refs/heads/main" ]; then\ | |
| export VERSION=${{ steps.lastrelease.outputs.release }}_snapshot | |
| else\ | |
| export REF_NAME=${{ github.ref_name }} | |
| export VERSION=${REF_NAME//\//_} | |
| fi | |
| else\ | |
| export VERSION=${{ steps.lastrelease.outputs.release }} | |
| fi | |
| echo $VERSION | |
| echo "VERSION=$VERSION" >> $GITHUB_ENV | |
| - name: Set up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@master | |
| - name: Cache Docker layers | |
| uses: actions/cache@v4 | |
| with: | |
| path: /tmp/.buildx-cache-pipeline | |
| key: ${{ runner.os }}-single-buildx-${{ github.sha }} | |
| restore-keys: | | |
| ${{ runner.os }}-single-buildx | |
| - name: Build image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| load: true | |
| builder: ${{ steps.buildx.outputs.name }} | |
| file: infra/docker/datapipeline/Dockerfile | |
| push: false | |
| tags: monitorenv-pipeline:${{ env.VERSION }} | |
| cache-from: type=local,src=/tmp/.buildx-cache-pipeline | |
| cache-to: type=local,dest=/tmp/.buildx-cache-pipeline-new | |
| # Temp fix | |
| # https://github.com/docker/build-push-action/issues/252 | |
| # https://github.com/moby/buildkit/issues/1896 | |
| - name: Move cache | |
| run: | | |
| rm -rf /tmp/.buildx-cache-pipeline | |
| mv /tmp/.buildx-cache-pipeline-new /tmp/.buildx-cache-pipeline | |
| - name: Set DOCKER_GROUP | |
| run: echo "DOCKER_GROUP=$(getent group docker | cut --delimiter=":" -f3)" >> $GITHUB_ENV | |
| - name: Test docker image | |
| run: make docker-test-pipeline | |
| - name: Push docker image to registry | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${GITHUB_ACTOR} --password-stdin | |
| make docker-tag-pipeline | |
| make docker-push-pipeline |