pipeline #2531
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: pipeline | |
| on: | |
| schedule: | |
| - cron: 0 0 * * * | |
| push: | |
| branches: | |
| - develop | |
| - feat/* | |
| - hotfix/* | |
| - main | |
| pull_request: | |
| branches: | |
| - develop | |
| - feat/* | |
| - hotfix/* | |
| - main | |
| env: | |
| CONTAINER_NAME: ${{ github.repository }} | |
| CONTAINER_REGISTRY_GHCR: ghcr.io | |
| CONTAINER_REGISTRY_DOCKER_HUB: docker.io | |
| # https://github.com/sigstore/cosign/releases | |
| COSIGN_VERSION: 2.4.1 | |
| # https://npmjs.com/package/@microsoft/sarif-multitool?activeTab=versions | |
| SARIF_MULTITOOL_VERSION: 4.5.4 | |
| # https://npmjs.com/package/snyk?activeTab=versions | |
| SNYK_VERSION: 1.1293.1 | |
| # https://github.com/microsoft/azure-pipelines-agent/releases | |
| AZP_AGENT_VERSION: 3.244.1 | |
| # https://github.com/PowerShell/PowerShell/releases | |
| POWERSHELL_VERSION: 7.2.23 | |
| # https://github.com/krallin/tini/releases | |
| TINI_VERSION: 0.19.0 | |
| # https://github.com/mikefarah/yq/releases | |
| YQ_VERSION: 4.44.3 | |
| # https://go.dev/dl | |
| GO_VERSION: 1.23.2 | |
| # https://github.com/rootless-containers/rootlesskit/releases | |
| ROOTLESSKIT_VERSION: 2.3.1 | |
| # https://github.com/moby/buildkit/releases | |
| BUILDKIT_VERSION: 0.16.0 | |
| # https://github.com/Azure/azure-cli/releases | |
| AZURE_CLI_VERSION: 2.65.0 | |
| # https://github.com/stedolan/jq/releases | |
| JQ_WIN_VERSION: 1.7.1 | |
| # https://github.com/aws/aws-cli/tags | |
| AWS_CLI_VERSION: 2.18.4 | |
| # https://console.cloud.google.com/artifacts/docker/google.com:cloudsdktool/us/gcr.io/google-cloud-cli | |
| # Note: To get thhe version number, spot the version tag on the latest pushed container | |
| GCLOUD_CLI_VERSION: 490.0.0 | |
| # https://github.com/git-for-windows/git/releases | |
| GIT_WIN_VERSION: 2.47.0 | |
| # https://github.com/facebook/zstd/releases | |
| ZSTD_WIN_VERSION: 1.5.6 | |
| # https://www.python.org/downloads | |
| PYTHON_VERSION_MAJOR_MINOR: 3.12 | |
| PYTHON_VERSION_PATCH: 7 | |
| # https://nodejs.org/en/download/releases | |
| NODE_VERSION: 20.18.0 | |
| # https://github.com/helm/helm/releases | |
| HELM_VERSION: 3.16.2 | |
| # https://github.com/oras-project/oras/releases | |
| ORAS_VERSION: 1.2.0 | |
| # https://github.com/docker/buildx/releases | |
| BUILDX_VERSION: 0.17.1 | |
| # https://github.com/hadolint/hadolint/releases | |
| HADOLINT_VERSION: 2.12.0 | |
| # https://learn.microsoft.com/en-us/visualstudio/releases/2022/release-history#fixed-version-bootstrappers | |
| VS_BUILDTOOLS_WIN_VERSION: 80c57218-b55f-4260-af46-a64ffd76e7a6/7fee719abc3ba9eced84ea258ccae39a7b0cc953b539c2ea3a98c3ff588b7870 | |
| # https://github.com/gohugoio/hugo/releases | |
| HUGO_VERSION: 0.135.0 | |
| # See: https://github.com/getsops/sops/releases | |
| SOPS_VERSION: 3.9.1 | |
| jobs: | |
| init: | |
| name: Initialize | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| BRANCH: ${{ steps.branch.outputs.branch }} | |
| VERSION_FULL: ${{ steps.version.outputs.version_full }} | |
| VERSION: ${{ steps.version.outputs.version }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| with: | |
| # We need all Git history for "version.sh" | |
| fetch-depth: 0 | |
| # Ensure "version.sh" submodule are up-to-date | |
| submodules: recursive | |
| - name: Version | |
| id: version | |
| run: | | |
| echo "version=$(bash cicd/version/version.sh -g . -c)" >> $GITHUB_OUTPUT | |
| echo "version_full=$(bash cicd/version/version.sh -g . -c -m)" >> $GITHUB_OUTPUT | |
| - name: Branch | |
| id: branch | |
| run: | | |
| branch=$(echo "${{ github.ref_name }}" | sed 's/\//-/g') | |
| echo "branch=$branch" >> $GITHUB_OUTPUT | |
| sast-creds: | |
| name: SAST - Credentials | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| with: | |
| # We need all Git history for testing credentials | |
| fetch-depth: 0 | |
| # Ensure all submodules up-to-date | |
| submodules: recursive | |
| - name: SAST - Credentials | |
| uses: trufflesecurity/[email protected] | |
| with: | |
| base: main | |
| extra_args: --only-verified | |
| head: HEAD~1 | |
| build-helm: | |
| name: Build Helm chart | |
| needs: | |
| - init | |
| - sast-creds | |
| - sast-semgrep | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| # Required for running "helm" CLI | |
| - name: Setup Helm | |
| uses: azure/[email protected] | |
| with: | |
| version: v${{ env.HELM_VERSION }} | |
| # Required for running "npx" CLI | |
| - name: Setup Node | |
| uses: actions/[email protected] | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| # Required for running "cosign" CLI | |
| - name: Setup Cosign | |
| # Only sign builds on main branch | |
| if: github.ref == 'refs/heads/main' | |
| uses: sigstore/[email protected] | |
| with: | |
| cosign-release: v${{ env.COSIGN_VERSION }} | |
| - name: Prepare GPG key | |
| run: | | |
| echo "${{ secrets.GPG_KEYRING }}" | gpg --dearmor > keyring.gpg | |
| - name: Package Helm chart | |
| run: | | |
| cp README.md src/helm/blue-agent/ | |
| helm package \ | |
| --app-version ${{ env.AZP_AGENT_VERSION }} \ | |
| --destination .cr-release-packages \ | |
| --key 'Clémence Lesné' \ | |
| --keyring keyring.gpg \ | |
| --sign \ | |
| --version ${{ needs.init.outputs.VERSION }} \ | |
| src/helm/blue-agent | |
| - name: Sign Helm chart | |
| # Only sign builds on main branch | |
| if: github.ref == 'refs/heads/main' | |
| env: | |
| COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }} | |
| COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} | |
| run: | | |
| cosign sign-blob \ | |
| --bundle .cr-release-packages/blue-agent-${{ needs.init.outputs.VERSION }}.tgz.bundle \ | |
| --key="env://COSIGN_PRIVATE_KEY" \ | |
| --yes \ | |
| .cr-release-packages/blue-agent-${{ needs.init.outputs.VERSION }}.tgz | |
| - name: Upload Helm chart | |
| uses: actions/[email protected] | |
| with: | |
| if-no-files-found: error # Fail if no files are uploaded | |
| include-hidden-files: true # Folder begins with a dot, if not checked the whole folder is ignored | |
| name: helm-chart | |
| path: .cr-release-packages | |
| - name: Render Helm chart locally | |
| run: | | |
| helm template \ | |
| --output-dir .helm-template \ | |
| --values test/helm/blue-agent/values.yaml \ | |
| .cr-release-packages/blue-agent-${{ needs.init.outputs.VERSION }}.tgz | |
| - name: Run SAST Snyk for Helm | |
| # Snyk can be used to break the build when it detects security issues. In this case we want to upload the issues to GitHub Security | |
| continue-on-error: true | |
| env: | |
| SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} | |
| run: | | |
| npx --yes snyk@${{ env.SNYK_VERSION }} iac test \ | |
| --sarif-file-output=snyk.sarif \ | |
| --severity-threshold=medium \ | |
| .helm-template | |
| # Fix issue "Error: Code Scanning could not process the submitted SARIF file: rejecting SARIF, as there are more runs than allowed (XX > 20)" | |
| # See: https://github.com/github/codeql-action/issues/220 | |
| - name: Merge SARIF files | |
| env: | |
| # See: https://github.com/dotnet/core/issues/2186#issuecomment-1935707348 | |
| DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: "1" | |
| run: | | |
| npx --yes @microsoft/sarif-multitool@${{ env.SARIF_MULTITOOL_VERSION }} merge \ | |
| --automation-id ${{ github.run_id }} \ | |
| --merge-runs \ | |
| --output-file merged.sarif \ | |
| snyk.sarif | |
| - name: Upload results to GitHub Security | |
| uses: github/codeql-action/[email protected] | |
| continue-on-error: true | |
| with: | |
| sarif_file: merged.sarif | |
| release-helm: | |
| name: Release Helm chart | |
| # Only deploy on non-scheduled main branch, as there is only one Helm repo and we cannot override an existing version | |
| if: (github.event_name != 'schedule') && (github.ref == 'refs/heads/main') | |
| needs: | |
| - build-helm | |
| - build-release-linux | |
| - build-release-win | |
| - static-test | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| with: | |
| # Chart Releaser needs to have local access to "gh-pages" plus current branch | |
| fetch-depth: 0 | |
| - name: Configure Git | |
| run: | | |
| git config user.name "${{ github.actor }}" | |
| git config user.email "${{ github.actor }}@users.noreply.github.com" | |
| - name: Download Helm chart | |
| uses: actions/[email protected] | |
| with: | |
| name: helm-chart | |
| path: .cr-release-packages | |
| - name: Archive Helm chart | |
| uses: helm/[email protected] | |
| with: | |
| charts_dir: src/helm | |
| skip_packaging: true | |
| env: | |
| CR_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| CR_SKIP_EXISTING: true # Avoid overriding existing files, compat with the Hugo static site | |
| static-test: | |
| name: Static test | |
| permissions: | |
| contents: read | |
| id-token: write | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| # Required for running "npx" CLI | |
| - name: Setup Node | |
| uses: actions/[email protected] | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: Setup Hadolint | |
| run: | | |
| sudo curl -LsSf --retry 8 --retry-all-errors https://github.com/hadolint/hadolint/releases/download/v${{ env.HADOLINT_VERSION }}/hadolint-Linux-x86_64 -o /usr/bin/hadolint | |
| sudo chmod +x /usr/bin/hadolint | |
| hadolint --version | |
| - name: Login to Azure | |
| uses: azure/[email protected] | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| - name: Run tests | |
| run: | | |
| make test | |
| build-release-linux: | |
| name: Build & release image (Linux ${{ matrix.os }}) | |
| needs: | |
| - init | |
| - sast-creds | |
| - sast-semgrep | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - os: bookworm | |
| arch: linux/amd64,linux/arm64 | |
| - os: bullseye | |
| arch: linux/amd64,linux/arm64 | |
| - os: focal | |
| arch: linux/amd64,linux/arm64 | |
| - os: jammy | |
| arch: linux/amd64,linux/arm64 | |
| - os: noble | |
| # On GitHub Actions, build is too slow to be done on ARM64 virtialized on QEMU | |
| # TODO: Re-enable ARM64 when GitHub Actions will provide native ARM64 runners | |
| arch: linux/amd64 | |
| - os: ubi8 | |
| arch: linux/amd64,linux/arm64 | |
| - os: ubi9 | |
| arch: linux/amd64,linux/arm64 | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| # Container build take a lot of disk space, there is no enough space on the runner to have both tools cache and container build | |
| - name: Clean up disk | |
| run: rm -rf /opt/hostedtoolcache | |
| # Required to build multi-arch images | |
| - name: Setup QEMU | |
| id: setup-qemu | |
| uses: docker/[email protected] | |
| with: | |
| platforms: ${{ matrix.arch }} | |
| # Required for "docker build" command | |
| - name: Setup Docker Buildx | |
| uses: docker/[email protected] | |
| with: | |
| version: v${{ env.BUILDX_VERSION }} | |
| driver-opts: | | |
| image=moby/buildkit:v${{ env.BUILDKIT_VERSION }} | |
| # Required for running "npx" CLI | |
| - name: Setup Node | |
| uses: actions/[email protected] | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| # Required for running "cosign" CLI | |
| - name: Setup Cosign | |
| # Only sign builds on main branch | |
| if: github.ref == 'refs/heads/main' | |
| uses: sigstore/[email protected] | |
| with: | |
| cosign-release: v${{ env.COSIGN_VERSION }} | |
| - name: Login to registry - GitHub | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.CONTAINER_REGISTRY_GHCR }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Login to registry - Docker Hub | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.CONTAINER_REGISTRY_DOCKER_HUB }} | |
| username: clemlesne | |
| password: ${{ secrets.DOCKER_HUB_PAT }} | |
| - name: Check if pre-release | |
| id: prerelease | |
| run: | | |
| if [ "${{ github.ref_name }}" == "main" ]; then | |
| echo "prerelease=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "prerelease=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Container meta | |
| id: meta | |
| uses: docker/[email protected] | |
| with: | |
| images: | | |
| ${{ env.CONTAINER_REGISTRY_GHCR }}/${{ env.CONTAINER_NAME }} | |
| ${{ env.CONTAINER_REGISTRY_DOCKER_HUB }}/${{ env.CONTAINER_NAME }} | |
| flavor: | | |
| prefix=${{ matrix.os }}- | |
| tags: | | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=ref,event=branch | |
| type=ref,event=pr | |
| type=schedule | |
| type=schedule,pattern={{date 'YYYYMMDD'}} | |
| type=semver,pattern={{version}},value=${{ needs.init.outputs.VERSION_FULL }} | |
| type=sha,prefix=${{ matrix.os }}-sha- | |
| labels: | | |
| io.artifacthub.package.category=integration-delivery | |
| io.artifacthub.package.keywords=agent,azure,azure-devops,azure-pipelines,container,devops,docker,helm,kubernetes,pipelines,self-hosted,self-hosted-agent,auto-scale,keda | |
| io.artifacthub.package.license=Apache-2.0 | |
| io.artifacthub.package.logo-url=https://raw.githubusercontent.com/${{ env.CONTAINER_NAME }}/${{ github.sha }}/docs/static/favicon.svg | |
| io.artifacthub.package.maintainers=[{"name":"${{ github.actor }}","email":"${{ github.actor }}@users.noreply.github.com"}] | |
| io.artifacthub.package.prerelease=${{ steps.prerelease.outputs.prerelease }} | |
| io.artifacthub.package.readme-url=https://raw.githubusercontent.com/${{ env.CONTAINER_NAME }}/${{ github.sha }}/README.md | |
| org.opencontainers.image.documentation=https://github.com/${{ env.CONTAINER_NAME }} | |
| org.opencontainers.image.vendor=${{ github.actor }} | |
| - name: Store tag | |
| id: tag | |
| run: | | |
| ref="${{ github.ref }}" | |
| branch=$(echo "${ref#refs/heads/}" | sed 's/\//-/g') | |
| tag=$(echo "${{ steps.meta.outputs.tags }}" | grep -m1 $branch) | |
| echo "tag=$tag" >> $GITHUB_OUTPUT | |
| - name: Build & push container | |
| uses: docker/[email protected] | |
| with: | |
| build-args: | | |
| AWS_CLI_VERSION=${{ env.AWS_CLI_VERSION }} | |
| AZP_AGENT_VERSION=${{ env.AZP_AGENT_VERSION }} | |
| AZURE_CLI_VERSION=${{ env.AZURE_CLI_VERSION }} | |
| BUILDKIT_VERSION=${{ env.BUILDKIT_VERSION }} | |
| GCLOUD_CLI_VERSION=${{ env.GCLOUD_CLI_VERSION }} | |
| GO_VERSION=${{ env.GO_VERSION }} | |
| POWERSHELL_VERSION=${{ env.POWERSHELL_VERSION }} | |
| PYTHON_VERSION_MAJOR_MINOR=${{ env.PYTHON_VERSION_MAJOR_MINOR }} | |
| PYTHON_VERSION_PATCH=${{ env.PYTHON_VERSION_PATCH }} | |
| ROOTLESSKIT_VERSION=${{ env.ROOTLESSKIT_VERSION }} | |
| TINI_VERSION=${{ env.TINI_VERSION }} | |
| YQ_VERSION=${{ env.YQ_VERSION }} | |
| cache-from: | | |
| type=gha,scope=buildkit-${{ matrix.os }}-${{ github.ref_name }} | |
| type=gha,scope=buildkit-${{ matrix.os }}-develop | |
| type=gha,scope=buildkit-${{ matrix.os }}-main | |
| cache-to: type=gha,scope=buildkit-${{ matrix.os }}-${{ github.ref_name }} | |
| context: src/docker | |
| file: src/docker/Dockerfile-${{ matrix.os }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| outputs: type=registry,oci-mediatypes=true,compression=estargz,compression-level=9,force-compression=true | |
| platforms: ${{ matrix.arch }} | |
| provenance: true | |
| sbom: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| # Cosign is voluntarily retried indefinitely to avoid breaking the build when the container registry is slow or throttle, with an exponential backoff delay with jitter | |
| # See: https://github.com/clemlesne/blue-agent/issues/264 | |
| - name: Sign containers | |
| # Only sign builds on main branch | |
| if: github.ref == 'refs/heads/main' | |
| env: | |
| COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }} | |
| COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} | |
| run: | | |
| while IFS= read -r tag; do | |
| echo "Signing $tag" | |
| i=1 | |
| while true; do | |
| cosign sign \ | |
| --key="env://COSIGN_PRIVATE_KEY" \ | |
| --recursive \ | |
| --yes \ | |
| $tag \ | |
| && break | |
| jitter=$(python3 -c "import random; print(random.randint(-20, 20))") | |
| backoff=$(python3 -c "import math; print(int(math.pow(3, $i) * (1 + $jitter / 100)))") | |
| echo "retry: cosign returned $?, backing off for $backoff seconds and trying again ($i)..." | |
| sleep $backoff | |
| i=$((i + 1)) | |
| done | |
| done <<< "${{ steps.meta.outputs.tags }}" | |
| - name: Run SAST Snyk against containers | |
| # Snyk can be used to break the build when it detects security issues. In this case we want to upload the issues to GitHub Security | |
| continue-on-error: true | |
| env: | |
| SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} | |
| run: | | |
| for arch in $(echo ${{ matrix.arch }} | tr "," "\n"); do | |
| echo "Running Snyk for $arch" | |
| npx --yes snyk@${{ env.SNYK_VERSION }} container test \ | |
| --architecture=$arch \ | |
| --fail-on=upgradable \ | |
| --file=src/docker/Dockerfile-${{ matrix.os }} \ | |
| --sarif-file-output=snyk-$(echo $arch | sed -e 's#/#-#g').sarif \ | |
| --severity-threshold=medium \ | |
| ${{ steps.tag.outputs.tag }} | |
| done | |
| # Fix issue "Error: Code Scanning could not process the submitted SARIF file: rejecting SARIF, as there are more runs than allowed (XX > 20)" | |
| # See: https://github.com/github/codeql-action/issues/220 | |
| - name: Merge SARIF files | |
| env: | |
| # See: https://github.com/dotnet/core/issues/2186#issuecomment-1935707348 | |
| DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: "1" | |
| run: | | |
| npx --yes @microsoft/sarif-multitool@${{ env.SARIF_MULTITOOL_VERSION }} merge \ | |
| --automation-id ${{ github.run_id }} \ | |
| --merge-runs \ | |
| --output-file merged.sarif \ | |
| *.sarif | |
| - name: Upload results to GitHub Security | |
| uses: github/codeql-action/[email protected] | |
| continue-on-error: true | |
| with: | |
| sarif_file: merged.sarif | |
| build-release-win: | |
| name: Build & release image (Windows ${{ matrix.os }}) | |
| needs: | |
| - init | |
| - sast-creds | |
| - sast-semgrep | |
| runs-on: ${{ matrix.runs-on }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - os: win-ltsc2022 | |
| runs-on: windows-2022 | |
| - os: win-ltsc2019 | |
| runs-on: windows-2019 | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| # Container build take a lot of disk space, there is no enough space on the runner to have both tools cache and container build | |
| - name: Clean up disk | |
| run: Remove-Item -Path C:\hostedtoolcache -Recurse -Force | |
| # Required for running "npx" CLI | |
| - name: Setup Node | |
| uses: actions/[email protected] | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| # Required for running "cosign" CLI | |
| - name: Setup Cosign | |
| # Only sign builds on main branch | |
| if: github.ref == 'refs/heads/main' | |
| uses: sigstore/[email protected] | |
| with: | |
| cosign-release: v${{ env.COSIGN_VERSION }} | |
| - name: Login to registry - GitHub | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.CONTAINER_REGISTRY_GHCR }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Login to registry - Docker Hub | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.CONTAINER_REGISTRY_DOCKER_HUB }} | |
| username: clemlesne | |
| password: ${{ secrets.DOCKER_HUB_PAT }} | |
| - name: Check if pre-release | |
| id: prerelease | |
| run: | | |
| if ('${{ github.ref_name }}' -eq 'main') { | |
| echo "prerelease=false" >> $env:GITHUB_OUTPUT | |
| } else { | |
| echo "prerelease=true" >> $env:GITHUB_OUTPUT | |
| } | |
| - name: Container meta | |
| id: meta | |
| uses: docker/[email protected] | |
| with: | |
| images: | | |
| ${{ env.CONTAINER_REGISTRY_GHCR }}/${{ env.CONTAINER_NAME }} | |
| ${{ env.CONTAINER_REGISTRY_DOCKER_HUB }}/${{ env.CONTAINER_NAME }} | |
| flavor: | | |
| prefix=${{ matrix.os }}- | |
| tags: | | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=ref,event=branch | |
| type=ref,event=pr | |
| type=schedule | |
| type=schedule,pattern={{date 'YYYYMMDD'}} | |
| type=semver,pattern={{version}},value=${{ needs.init.outputs.VERSION_FULL }} | |
| type=sha,prefix=${{ matrix.os }}-sha- | |
| labels: | | |
| io.artifacthub.package.category=integration-delivery | |
| io.artifacthub.package.keywords=agent,azure,azure-devops,azure-pipelines,container,devops,docker,helm,kubernetes,pipelines,self-hosted,self-hosted-agent,auto-scale,keda | |
| io.artifacthub.package.license=Apache-2.0 | |
| io.artifacthub.package.logo-url=https://raw.githubusercontent.com/${{ env.CONTAINER_NAME }}/${{ github.sha }}/docs/static/favicon.svg | |
| io.artifacthub.package.maintainers=[{"name":"${{ github.actor }}","email":"${{ github.actor }}@users.noreply.github.com"}] | |
| io.artifacthub.package.prerelease=${{ steps.prerelease.outputs.prerelease }} | |
| io.artifacthub.package.readme-url=https://raw.githubusercontent.com/${{ env.CONTAINER_NAME }}/${{ github.sha }}/README.md | |
| org.opencontainers.image.documentation=https://github.com/${{ env.CONTAINER_NAME }} | |
| org.opencontainers.image.vendor=${{ github.actor }} | |
| - name: Store tag | |
| id: tag | |
| run: | | |
| $branch = "${{ github.ref }}".Substring("refs/heads/".Length).Replace("/", "-") | |
| $tag = ('${{ steps.meta.outputs.tags }}').Split([Environment]::NewLine) | Where-Object { $_ -like "*$branch*" } | Select-Object -First 1 | |
| echo "tag=$tag" >> $Env:GITHUB_OUTPUT | |
| - name: Build & push container | |
| run: | | |
| $params = @( | |
| # Required build arguments | |
| "--build-arg", "AWS_CLI_VERSION=${{ env.AWS_CLI_VERSION }}", | |
| "--build-arg", "AZP_AGENT_VERSION=${{ env.AZP_AGENT_VERSION }}", | |
| "--build-arg", "AZURE_CLI_VERSION=${{ env.AZURE_CLI_VERSION }}", | |
| "--build-arg", "GCLOUD_CLI_VERSION=${{ env.GCLOUD_CLI_VERSION }}", | |
| "--build-arg", "GIT_VERSION=${{ env.GIT_WIN_VERSION }}", | |
| "--build-arg", "JQ_VERSION=${{ env.JQ_WIN_VERSION }}", | |
| "--build-arg", "POWERSHELL_VERSION=${{ env.POWERSHELL_VERSION }}", | |
| "--build-arg", "PYTHON_VERSION_MAJOR_MINOR=${{ env.PYTHON_VERSION_MAJOR_MINOR }}", | |
| "--build-arg", "PYTHON_VERSION_PATCH=${{ env.PYTHON_VERSION_PATCH }}", | |
| "--build-arg", "VS_BUILDTOOLS_VERSION=${{ env.VS_BUILDTOOLS_WIN_VERSION }}", | |
| "--build-arg", "YQ_VERSION=${{ env.YQ_VERSION }}", | |
| "--build-arg", "ZSTD_VERSION=${{ env.ZSTD_WIN_VERSION }}", | |
| "--file", "src/docker/Dockerfile-${{ matrix.os }}" | |
| ) | |
| $tags = ('${{ steps.meta.outputs.tags }}').Split([Environment]::NewLine) | |
| foreach ($tag in $tags) { | |
| $params += "--tag", $tag | |
| } | |
| $labels = ('${{ steps.meta.outputs.labels }}').Split([Environment]::NewLine) | |
| foreach ($label in $labels) { | |
| $params += "--label", $label | |
| } | |
| Write-Host "Build arguments:" | |
| $params | ForEach-Object -Begin { $i = 0 } -Process { | |
| if ($i % 2 -eq 0) { | |
| Write-Host -NoNewline "`n $_" | |
| } else { | |
| Write-Host -NoNewline " $_" | |
| } | |
| $i++ | |
| } | |
| Write-Host | |
| Write-Host "Building" | |
| docker build @params src\docker | |
| Write-Host "Pushing" | |
| foreach ($tag in $tags) { | |
| docker push --quiet $tag | |
| } | |
| # Cosign is voluntarily retried indefinitely to avoid breaking the build when the container registry is slow or throttle, with an exponential backoff delay with jitter | |
| # See: https://github.com/clemlesne/blue-agent/issues/264 | |
| - name: Sign containers | |
| # Only sign builds on main branch | |
| if: github.ref == 'refs/heads/main' | |
| env: | |
| COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }} | |
| COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} | |
| run: | | |
| $tags = ('${{ steps.meta.outputs.tags }}').Split([Environment]::NewLine) | |
| foreach ($tag in $tags) { | |
| Write-Host "Signing $tag" | |
| $i = 1 | |
| while ($true) { | |
| cosign sign ` | |
| --key="env://COSIGN_PRIVATE_KEY" ` | |
| --recursive ` | |
| --yes ` | |
| $tag | |
| if ($?) { | |
| break | |
| } | |
| $jitter = (Get-Random -Minimum -20 -Maximum 21) / 100 | |
| $backoff = [math]::Round([math]::Pow(3, $i) * (1 + $jitter)) | |
| Write-Host "retry: cosign returned $LASTEXITCODE, backing off for $backoff seconds and trying again ($i)..." | |
| Start-Sleep -Seconds $backoff | |
| $i++ | |
| } | |
| } | |
| - name: Run SAST Snyk against containers | |
| # Snyk can be used to break the build when it detects security issues. In this case we want to upload the issues to GitHub Security | |
| continue-on-error: true | |
| env: | |
| SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} | |
| run: | | |
| npx --yes snyk@${{ env.SNYK_VERSION }} container test ` | |
| --fail-on=upgradable ` | |
| --file=src/docker/Dockerfile-${{ matrix.os }} ` | |
| --sarif-file-output=snyk.sarif ` | |
| --severity-threshold=medium ` | |
| ${{ steps.tag.outputs.tag }} | |
| # Fix issue "Error: Code Scanning could not process the submitted SARIF file: rejecting SARIF, as there are more runs than allowed (XX > 20)" | |
| # See: https://github.com/github/codeql-action/issues/220 | |
| - name: Merge SARIF files | |
| run: | | |
| npx --yes @microsoft/sarif-multitool@${{ env.SARIF_MULTITOOL_VERSION }} merge ` | |
| --automation-id ${{ github.run_id }} ` | |
| --merge-runs ` | |
| --output-file merged.sarif ` | |
| snyk.sarif | |
| - name: Upload results to GitHub Security | |
| uses: github/codeql-action/[email protected] | |
| continue-on-error: true | |
| with: | |
| sarif_file: merged.sarif | |
| sast-semgrep: | |
| name: SAST - Semgrep | |
| runs-on: ubuntu-24.04 | |
| container: | |
| image: returntocorp/semgrep | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Run tests | |
| # Semgrep can be used to break the build when it detects security issues. In this case we want to upload the issues to GitHub Security | |
| continue-on-error: true | |
| env: | |
| SEMGREP_RULES: p/cwe-top-25 p/owasp-top-ten p/kubernetes p/dockerfile | |
| run: semgrep ci --sarif --output=semgrep.sarif | |
| - name: Upload results to GitHub Security | |
| uses: github/codeql-action/[email protected] | |
| continue-on-error: true | |
| with: | |
| sarif_file: semgrep.sarif | |
| deploy-artifacthub-metadata: | |
| name: Deploy ArtifactHub metadata | |
| # Only deploy on main branch, as we don't want to break verification with a bad metadata file during development | |
| if: github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| # Required for running "oras" CLI | |
| - name: Setup ORAS | |
| uses: oras-project/[email protected] | |
| with: | |
| version: ${{ env.ORAS_VERSION }} | |
| - name: Login to registry - GitHub | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.CONTAINER_REGISTRY_GHCR }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Login to registry - Docker Hub | |
| uses: docker/[email protected] | |
| with: | |
| registry: ${{ env.CONTAINER_REGISTRY_DOCKER_HUB }} | |
| username: clemlesne | |
| password: ${{ secrets.DOCKER_HUB_PAT }} | |
| - name: Push to registry | |
| run: | | |
| oras push \ | |
| ${{ env.CONTAINER_REGISTRY_GHCR }}/${{ env.CONTAINER_NAME }}:artifacthub.io \ | |
| artifacthub-repo.yml:application/vnd.cncf.artifacthub.repository-metadata.layer.v1.yaml \ | |
| --config /dev/null:application/vnd.cncf.artifacthub.config.v1+yaml | |
| oras push \ | |
| ${{ env.CONTAINER_REGISTRY_DOCKER_HUB }}/${{ env.CONTAINER_NAME }}:artifacthub.io \ | |
| artifacthub-repo.yml:application/vnd.cncf.artifacthub.repository-metadata.layer.v1.yaml \ | |
| --config /dev/null:application/vnd.cncf.artifacthub.config.v1+yaml | |
| update-docker-hub-description: | |
| name: Update Docker Hub description | |
| # Only deploy on non-scheduled main branch, as there is only one Helm repo and we cannot override an existing version | |
| if: (github.event_name != 'schedule') && (github.ref == 'refs/heads/main') | |
| needs: | |
| - build-release-linux | |
| - build-release-win | |
| - static-test | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Push README to Docker Hub | |
| uses: peter-evans/[email protected] | |
| with: | |
| enable-url-completion: true | |
| password: ${{ secrets.DOCKER_HUB_PAT }} | |
| repository: ${{ env.CONTAINER_NAME }} | |
| short-description: ${{ github.event.repository.description }} | |
| username: clemlesne | |
| build-hugo: | |
| name: Build Hugo site | |
| needs: | |
| - sast-creds | |
| - sast-semgrep | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| # Required for running "hugo" CLI | |
| - name: Setup Hugo CLI | |
| run: | | |
| wget -O ${{ runner.temp }}/hugo.deb https://github.com/gohugoio/hugo/releases/download/v${{ env.HUGO_VERSION }}/hugo_extended_${{ env.HUGO_VERSION }}_linux-amd64.deb | |
| sudo dpkg -i ${{ runner.temp }}/hugo.deb | |
| # Required as a Hugo build dependency | |
| - name: Setup Dart Sass | |
| run: sudo snap install dart-sass | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - name: Setup Pages | |
| id: pages | |
| uses: actions/[email protected] | |
| - name: Build with Hugo | |
| working-directory: docs | |
| env: | |
| # For maximum backward compatibility with Hugo modules | |
| HUGO_ENVIRONMENT: production | |
| HUGO_ENV: production | |
| run: | | |
| hugo \ | |
| --gc \ | |
| --minify \ | |
| --baseURL "${{ steps.pages.outputs.base_url }}/" | |
| - name: Upload build artifact | |
| uses: actions/[email protected] | |
| with: | |
| if-no-files-found: error # Fail if no files are uploaded | |
| name: hugo | |
| path: docs/public | |
| deploy-hugo: | |
| name: Deploy Hugo site | |
| needs: | |
| - build-hugo | |
| - init | |
| - static-test | |
| # Only deploy on non-scheduled main branch, as there is only one Helm repo and we cannot override an existing version | |
| if: (github.event_name != 'schedule') && (github.ref == 'refs/heads/main') | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Pull from gh-pages | |
| uses: actions/[email protected] | |
| with: | |
| ref: gh-pages | |
| - name: Clean previous Hugo build | |
| run: | | |
| # Clean all except Helm index | |
| find . -type f -not -name "index.yaml" -not -path "*/.git/*" -delete | |
| - name: Download build artifact | |
| uses: actions/[email protected] | |
| with: | |
| name: hugo | |
| - name: Commit and push | |
| run: | | |
| # Configure git | |
| git config --global user.name "${{ github.actor }}" | |
| git config --global user.email "${{ github.actor }}@users.noreply.github.com" | |
| # Commit and push | |
| if [[ -z "$(git status --porcelain)" ]]; then | |
| echo "Nothing to commit" | |
| else | |
| git add . | |
| git commit -m "Deploy Hugo site v${{ needs.init.outputs.VERSION }}" | |
| git push origin gh-pages | |
| fi | |
| integration-test-linux: | |
| name: Integration test (Linux ${{ matrix.os }}) | |
| permissions: | |
| contents: read | |
| id-token: write | |
| needs: | |
| - build-release-linux | |
| - init | |
| - static-test | |
| runs-on: ubuntu-24.04 | |
| concurrency: integration-test-linux-${{ needs.init.outputs.BRANCH }}-${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| # Rate limiting on Azure DevOps SaaS APIs is triggered quickly by integration tests, so we need to limit the number of parallel jobs | |
| max-parallel: 3 | |
| matrix: | |
| os: [bookworm, bullseye, focal, jammy, noble, ubi8, ubi9] | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| # Required for running "sops" CLI | |
| - name: Setup SOPS | |
| run: | | |
| curl -LO https://github.com/getsops/sops/releases/download/v${{ env.SOPS_VERSION }}/sops-v${{ env.SOPS_VERSION }}.linux.amd64 | |
| mv sops-v${{ env.SOPS_VERSION }}.linux.amd64 /usr/local/bin/sops | |
| chmod +x /usr/local/bin/sops | |
| # Configure local configuration encryption key | |
| - name: Setup AGE key | |
| run: | | |
| age_folder="$XDG_CONFIG_HOME/sops/age" | |
| mkdir -p ${age_folder} | |
| echo "${{ secrets.AGE_KEY }}" > ${age_folder}/keys.txt | |
| - name: Login to Azure | |
| uses: azure/[email protected] | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| - name: Deploy Bicep | |
| run: | | |
| make deploy-bicep \ | |
| flavor="${{ matrix.os }}" \ | |
| prefix="${{ needs.init.outputs.BRANCH }}" \ | |
| version="sha-$(git rev-parse --short HEAD)" | |
| - name: Integration | |
| env: | |
| # Permissions: agent pools (read & manage); build (read & execute); pipeline resources (use & manage); project and team (read, write, & manage); service connections (read, query, & manage) | |
| # Recommended group membership: Project Collection Build Service Accounts | |
| AZURE_DEVOPS_EXT_PAT: ${{ secrets.AZURE_DEVOPS_PAT }} | |
| # Scope: clemlesne/blue-agent | |
| # Permissions: contents (read-only); metadata (read-only); webhooks (read & write) | |
| AZURE_DEVOPS_EXT_GITHUB_PAT: ${{ secrets.AZURE_DEVOPS_GITHUB_PAT }} | |
| # Script wait indefinitely for external events, so we need to timeout | |
| timeout-minutes: 30 | |
| run: | | |
| make integration-run \ | |
| flavor="${{ matrix.os }}" \ | |
| prefix="${{ needs.init.outputs.BRANCH }}" \ | |
| version="sha-$(git rev-parse --short HEAD)" | |
| - name: Cleanup | |
| if: always() | |
| env: | |
| # Permissions: agent pools (read & manage); build (read & execute); pipeline resources (use & manage); project and team (read, write, & manage); service connections (read, query, & manage) | |
| # Recommended group membership: Project Collection Build Service Accounts | |
| AZURE_DEVOPS_EXT_PAT: ${{ secrets.AZURE_DEVOPS_PAT }} | |
| run: | | |
| make integration-cleanup \ | |
| flavor="${{ matrix.os }}" \ | |
| prefix="${{ needs.init.outputs.BRANCH }}" | |
| # make destroy-bicep \ | |
| # flavor="${{ matrix.os }}" \ | |
| # prefix="${{ needs.init.outputs.BRANCH }}" |