Add CI to build mulled containers with Wave (#4080)

* build: Add wave * build: Set strategy to dockerfile, conda then container * refactor: Remove container * build: Add a repo to push to * ci(wave): Add wave build https://github.com/nodejs/docker-node/blob/3c4fa6daf06a4786d202f2f610351837806a0380/.github/workflows/build-test.yml#L29 * ci(wave): Switch to all_changed_files * ci(wave): Only look for envronment.ymls * dummy: Change env * ci(wave): Remove raw format * ci(wave): Try a bunch of different things at once * ci(wave): Remove redundant fromJson and wrap in an array * ci(wave): I have no idea what I'm doing * ci(wave): Wrap it * ci(wave): Found an example https://github.com/tj-actions/changed-files/blob/main/.github/workflows/matrix-test.yml * ci(wave): Maybe quotes? * ci(wave): That'll do it * ci(wave): Fix wave install * ci(wave): Hard code an image * ci(wave): Add secrets * feat: Try a different files structure * ci(wave): First stab at building singularity images * fixup! feat: Try a different files structure * ci(wave): Add profile to matrix * ci(wave): Give up on fancy substitution * ci(wave): Add await Co-authored-by: ewels <[email protected]> * ci(wave): Switch to quay * test(wave): Add freeze and update build repo * refactor(wave): What happens if I add a container? * refactor(wave): Have both bowtie modules use the same env For the sake of demonstration * test: Cut out using wave on tests * refactor: What happens if we use the singularity one? * refactor: Keep container directives for offline download seqeralabs/wave#323 * feat: Try new singularity OCI setting nextflow-io/nextflow@f5362a7 * build: Update container name Guess #4327 broke that * chore: Bump wave-cli version * ci: Install runc * ci: Switch to singularityhub action nextflow-io/nextflow#4543 * ci: Install new singularity manually Why that action trys to build from source, idk. * ci: Install dependancies for singularity * ci: runc => crun * ci: Fix cgroup error https://blog.misharov.pro/2021-05-16/systemd-github-actions * ci: That'll do it * ci: Remove Dockerfile We'll have a seperate action for this I think * ci: Update name * ci: Push to the correct repos * ci: Remove OCI stuff * ci: Need a full URL * ci: Fix // in container name * ci: Remove push Build once, renovate should bump the images automagically * build: Add containers back * ci: Add cache repos Idk what this does exactly * ci: Change registry name to use _ Because "build" is a api end point on quay.io. So `bowtie/build` doesn't work. Other plus is this matches the conda env name. * build: / => _ in container name * Try ociAutoPull * chore: Add renovate comments to samtools Just to trigger wave build * test: Add ociAutoPull to nf-test * ci: Bump wave version * chore: Bump containers with new wave version Not sure why that's happening... * build: Update to use commity.wave.seqera.io * ci: Bump wave-cli to 1.4.1 * ci: Try apptainer * ci: Remove build-repo to see what happens * build: Bump Nextflow version requirement * fix: Get rid of the environment name? Maybe this will get the auto generated tag? * ci: Bump action versions * ci: Try name-strategy tagPrefix seqeralabs/wave-cli@269df0e * ci: Remove singularity build for now * ci: Try imageSuffix * ci: Try none * ci: What is the bowtie container name * ci: Remove --name-strategy * style: Add back in container elvis operator * ci: Remove cache repo * Revert "build: Bump Nextflow version requirement" This reverts commit 69e1ea5. * Revert "test: Add ociAutoPull to nf-test" This reverts commit 5a3d546. * test(#6505): Snapshot the versions contents, not the hash * ci(#6505): Update version snapshot after building containers * test(#6505): Attempt a topic channel with tests askimed/nf-test#258 * chore: Bump to 1.5.0 * fix: Remove shard and filter on test bumping * build: Bump images to match environment * ci: Fix nf-test setup * ci: Remove snapshot bumping * build: Fix containers in bowtie --------- Co-authored-by: ewels <[email protected]>
nf-core · Nov 7, 2024 · 669eb24 · 669eb24
1 parent a2db453
commit 669eb24
Show file tree

Hide file tree

Showing 8 changed files with 157 additions and 10 deletions.
diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml
@@ -0,0 +1,138 @@
+name: Wave
+# When environment.yml is changed
+on:
+  pull_request:
+    paths:
+      - "**/environment.yml"
+
+# TODO On complete call testing CI
+# TODO Skip testing CI if any changes to environment.yml
+
+env:
+  WAVE_VER: "1.5.0"
+  NFTEST_VER: "0.9.1"
+
+jobs:
+  gen-matrix:
+    name: generate-matrix
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
+
+      - name: Calculate file differences
+        id: diff
+        uses: tj-actions/changed-files@v44
+        with:
+          json: true
+          quotepath: false
+          # TODO Add Dockerfiles
+          files: |
+            modules/**/environment.yml
+      - name: Debug
+        run: echo ${{ steps.diff.outputs.all_changed_files }}
+      - id: set-matrix
+        run: echo "matrix={\"profile\":[\"docker\", \"singularity\"],\"files\":${{ steps.diff.outputs.all_changed_files }} }" >> "$GITHUB_OUTPUT"
+
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+
+  build:
+    if: "${{ fromJson(needs.gen-matrix.outputs.matrix) }}"
+    needs: gen-matrix
+    name: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix: "${{ fromJson(needs.gen-matrix.outputs.matrix) }}"
+    steps:
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
+
+      - name: Install wave-cli
+        run: |
+          wget -q https://github.com/seqeralabs/wave-cli/releases/download/v${WAVE_VER}/wave-${WAVE_VER}-linux-x86_64
+          sudo mv wave-${WAVE_VER}-linux-x86_64 /usr/local/bin/wave
+          chmod +x /usr/local/bin/wave
+
+      - name: Create a registry name
+        uses: actions/github-script@v7
+        id: registry-name
+        with:
+          result-encoding: string
+          script: |
+            return '${{ matrix.files }}'.replace('modules/nf-core/', '').replace('/environment.yml', '').replace('/', '_');
+
+      - name: Build container
+        if: matrix.profile == 'docker'
+        run: |
+          wave --conda-file "${{ matrix.files }}" \
+              --freeze \
+              --await \
+              --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \
+              --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }}
+
+      - name: Build Singularity
+        if: matrix.profile == 'singularity'
+        run: |
+          wave --conda-file "${{ matrix.files }}" \
+              --freeze \
+              --await \
+              --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \
+              --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \
+              --singularity
+
+    # TODO Build from Dockerfiles
+
+  # bump-versions:
+  #   needs: gen-matrix
+  #   name: bump-versions
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4
+  #       with:
+  #         distribution: "temurin"
+  #         java-version: "17"
+
+  #     - uses: nf-core/setup-nextflow@v2
+
+  #     - uses: nf-core/setup-nf-test@v1
+  #       with:
+  #         version: ${{ env.NFTEST_VER }}
+
+  #     - name: Bump Snapshot Versions
+  #       env:
+  #         # NFT_DIFF: "pdiff"
+  #         # NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2"
+  #         SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }}
+  #         SENTIEON_AUTH_MECH: "GitHub Actions - token"
+  #       run: |
+  #         # use "docker_self_hosted" if it runs on self-hosted runner and matrix.profile=docker
+  #         if [ "${{ matrix.profile }}" == "docker" ]; then
+  #           PROFILE="docker_self_hosted"
+  #         else
+  #           PROFILE=${{ matrix.profile }}
+  #         fi
+
+  #         NFT_WORKDIR=~ \
+  #         nf-test test \
+  #           --profile=${{ matrix.profile }} \
+  #           --tap=test.tap \
+  #           # --ci \
+  #           --verbose \
+  #           --only-changed \
+  #           # --shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }} \
+  #           # --filter ${{ matrix.filter }} \
+  #           --follow-dependencies \
+  #           --tag version \
+  #           --update-snapshot
+
+  #     - name: Commit & push version bumps
+  #       run: |
+  #         git config user.email "[email protected]"
+  #         git config user.name "nf-core-bot"
+  #         git config push.default upstream
+  #         git add .
+  #         git status
+  #         git commit -m "[automated] Bump versions snapshot"
+  #         git push
diff --git a/modules/nf-core/bowtie/align/environment.yml b/modules/nf-core/bowtie/align/environment.yml
@@ -2,5 +2,7 @@ channels:
   - conda-forge
   - bioconda
 dependencies:
-  - bioconda::bowtie=1.3.0
-  - bioconda::samtools=1.16.1
+  # renovate: datasource=conda depName=bioconda/bowtie
+  - bioconda::bowtie=1.3.1
+  # renovate: datasource=conda depName=bioconda/samtools
+  - bioconda::samtools=1.20
diff --git a/modules/nf-core/bowtie/align/main.nf b/modules/nf-core/bowtie/align/main.nf
@@ -4,8 +4,8 @@ process BOWTIE_ALIGN {
 
     conda "${moduleDir}/environment.yml"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/mulled-v2-ffbf83a6b0ab6ec567a336cf349b80637135bca3:c84c7c55c45af231883d9ff4fe706ac44c479c36-0' :
-        'biocontainers/mulled-v2-ffbf83a6b0ab6ec567a336cf349b80637135bca3:c84c7c55c45af231883d9ff4fe706ac44c479c36-0' }"
+        'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/c8/c8c0819a9b1f520c49c933e667ae50de2a0730ece4c8b9efe79ac5e403963a9f/data' :
+        'community.wave.seqera.io/library/bowtie_samtools:e1a14e1ce4e0170d' }"
 
     input:
     tuple val(meta), path(reads)

diff --git a/modules/nf-core/bowtie/build/environment.yml b/modules/nf-core/bowtie/build/environment.yml
@@ -2,4 +2,7 @@ channels:
   - conda-forge
   - bioconda
 dependencies:
-  - bioconda::bowtie=1.3.0
+  # renovate: datasource=conda depName=bioconda/bowtie
+  - bioconda::bowtie=1.3.1
+  # renovate: datasource=conda depName=bioconda/samtools
+  - bioconda::samtools=1.20
diff --git a/modules/nf-core/bowtie/build/main.nf b/modules/nf-core/bowtie/build/main.nf
@@ -4,8 +4,8 @@ process BOWTIE_BUILD {
 
     conda "${moduleDir}/environment.yml"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/bowtie:1.3.0--py38hed8969a_1' :
-        'biocontainers/bowtie:1.3.0--py38hed8969a_1' }"
+        'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/c8/c8c0819a9b1f520c49c933e667ae50de2a0730ece4c8b9efe79ac5e403963a9f/data' :
+        'community.wave.seqera.io/library/bowtie_samtools:e1a14e1ce4e0170d' }"
 
     input:
     tuple val(meta), path(fasta)

diff --git a/modules/nf-core/bowtie/build/tests/main.nf.test b/modules/nf-core/bowtie/build/tests/main.nf.test
@@ -34,6 +34,7 @@ nextflow_process {
     test("sarscov2 - fasta - stub") {
 
         options "-stub"
+        tag "stub"
 
         when {
             process {
@@ -48,7 +49,8 @@ nextflow_process {
         then {
             assertAll(
                 { assert process.success },
-                { assert snapshot(process.out).match() }
+                { assert snapshot(process.out).match() },
+                { assert snapshot(path(process.out.versions.get(0)).yaml).match("versions") },
             )
         }
 

diff --git a/modules/nf-core/samtools/view/environment.yml b/modules/nf-core/samtools/view/environment.yml
@@ -4,5 +4,7 @@ channels:
   - conda-forge
   - bioconda
 dependencies:
+  # renovate: datasource=conda depName=bioconda/htslib
   - bioconda::htslib=1.21
+  # renovate: datasource=conda depName=bioconda/samtools
   - bioconda::samtools=1.21
diff --git a/modules/nf-core/samtools/view/main.nf b/modules/nf-core/samtools/view/main.nf
@@ -4,8 +4,8 @@ process SAMTOOLS_VIEW {
 
     conda "${moduleDir}/environment.yml"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/samtools:1.21--h50ea8bc_0' :
-        'biocontainers/samtools:1.21--h50ea8bc_0' }"
+        'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/9e/9edc2564215d5cd137a8b25ca8a311600987186d406b092022444adf3c4447f7/data' :
+        'community.wave.seqera.io/library/htslib_samtools:1.21--6cb89bfd40cbaabf' }"
 
     input:
     tuple val(meta), path(input), path(index)