From 4a1e847624939e9c5479772be46bf927967bf9b6 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 29 Jul 2024 07:38:05 +1000 Subject: [PATCH 1/8] Update friendbot --- .github/workflows/build-future.yml | 6 ++++-- .github/workflows/build-latest.yml | 6 ++++-- .github/workflows/build-testing.yml | 6 ++++-- .github/workflows/build.yml | 15 ++++++++++----- Makefile | 9 ++++++--- 5 files changed, 28 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-future.yml b/.github/workflows/build-future.yml index a0d33039..90161a6d 100644 --- a/.github/workflows/build-future.yml +++ b/.github/workflows/build-future.yml @@ -34,8 +34,9 @@ jobs: protocol_version_default: 21 xdr_ref: v21.1.0 core_ref: v21.2.0 - go_ref: horizon-v2.31.0 + horizon_ref: horizon-v2.31.0 soroban_rpc_ref: v21.2.0 + friendbot_ref: horizon-v2.31.0 test_matrix: | { "network": ["local"], @@ -58,9 +59,10 @@ jobs: xdr_ref: v21.1.0 core_ref: v21.2.0 core_build_runner_type: ubuntu-latest-16-cores - go_ref: horizon-v2.31.0 + horizon_ref: horizon-v2.31.0 soroban_rpc_ref: v21.2.0 soroban_rpc_build_runner_type: ubuntu-latest-16-cores + friendbot_ref: horizon-v2.31.0 test_matrix: | { "network": ["local"], diff --git a/.github/workflows/build-latest.yml b/.github/workflows/build-latest.yml index 7314b2ab..2de1195f 100644 --- a/.github/workflows/build-latest.yml +++ b/.github/workflows/build-latest.yml @@ -35,8 +35,9 @@ jobs: protocol_version_default: 20 xdr_ref: v21.1.0 core_ref: v21.2.0 - go_ref: horizon-v2.31.0 + horizon_ref: horizon-v2.31.0 soroban_rpc_ref: v21.2.0 + friendbot_ref: horizon-v2.31.0 test_matrix: | { "network": ["pubnet", "local"], @@ -59,9 +60,10 @@ jobs: xdr_ref: v21.1.0 core_ref: v21.2.0 core_build_runner_type: ubuntu-latest-16-cores - go_ref: horizon-v2.31.0 + horizon_ref: horizon-v2.31.0 soroban_rpc_ref: v21.2.0 soroban_rpc_build_runner_type: ubuntu-latest-16-cores + friendbot_ref: horizon-v2.31.0 test_matrix: | { "network": ["pubnet", "local"], diff --git a/.github/workflows/build-testing.yml b/.github/workflows/build-testing.yml index 0206398b..b7ce93db 100644 --- a/.github/workflows/build-testing.yml +++ b/.github/workflows/build-testing.yml @@ -36,8 +36,9 @@ jobs: protocol_version_default: 21 xdr_ref: v21.1.0 core_ref: v21.2.0 - go_ref: horizon-v2.31.0 + horizon_ref: horizon-v2.31.0 soroban_rpc_ref: v21.2.0 + friendbot_ref: horizon-v2.31.0 test_matrix: | { "network": ["testnet", "pubnet", "local"], @@ -60,9 +61,10 @@ jobs: xdr_ref: v21.1.0 core_ref: v21.2.0 core_build_runner_type: ubuntu-latest-16-cores - go_ref: horizon-v2.31.0 + horizon_ref: horizon-v2.31.0 soroban_rpc_ref: v21.2.0 soroban_rpc_build_runner_type: ubuntu-latest-16-cores + friendbot_ref: horizon-v2.31.0 test_matrix: | { "network": ["testnet", "pubnet", "local"], diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 90db4712..b28e13dd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,8 +38,8 @@ on: description: 'The GitHub Runner instance type to build stellar-core on' type: 'string' default: 'ubuntu-latest' - go_ref: - description: 'Git ref for the stellar/go repo (stellar-horizon, stellar-friendbot)' + horizon_ref: + description: 'Git ref for the stellar/go repo (horizon)' type: 'string' required: true xdr_ref: @@ -54,6 +54,10 @@ on: description: 'The GitHub Runner instance type to build soroban-rpc on' type: 'string' default: 'ubuntu-latest' + friendbot_ref: + description: 'Git ref for the stellar/go repo (friendbot)' + type: 'string' + required: true test_matrix: description: 'JSON matrix for the test job' type: 'string' @@ -65,7 +69,8 @@ on: env: IMAGE: ${{ format('{0}/{1}:{2}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository, github.event_name == 'pull_request' && format('pr{0}-{1}', github.event.pull_request.number, inputs.tag) || inputs.tag) }} - GO_REPO_BRANCH: ${{ inputs.go_ref }} + HORIZON_REPO_REF: ${{ inputs.horizon_ref }} + FRIENDBOT_REPO_REF: ${{ inputs.friendbot_ref }} SOROBAN_RPC_REPO_BRANCH: ${{ inputs.soroban_rpc_ref }} CORE_REPO: ${{ inputs.core_repo }} CORE_REPO_REF: ${{ inputs.core_ref }} @@ -122,7 +127,7 @@ jobs: docker buildx build --platform linux/${{ inputs.arch }} -f Dockerfile.horizon --target builder -t stellar-horizon:${{ inputs.arch }} -o type=docker,dest=/tmp/image - --build-arg REF="${{ env.GO_REPO_BRANCH }}" . + --build-arg REF="${{ env.HORIZON_REPO_REF }}" . - name: Upload Stellar-Horizon Image uses: actions/upload-artifact@v4 with: @@ -146,7 +151,7 @@ jobs: -f services/friendbot/docker/Dockerfile -t stellar-friendbot:${{ inputs.arch }} -o type=docker,dest=/tmp/image --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true - https://github.com/stellar/go.git#${{ env.GO_REPO_BRANCH }} + https://github.com/stellar/go.git#${{ env.FRIENDBOT_REPO_REF }} - name: Upload Stellar-Friendbot Image uses: actions/upload-artifact@v4 with: diff --git a/Makefile b/Makefile index 63d24479..145a500a 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,8 @@ build-latest: XDR_REF=v21.1.0 \ CORE_REF=v21.2.0 \ HORIZON_REF=horizon-v2.31.0 \ - SOROBAN_RPC_REF=v21.2.0 + SOROBAN_RPC_REF=v21.2.0 \ + FRIENDBOT_REF=31fc8f4236388f12fc609228b7a7f5494867a1f9 build-testing: $(MAKE) build TAG=testing \ @@ -35,7 +36,8 @@ build-testing: XDR_REF=v21.1.0 \ CORE_REF=v21.2.0 \ HORIZON_REF=horizon-v2.31.0 \ - SOROBAN_RPC_REF=v21.2.0 + SOROBAN_RPC_REF=v21.2.0 \ + FRIENDBOT_REF=31fc8f4236388f12fc609228b7a7f5494867a1f9 build-future: $(MAKE) build TAG=future \ @@ -43,7 +45,8 @@ build-future: XDR_REF=v21.1.0 \ CORE_REF=v21.2.0 \ HORIZON_REF=horizon-v2.31.0 \ - SOROBAN_RPC_REF=v21.2.0 + SOROBAN_RPC_REF=v21.2.0 \ + FRIENDBOT_REF=31fc8f4236388f12fc609228b7a7f5494867a1f9 build: $(MAKE) -j 4 build-deps From 7628cb73daa7ddca9474372bace6fc57d9f2672f Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 29 Jul 2024 08:16:24 +1000 Subject: [PATCH 2/8] upd v --- .github/workflows/build-future.yml | 4 ++-- .github/workflows/build-latest.yml | 4 ++-- .github/workflows/build-testing.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-future.yml b/.github/workflows/build-future.yml index 90161a6d..9b7c189b 100644 --- a/.github/workflows/build-future.yml +++ b/.github/workflows/build-future.yml @@ -36,7 +36,7 @@ jobs: core_ref: v21.2.0 horizon_ref: horizon-v2.31.0 soroban_rpc_ref: v21.2.0 - friendbot_ref: horizon-v2.31.0 + friendbot_ref: 31fc8f4236388f12fc609228b7a7f5494867a1f9 test_matrix: | { "network": ["local"], @@ -62,7 +62,7 @@ jobs: horizon_ref: horizon-v2.31.0 soroban_rpc_ref: v21.2.0 soroban_rpc_build_runner_type: ubuntu-latest-16-cores - friendbot_ref: horizon-v2.31.0 + friendbot_ref: 31fc8f4236388f12fc609228b7a7f5494867a1f9 test_matrix: | { "network": ["local"], diff --git a/.github/workflows/build-latest.yml b/.github/workflows/build-latest.yml index 2de1195f..6bf8c8dc 100644 --- a/.github/workflows/build-latest.yml +++ b/.github/workflows/build-latest.yml @@ -37,7 +37,7 @@ jobs: core_ref: v21.2.0 horizon_ref: horizon-v2.31.0 soroban_rpc_ref: v21.2.0 - friendbot_ref: horizon-v2.31.0 + friendbot_ref: 31fc8f4236388f12fc609228b7a7f5494867a1f9 test_matrix: | { "network": ["pubnet", "local"], @@ -63,7 +63,7 @@ jobs: horizon_ref: horizon-v2.31.0 soroban_rpc_ref: v21.2.0 soroban_rpc_build_runner_type: ubuntu-latest-16-cores - friendbot_ref: horizon-v2.31.0 + friendbot_ref: 31fc8f4236388f12fc609228b7a7f5494867a1f9 test_matrix: | { "network": ["pubnet", "local"], diff --git a/.github/workflows/build-testing.yml b/.github/workflows/build-testing.yml index b7ce93db..301ba372 100644 --- a/.github/workflows/build-testing.yml +++ b/.github/workflows/build-testing.yml @@ -38,7 +38,7 @@ jobs: core_ref: v21.2.0 horizon_ref: horizon-v2.31.0 soroban_rpc_ref: v21.2.0 - friendbot_ref: horizon-v2.31.0 + friendbot_ref: 31fc8f4236388f12fc609228b7a7f5494867a1f9 test_matrix: | { "network": ["testnet", "pubnet", "local"], @@ -64,7 +64,7 @@ jobs: horizon_ref: horizon-v2.31.0 soroban_rpc_ref: v21.2.0 soroban_rpc_build_runner_type: ubuntu-latest-16-cores - friendbot_ref: horizon-v2.31.0 + friendbot_ref: 31fc8f4236388f12fc609228b7a7f5494867a1f9 test_matrix: | { "network": ["testnet", "pubnet", "local"], From 0c8c9005fd4cfd0229993288d849e8c5d22f8921 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 29 Jul 2024 08:36:06 +1000 Subject: [PATCH 3/8] Use arm runners --- .github/workflows/build-future.yml | 6 +- .github/workflows/build-latest.yml | 6 +- .github/workflows/build-testing.yml | 6 +- .github/workflows/build.yml | 86 ++++++++++++++--------------- 4 files changed, 55 insertions(+), 49 deletions(-) diff --git a/.github/workflows/build-future.yml b/.github/workflows/build-future.yml index 9b7c189b..50dc492f 100644 --- a/.github/workflows/build-future.yml +++ b/.github/workflows/build-future.yml @@ -28,6 +28,8 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: + runner: ubuntu-latest + runner_big: ubuntu-latest-16-cores sha: ${{ inputs.sha }} arch: amd64 tag: ${{ inputs.tag-prefix }}future-amd64 @@ -52,16 +54,16 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: + runner: ubuntu-jammy-4-cores-arm64 + runner_big: ubuntu-jammy-16-cores-arm64 sha: ${{ inputs.sha }} arch: arm64 tag: ${{ inputs.tag-prefix }}future-arm64 protocol_version_default: 21 xdr_ref: v21.1.0 core_ref: v21.2.0 - core_build_runner_type: ubuntu-latest-16-cores horizon_ref: horizon-v2.31.0 soroban_rpc_ref: v21.2.0 - soroban_rpc_build_runner_type: ubuntu-latest-16-cores friendbot_ref: 31fc8f4236388f12fc609228b7a7f5494867a1f9 test_matrix: | { diff --git a/.github/workflows/build-latest.yml b/.github/workflows/build-latest.yml index 6bf8c8dc..a7e10cdc 100644 --- a/.github/workflows/build-latest.yml +++ b/.github/workflows/build-latest.yml @@ -29,6 +29,8 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: + runner: ubuntu-latest + runner_big: ubuntu-latest-16-cores sha: ${{ inputs.sha }} arch: amd64 tag: ${{ inputs.tag-prefix }}latest-amd64 @@ -53,16 +55,16 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: + runner: ubuntu-jammy-4-cores-arm64 + runner_big: ubuntu-jammy-16-cores-arm64 sha: ${{ inputs.sha }} arch: arm64 tag: ${{ inputs.tag-prefix }}latest-arm64 protocol_version_default: 20 xdr_ref: v21.1.0 core_ref: v21.2.0 - core_build_runner_type: ubuntu-latest-16-cores horizon_ref: horizon-v2.31.0 soroban_rpc_ref: v21.2.0 - soroban_rpc_build_runner_type: ubuntu-latest-16-cores friendbot_ref: 31fc8f4236388f12fc609228b7a7f5494867a1f9 test_matrix: | { diff --git a/.github/workflows/build-testing.yml b/.github/workflows/build-testing.yml index 301ba372..00a622ff 100644 --- a/.github/workflows/build-testing.yml +++ b/.github/workflows/build-testing.yml @@ -30,6 +30,8 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: + runner: ubuntu-latest + runner_big: ubuntu-latest-16-cores sha: ${{ inputs.sha }} arch: amd64 tag: ${{ inputs.tag-prefix }}testing-amd64 @@ -54,16 +56,16 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: + runner: ubuntu-jammy-4-cores-arm64 + runner_big: ubuntu-jammy-16-cores-arm64 sha: ${{ inputs.sha }} arch: arm64 tag: ${{ inputs.tag-prefix }}testing-arm64 protocol_version_default: 21 xdr_ref: v21.1.0 core_ref: v21.2.0 - core_build_runner_type: ubuntu-latest-16-cores horizon_ref: horizon-v2.31.0 soroban_rpc_ref: v21.2.0 - soroban_rpc_build_runner_type: ubuntu-latest-16-cores friendbot_ref: 31fc8f4236388f12fc609228b7a7f5494867a1f9 test_matrix: | { diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b28e13dd..63e906a3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,6 +6,14 @@ on: DOCKERHUB_TOKEN: required: false inputs: + runner: + description: 'GitHub runner to run jobs on' + type: 'string' + required: true + runner_big: + description: 'GitHub runner to run larger jobs on' + type: 'string' + required: true sha: description: 'Sha to build' type: 'string' @@ -34,10 +42,6 @@ on: description: 'CONFIGURE_FLAGS used when building stellar-core' type: 'string' default: '--disable-tests' - core_build_runner_type: - description: 'The GitHub Runner instance type to build stellar-core on' - type: 'string' - default: 'ubuntu-latest' horizon_ref: description: 'Git ref for the stellar/go repo (horizon)' type: 'string' @@ -50,10 +54,6 @@ on: description: 'Git ref for the stellar/soroban-rpc repo (soroban-rpc)' type: 'string' required: true - soroban_rpc_build_runner_type: - description: 'The GitHub Runner instance type to build soroban-rpc on' - type: 'string' - default: 'ubuntu-latest' friendbot_ref: description: 'Git ref for the stellar/go repo (friendbot)' type: 'string' @@ -79,17 +79,17 @@ env: jobs: build-stellar-core: - runs-on: ${{ inputs.core_build_runner_type }} + runs-on: ${{ inputs.runner_big }} steps: - id: cache uses: actions/cache@v3 with: path: /tmp/image key: image-stellar-core-${{ inputs.arch }}-${{ env.CORE_REPO_REF }}-${{ inputs.core_configure_flags }} - - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 + # - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' + # uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + # with: + # platforms: arm64 - if: steps.cache.outputs.cache-hit != 'true' uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - if: steps.cache.outputs.cache-hit != 'true' @@ -110,16 +110,16 @@ jobs: # don't use caches on horizon builds as the git ref for it can be a branch name that refers to # different commits over time, cache key won't invalidate correctly. build-stellar-horizon: - runs-on: ubuntu-latest + runs-on: ${{ inputs.runner }} steps: - name: Checkout Quickstart for Horizon docker file uses: actions/checkout@v3 with: ref: ${{ inputs.sha }} - - if: inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 + # - if: inputs.arch == 'arm64' + # uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + # with: + # platforms: arm64 - name: Setup buildx uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - name: Build Stellar-Horizon Image @@ -137,12 +137,12 @@ jobs: # don't use caches on friendbot builds as the git ref for it can be a branch name that refers to # different commits over time, cache key won't invalidate correctly. build-stellar-friendbot: - runs-on: ubuntu-latest + runs-on: ${{ inputs.runner }} steps: - - if: inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 + # - if: inputs.arch == 'arm64' + # uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + # with: + # platforms: arm64 - name: Setup buildx uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - name: Build Stellar-Friendbot Image @@ -159,17 +159,17 @@ jobs: path: /tmp/image build-stellar-soroban-rpc: - runs-on: ${{ inputs.soroban_rpc_build_runner_type }} + runs-on: ${{ inputs.runner_big }} steps: - id: cache uses: actions/cache@v3 with: path: /tmp/image key: image-stellar-soroban-rpc-${{ inputs.arch }}-${{ env.SOROBAN_RPC_REPO_BRANCH }} - - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 + # - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' + # uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + # with: + # platforms: arm64 - if: steps.cache.outputs.cache-hit != 'true' uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - if: steps.cache.outputs.cache-hit != 'true' @@ -188,7 +188,7 @@ jobs: path: /tmp/image build-rs-stellar-xdr: - runs-on: ubuntu-latest + runs-on: ${{ inputs.runner }} steps: - name: Checkout Quickstart for Horizon docker file uses: actions/checkout@v3 @@ -199,10 +199,10 @@ jobs: with: path: /tmp/image key: image-rs-stellar-xdr-${{ inputs.arch }}-${{ env.XDR_REPO_REF }} - - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 + # - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' + # uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + # with: + # platforms: arm64 - if: steps.cache.outputs.cache-hit != 'true' uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - if: steps.cache.outputs.cache-hit != 'true' @@ -224,7 +224,7 @@ jobs: needs: [build-stellar-core, build-stellar-horizon, build-rs-stellar-xdr, build-stellar-friendbot, build-stellar-soroban-rpc] outputs: image: ${{ steps.image.outputs.name }} - runs-on: ubuntu-latest + runs-on: ${{ inputs.runner }} steps: - uses: actions/checkout@v3 with: @@ -264,10 +264,10 @@ jobs: run: docker load -i /tmp/stellar-soroban-rpc/image - name: Load Stellar-Rs-Xdr Image run: docker load -i /tmp/stellar-xdr/image - - if: inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 + # - if: inputs.arch == 'arm64' + # uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + # with: + # platforms: arm64 - id: image name: Image Name run: echo "name=$IMAGE" >> $GITHUB_OUTPUT @@ -307,7 +307,7 @@ jobs: strategy: matrix: ${{ fromJSON(inputs.test_matrix) }} fail-fast: false - runs-on: ${{ matrix.network == 'pubnet' && 'ubuntu-latest-8-cores' || 'ubuntu-latest' }} + runs-on: ${{ matrix.network == 'pubnet' && inputs.runner_big || inputs.runner }} steps: - uses: actions/checkout@v2 with: @@ -319,10 +319,10 @@ jobs: path: /tmp/ - name: Load Quickstart Image run: docker load -i /tmp/image - - if: inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 + # - if: inputs.arch == 'arm64' + # uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + # with: + # platforms: arm64 - name: Prepare Logs Directory run: mkdir -p logs - name: Run Quickstart Image From 01bcb57928aaefaf09f341ae27622b0512362791 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 29 Jul 2024 20:33:29 +1000 Subject: [PATCH 4/8] reduce costs --- .github/workflows/build-future.yml | 4 -- .github/workflows/build-latest.yml | 4 -- .github/workflows/build-testing.yml | 4 -- .github/workflows/build.yml | 78 +++++++++++++---------------- 4 files changed, 35 insertions(+), 55 deletions(-) diff --git a/.github/workflows/build-future.yml b/.github/workflows/build-future.yml index 50dc492f..16473146 100644 --- a/.github/workflows/build-future.yml +++ b/.github/workflows/build-future.yml @@ -28,8 +28,6 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: - runner: ubuntu-latest - runner_big: ubuntu-latest-16-cores sha: ${{ inputs.sha }} arch: amd64 tag: ${{ inputs.tag-prefix }}future-amd64 @@ -54,8 +52,6 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: - runner: ubuntu-jammy-4-cores-arm64 - runner_big: ubuntu-jammy-16-cores-arm64 sha: ${{ inputs.sha }} arch: arm64 tag: ${{ inputs.tag-prefix }}future-arm64 diff --git a/.github/workflows/build-latest.yml b/.github/workflows/build-latest.yml index a7e10cdc..1292660b 100644 --- a/.github/workflows/build-latest.yml +++ b/.github/workflows/build-latest.yml @@ -29,8 +29,6 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: - runner: ubuntu-latest - runner_big: ubuntu-latest-16-cores sha: ${{ inputs.sha }} arch: amd64 tag: ${{ inputs.tag-prefix }}latest-amd64 @@ -55,8 +53,6 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: - runner: ubuntu-jammy-4-cores-arm64 - runner_big: ubuntu-jammy-16-cores-arm64 sha: ${{ inputs.sha }} arch: arm64 tag: ${{ inputs.tag-prefix }}latest-arm64 diff --git a/.github/workflows/build-testing.yml b/.github/workflows/build-testing.yml index 00a622ff..43477063 100644 --- a/.github/workflows/build-testing.yml +++ b/.github/workflows/build-testing.yml @@ -30,8 +30,6 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: - runner: ubuntu-latest - runner_big: ubuntu-latest-16-cores sha: ${{ inputs.sha }} arch: amd64 tag: ${{ inputs.tag-prefix }}testing-amd64 @@ -56,8 +54,6 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: - runner: ubuntu-jammy-4-cores-arm64 - runner_big: ubuntu-jammy-16-cores-arm64 sha: ${{ inputs.sha }} arch: arm64 tag: ${{ inputs.tag-prefix }}testing-arm64 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 63e906a3..c63c4342 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,14 +6,6 @@ on: DOCKERHUB_TOKEN: required: false inputs: - runner: - description: 'GitHub runner to run jobs on' - type: 'string' - required: true - runner_big: - description: 'GitHub runner to run larger jobs on' - type: 'string' - required: true sha: description: 'Sha to build' type: 'string' @@ -79,17 +71,17 @@ env: jobs: build-stellar-core: - runs-on: ${{ inputs.runner_big }} + runs-on: ${{ inputs.arch == 'arm64' && 'ubuntu-jammy-4-cores-arm64' || 'ubuntu-latest' }} steps: - id: cache uses: actions/cache@v3 with: path: /tmp/image key: image-stellar-core-${{ inputs.arch }}-${{ env.CORE_REPO_REF }}-${{ inputs.core_configure_flags }} - # - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' - # uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - # with: - # platforms: arm64 + - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 - if: steps.cache.outputs.cache-hit != 'true' uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - if: steps.cache.outputs.cache-hit != 'true' @@ -110,16 +102,16 @@ jobs: # don't use caches on horizon builds as the git ref for it can be a branch name that refers to # different commits over time, cache key won't invalidate correctly. build-stellar-horizon: - runs-on: ${{ inputs.runner }} + runs-on: ubuntu-latest steps: - name: Checkout Quickstart for Horizon docker file uses: actions/checkout@v3 with: ref: ${{ inputs.sha }} - # - if: inputs.arch == 'arm64' - # uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - # with: - # platforms: arm64 + - if: inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 - name: Setup buildx uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - name: Build Stellar-Horizon Image @@ -137,12 +129,12 @@ jobs: # don't use caches on friendbot builds as the git ref for it can be a branch name that refers to # different commits over time, cache key won't invalidate correctly. build-stellar-friendbot: - runs-on: ${{ inputs.runner }} + runs-on: ubuntu-latest steps: - # - if: inputs.arch == 'arm64' - # uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - # with: - # platforms: arm64 + - if: inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 - name: Setup buildx uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - name: Build Stellar-Friendbot Image @@ -159,17 +151,17 @@ jobs: path: /tmp/image build-stellar-soroban-rpc: - runs-on: ${{ inputs.runner_big }} + runs-on: ${{ inputs.arch == 'arm64' && 'ubuntu-jammy-4-cores-arm64' || 'ubuntu-latest' }} steps: - id: cache uses: actions/cache@v3 with: path: /tmp/image key: image-stellar-soroban-rpc-${{ inputs.arch }}-${{ env.SOROBAN_RPC_REPO_BRANCH }} - # - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' - # uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - # with: - # platforms: arm64 + - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 - if: steps.cache.outputs.cache-hit != 'true' uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - if: steps.cache.outputs.cache-hit != 'true' @@ -188,7 +180,7 @@ jobs: path: /tmp/image build-rs-stellar-xdr: - runs-on: ${{ inputs.runner }} + runs-on: ${{ inputs.arch == 'arm64' && 'ubuntu-jammy-4-cores-arm64' || 'ubuntu-latest' }} steps: - name: Checkout Quickstart for Horizon docker file uses: actions/checkout@v3 @@ -199,10 +191,10 @@ jobs: with: path: /tmp/image key: image-rs-stellar-xdr-${{ inputs.arch }}-${{ env.XDR_REPO_REF }} - # - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' - # uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - # with: - # platforms: arm64 + - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 - if: steps.cache.outputs.cache-hit != 'true' uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - if: steps.cache.outputs.cache-hit != 'true' @@ -224,7 +216,7 @@ jobs: needs: [build-stellar-core, build-stellar-horizon, build-rs-stellar-xdr, build-stellar-friendbot, build-stellar-soroban-rpc] outputs: image: ${{ steps.image.outputs.name }} - runs-on: ${{ inputs.runner }} + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: @@ -264,10 +256,10 @@ jobs: run: docker load -i /tmp/stellar-soroban-rpc/image - name: Load Stellar-Rs-Xdr Image run: docker load -i /tmp/stellar-xdr/image - # - if: inputs.arch == 'arm64' - # uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - # with: - # platforms: arm64 + - if: inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 - id: image name: Image Name run: echo "name=$IMAGE" >> $GITHUB_OUTPUT @@ -307,7 +299,7 @@ jobs: strategy: matrix: ${{ fromJSON(inputs.test_matrix) }} fail-fast: false - runs-on: ${{ matrix.network == 'pubnet' && inputs.runner_big || inputs.runner }} + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 with: @@ -319,10 +311,10 @@ jobs: path: /tmp/ - name: Load Quickstart Image run: docker load -i /tmp/image - # - if: inputs.arch == 'arm64' - # uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - # with: - # platforms: arm64 + - if: inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 - name: Prepare Logs Directory run: mkdir -p logs - name: Run Quickstart Image From 15716e62c7798cf064befcec3bebe89118c407c9 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 29 Jul 2024 21:15:08 +1000 Subject: [PATCH 5/8] load from cache on cheap instances --- .github/workflows/build.yml | 85 ++++++++++++++++++++++++++++--------- 1 file changed, 66 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c63c4342..1a843a63 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -70,7 +70,26 @@ env: jobs: + load-stellar-core-from-cache: + runs-on: ubuntu-latest + outputs: + cache-hit: ${{ steps.cache.outputs.cache-hit }} + steps: + - id: cache + uses: actions/cache@v3 + with: + path: /tmp/image + key: image-stellar-core-${{ inputs.arch }}-${{ env.CORE_REPO_REF }}-${{ inputs.core_configure_flags }} + - name: Upload Stellar-Core Image + if: steps.cache.outputs.cache-hit == 'true' && inputs.arch == 'arm64' + uses: actions/upload-artifact@v4 + with: + name: image-stellar-core-${{ inputs.tag }}-${{ inputs.arch }} + path: /tmp/image + build-stellar-core: + needs: [load-stellar-core-from-cache] + if: ${{ needs.load-stellar-core-from-cache.outputs.cache-hit != 'true' }} runs-on: ${{ inputs.arch == 'arm64' && 'ubuntu-jammy-4-cores-arm64' || 'ubuntu-latest' }} steps: - id: cache @@ -78,14 +97,12 @@ jobs: with: path: /tmp/image key: image-stellar-core-${{ inputs.arch }}-${{ env.CORE_REPO_REF }}-${{ inputs.core_configure_flags }} - - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' + - if: inputs.arch == 'arm64' uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 with: platforms: arm64 - - if: steps.cache.outputs.cache-hit != 'true' - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - if: steps.cache.outputs.cache-hit != 'true' - name: Build Stellar-Core Image + - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f + - name: Build Stellar-Core Image run: > docker buildx build --platform linux/${{ inputs.arch }} -f docker/Dockerfile.testing -t stellar-core:${{ inputs.arch }} @@ -99,8 +116,6 @@ jobs: name: image-stellar-core-${{ inputs.tag }}-${{ inputs.arch }} path: /tmp/image -# don't use caches on horizon builds as the git ref for it can be a branch name that refers to -# different commits over time, cache key won't invalidate correctly. build-stellar-horizon: runs-on: ubuntu-latest steps: @@ -126,8 +141,6 @@ jobs: name: image-stellar-horizon-${{ inputs.tag }}-${{ inputs.arch }} path: /tmp/image -# don't use caches on friendbot builds as the git ref for it can be a branch name that refers to -# different commits over time, cache key won't invalidate correctly. build-stellar-friendbot: runs-on: ubuntu-latest steps: @@ -150,7 +163,26 @@ jobs: name: image-stellar-friendbot-${{ inputs.tag }}-${{ inputs.arch }} path: /tmp/image + load-stellar-soroban-rpc-from-cache: + runs-on: ubuntu-latest + outputs: + cache-hit: ${{ steps.cache.outputs.cache-hit }} + steps: + - id: cache + uses: actions/cache@v3 + with: + path: /tmp/image + key: image-stellar-soroban-rpc-${{ inputs.arch }}-${{ env.SOROBAN_RPC_REPO_BRANCH }} + - name: Upload Stellar-Core Image + if: steps.cache.outputs.cache-hit == 'true' && inputs.arch == 'arm64' + uses: actions/upload-artifact@v4 + with: + name: image-stellar-soroban-rpc-${{ inputs.tag }}-${{ inputs.arch }} + path: /tmp/image + build-stellar-soroban-rpc: + needs: [load-stellar-soroban-rpc-from-cache] + if: ${{ needs.load-stellar-soroban-rpc-from-cache.outputs.cache-hit != 'true' }} runs-on: ${{ inputs.arch == 'arm64' && 'ubuntu-jammy-4-cores-arm64' || 'ubuntu-latest' }} steps: - id: cache @@ -158,14 +190,12 @@ jobs: with: path: /tmp/image key: image-stellar-soroban-rpc-${{ inputs.arch }}-${{ env.SOROBAN_RPC_REPO_BRANCH }} - - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' + - if: inputs.arch == 'arm64' uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 with: platforms: arm64 - - if: steps.cache.outputs.cache-hit != 'true' - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - if: steps.cache.outputs.cache-hit != 'true' - name: Build Stellar-Soroban-Rpc Image + - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f + - name: Build Stellar-Soroban-Rpc Image run: > docker buildx build --platform linux/${{ inputs.arch }} -f cmd/soroban-rpc/docker/Dockerfile --target build @@ -179,7 +209,26 @@ jobs: name: image-stellar-soroban-rpc-${{ inputs.tag }}-${{ inputs.arch }} path: /tmp/image + load-rs-stellar-xdr-from-cache: + runs-on: ubuntu-latest + outputs: + cache-hit: ${{ steps.cache.outputs.cache-hit }} + steps: + - id: cache + uses: actions/cache@v3 + with: + path: /tmp/image + key: image-rs-stellar-xdr-${{ inputs.arch }}-${{ env.XDR_REPO_REF }} + - name: Upload Stellar-Core Image + if: steps.cache.outputs.cache-hit == 'true' && inputs.arch == 'arm64' + uses: actions/upload-artifact@v4 + with: + name: image-rs-stellar-xdr-${{ inputs.tag }}-${{ inputs.arch }} + path: /tmp/image + build-rs-stellar-xdr: + needs: [load-rs-stellar-xdr-from-cache] + if: ${{ needs.load-rs-stellar-xdr-from-cache.outputs.cache-hit != 'true' }} runs-on: ${{ inputs.arch == 'arm64' && 'ubuntu-jammy-4-cores-arm64' || 'ubuntu-latest' }} steps: - name: Checkout Quickstart for Horizon docker file @@ -191,14 +240,12 @@ jobs: with: path: /tmp/image key: image-rs-stellar-xdr-${{ inputs.arch }}-${{ env.XDR_REPO_REF }} - - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' + - if: inputs.arch == 'arm64' uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 with: platforms: arm64 - - if: steps.cache.outputs.cache-hit != 'true' - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - if: steps.cache.outputs.cache-hit != 'true' - name: Build Stellar-Rs-Xdr Image + - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f + - name: Build Stellar-Rs-Xdr Image run: > docker buildx build --platform linux/${{ inputs.arch }} -f Dockerfile.xdr --target builder From 29ce806746333c54bedcc043a3c2666d85174aef Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 29 Jul 2024 21:31:19 +1000 Subject: [PATCH 6/8] always --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1a843a63..27cfa8e4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -261,6 +261,7 @@ jobs: build: needs: [build-stellar-core, build-stellar-horizon, build-rs-stellar-xdr, build-stellar-friendbot, build-stellar-soroban-rpc] + if: always() outputs: image: ${{ steps.image.outputs.name }} runs-on: ubuntu-latest From f2d2cf55489dfaf279ecb12b4e0610c4b6353827 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 29 Jul 2024 21:38:52 +1000 Subject: [PATCH 7/8] fix --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 27cfa8e4..c8c70e5c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -81,7 +81,7 @@ jobs: path: /tmp/image key: image-stellar-core-${{ inputs.arch }}-${{ env.CORE_REPO_REF }}-${{ inputs.core_configure_flags }} - name: Upload Stellar-Core Image - if: steps.cache.outputs.cache-hit == 'true' && inputs.arch == 'arm64' + if: steps.cache.outputs.cache-hit == 'true' uses: actions/upload-artifact@v4 with: name: image-stellar-core-${{ inputs.tag }}-${{ inputs.arch }} @@ -174,7 +174,7 @@ jobs: path: /tmp/image key: image-stellar-soroban-rpc-${{ inputs.arch }}-${{ env.SOROBAN_RPC_REPO_BRANCH }} - name: Upload Stellar-Core Image - if: steps.cache.outputs.cache-hit == 'true' && inputs.arch == 'arm64' + if: steps.cache.outputs.cache-hit == 'true' uses: actions/upload-artifact@v4 with: name: image-stellar-soroban-rpc-${{ inputs.tag }}-${{ inputs.arch }} @@ -220,7 +220,7 @@ jobs: path: /tmp/image key: image-rs-stellar-xdr-${{ inputs.arch }}-${{ env.XDR_REPO_REF }} - name: Upload Stellar-Core Image - if: steps.cache.outputs.cache-hit == 'true' && inputs.arch == 'arm64' + if: steps.cache.outputs.cache-hit == 'true' uses: actions/upload-artifact@v4 with: name: image-rs-stellar-xdr-${{ inputs.tag }}-${{ inputs.arch }} From e903329e1ee9deab5f1f9cb532b4b795990545d5 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 29 Jul 2024 22:03:21 +1000 Subject: [PATCH 8/8] fix --- .github/workflows/build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c8c70e5c..24820387 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -344,6 +344,7 @@ jobs: test: needs: build + if: always() strategy: matrix: ${{ fromJSON(inputs.test_matrix) }} fail-fast: false @@ -446,7 +447,7 @@ jobs: push-pr: # Push image to registry after build for pull requests from a local branch. - if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} + if: ${{ always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} needs: build permissions: packages: write @@ -470,7 +471,7 @@ jobs: push-release: # Push image to registry after test for master. - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + if: ${{ always() && github.event_name == 'push' && github.ref == 'refs/heads/master' }} needs: [build, test] permissions: packages: write