From 588bb6b9dd92574f7cf88c6ed5d44fbd09163b27 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Tue, 17 Oct 2023 09:56:11 +1000
Subject: [PATCH 01/15] Remove duplicate CI docker job and make features
 consistent

---
 .../workflows/ci-unit-tests-docker.patch.yml  |  6 --
 .github/workflows/ci-unit-tests-docker.yml    | 66 ++++++++-----------
 2 files changed, 28 insertions(+), 44 deletions(-)

diff --git a/.github/workflows/ci-unit-tests-docker.patch.yml b/.github/workflows/ci-unit-tests-docker.patch.yml
index e3e629fb490..3ddb5f51694 100644
--- a/.github/workflows/ci-unit-tests-docker.patch.yml
+++ b/.github/workflows/ci-unit-tests-docker.patch.yml
@@ -45,12 +45,6 @@ jobs:
     steps:
       - run: 'echo "No build required"'
 
-  test-all-getblocktemplate-rpcs:
-    name: Test all with getblocktemplate-rpcs feature
-    runs-on: ubuntu-latest
-    steps:
-      - run: 'echo "No build required"'
-
   test-fake-activation-heights:
     name: Test with fake activation heights
     runs-on: ubuntu-latest
diff --git a/.github/workflows/ci-unit-tests-docker.yml b/.github/workflows/ci-unit-tests-docker.yml
index fcd8e74d41f..002f434aa6e 100644
--- a/.github/workflows/ci-unit-tests-docker.yml
+++ b/.github/workflows/ci-unit-tests-docker.yml
@@ -65,6 +65,16 @@ on:
       - '.github/workflows/sub-find-cached-disks.yml'
       - '.github/workflows/sub-build-docker-image.yml'
 
+env:
+  # We need to combine the features manually because some tests don't use the Docker entrypoint
+  TEST_FEATURES: ${{ format('{0} {1}', vars.RUST_PROD_FEATURES, vars.RUST_TEST_FEATURES) }}
+  EXPERIMENTAL_FEATURES: ${{ format('{0} {1} {2}', vars.RUST_PROD_FEATURES, vars.RUST_TEST_FEATURES, vars.RUST_EXPERIMENTAL_FEATURES) }}
+  RUST_LOG: ${{ vars.RUST_LOG }}
+  RUST_BACKTRACE: ${{ vars.RUST_BACKTRACE }}
+  RUST_LIB_BACKTRACE: ${{ vars.RUST_LIB_BACKTRACE }}
+  COLORBT_SHOW_HIDDEN: ${{ vars.COLORBT_SHOW_HIDDEN }}
+  CARGO_INCREMENTAL: ${{ vars.CARGO_INCREMENTAL }}
+
 jobs:
   # Build the docker image used by the tests.
   #
@@ -85,10 +95,7 @@ jobs:
 
   # Run all the zebra tests, including tests that are ignored by default.
   #
-  # - We run all the tests behind the `getblocktemplate-rpcs` feature as a separated step.
   # - We activate the gRPC feature to avoid recompiling `zebrad`, but we don't actually run any gRPC tests.
-  #
-  # TODO: turn this test and the getblocktemplate test into a matrix, so the jobs use exactly the same diagnostics settings
   test-all:
     name: Test all
     runs-on: ubuntu-latest-xl
@@ -104,32 +111,16 @@ jobs:
       # Run unit, basic acceptance tests, and ignored tests, only showing command output if the test fails.
       #
       # If some tests hang, add "-- --nocapture" for just that test, or for all the tests.
+      #
+      # TODO: move this test command into entrypoint.sh
+      #       add a separate experimental workflow job if this job is slow
       - name: Run zebrad tests
         run: |
-          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }}
-          docker run -e NETWORK --name zebrad-tests --tty ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} cargo test --locked --release --features "lightwalletd-grpc-tests" --workspace -- --include-ignored
-        env:
-          NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }}
-
-  # zebrad tests without cached state with `getblocktemplate-rpcs` feature
-  #
-  # Same as above but we run all the tests behind the `getblocktemplate-rpcs` feature.
-  test-all-getblocktemplate-rpcs:
-    name: Test all with getblocktemplate-rpcs feature
-    runs-on: ubuntu-latest-xl
-    needs: build
-    steps:
-      - uses: r7kamura/rust-problem-matchers@v1.4.0
-
-      - name: Inject slug/short variables
-        uses: rlespinasse/github-slug-action@v4
-        with:
-          short-length: 7
-
-      - name: Run zebrad tests
-        run: |
-          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }}
-          docker run -e NETWORK --name zebrad-tests --tty -e ${{ inputs.network || vars.ZCASH_NETWORK }} ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} cargo test --locked --release --features "lightwalletd-grpc-tests getblocktemplate-rpcs" --workspace -- --include-ignored
+          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
+          docker run -e NETWORK --name zebrad-tests --tty ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "${{ env.TEST_FEATURES }}" --workspace -- --include-ignored
+          if [[ -n "${{ vars.RUST_EXPERIMENTAL_FEATURES }}" ]]; then
+            docker run -e NETWORK --name zebrad-tests --tty ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "${{ env.EXPERIMENTAL_FEATURES }} " --workspace -- --include-ignored
+          fi
         env:
           NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }}
 
@@ -141,7 +132,7 @@ jobs:
   #
   # Also, we don't want to accidentally use the fake heights in other tests.
   #
-  # (The gRPC feature is a zebrad feature, so it isn't needed here.)
+  # (We activate the test features to avoid recompiling dependencies, but we don't actually run any gRPC tests.)
   test-fake-activation-heights:
     name: Test with fake activation heights
     runs-on: ubuntu-latest
@@ -154,17 +145,16 @@ jobs:
         with:
           short-length: 7
 
+      # TODO: move this test command into entrypoint.sh
       - name: Run tests with fake activation heights
         run: |
-          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }}
-          docker run -e NETWORK -e TEST_FAKE_ACTIVATION_HEIGHTS --name zebrad-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} cargo test --locked --release --package zebra-state --lib -- --nocapture --include-ignored with_fake_activation_heights
+          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
+          docker run -e NETWORK -e TEST_FAKE_ACTIVATION_HEIGHTS --name zebrad-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "${{ env.TEST_FEATURES }}" --package zebra-state --lib -- --nocapture --include-ignored with_fake_activation_heights
         env:
           TEST_FAKE_ACTIVATION_HEIGHTS: '1'
           NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }}
 
   # Test that Zebra syncs and checkpoints a few thousand blocks from an empty state.
-  #
-  # (We activate the gRPC feature to avoid recompiling `zebrad`, but we don't actually run any gRPC tests.)
   test-empty-sync:
     name: Test checkpoint sync from empty state
     runs-on: ubuntu-latest
@@ -177,16 +167,15 @@ jobs:
         with:
           short-length: 7
 
+      # TODO: move this test command into entrypoint.sh
       - name: Run zebrad large sync tests
         run: |
-          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }}
-          docker run -e NETWORK --name zebrad-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored sync_large_checkpoints_
+          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
+          docker run -e NETWORK --name zebrad-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "${{ env.TEST_FEATURES }}" --package zebrad --test acceptance -- --nocapture --include-ignored sync_large_checkpoints_
         env:
           NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }}
 
   # Test launching lightwalletd with an empty lightwalletd and Zebra state.
-  #
-  # (We activate the gRPC feature to avoid recompiling `zebrad`, but we don't actually run any gRPC tests.)
   test-lightwalletd-integration:
     name: Test integration with lightwalletd
     runs-on: ubuntu-latest
@@ -199,10 +188,11 @@ jobs:
         with:
           short-length: 7
 
+      # TODO: move this test command into entrypoint.sh
       - name: Run tests with empty lightwalletd launch
         run: |
-          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }}
-          docker run -e NETWORK -e ZEBRA_TEST_LIGHTWALLETD --name lightwalletd-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_integration
+          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
+          docker run -e NETWORK -e ZEBRA_TEST_LIGHTWALLETD --name lightwalletd-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "${{ env.TEST_FEATURES }}" --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_integration
         env:
           ZEBRA_TEST_LIGHTWALLETD: '1'
           NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }}

From fdf84fb095a597c955bb48ce77440297f6ae688a Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Tue, 17 Oct 2023 09:56:43 +1000
Subject: [PATCH 02/15] Remove duplicate OS job

---
 .github/workflows/ci-unit-tests-os.patch.yml |  9 +--------
 .github/workflows/ci-unit-tests-os.yml       | 16 ++++------------
 2 files changed, 5 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/ci-unit-tests-os.patch.yml b/.github/workflows/ci-unit-tests-os.patch.yml
index 2d126cba054..549f1809a51 100644
--- a/.github/workflows/ci-unit-tests-os.patch.yml
+++ b/.github/workflows/ci-unit-tests-os.patch.yml
@@ -21,19 +21,12 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        # TODO: Windows was removed for now, see https://github.com/ZcashFoundation/zebra/issues/3801
-        # TODO: macOS tests were removed for now, see https://github.com/ZcashFoundation/zebra/issues/6824
         os: [ubuntu-latest]
         rust: [stable, beta]
-        features: ["", " --features getblocktemplate-rpcs"]
+        features: [""]
         exclude:
           - os: macos-latest
             rust: beta
-          - os: macos-latest
-            features: " --features getblocktemplate-rpcs"
-          - os: ubuntu-latest
-            rust: beta
-            features: " --features getblocktemplate-rpcs"
 
     steps:
       - run: 'echo "No build required"'
diff --git a/.github/workflows/ci-unit-tests-os.yml b/.github/workflows/ci-unit-tests-os.yml
index 00594afacd5..5059a0377f5 100644
--- a/.github/workflows/ci-unit-tests-os.yml
+++ b/.github/workflows/ci-unit-tests-os.yml
@@ -78,21 +78,15 @@ jobs:
         # TODO: macOS tests were removed for now, see https://github.com/ZcashFoundation/zebra/issues/6824
         os: [ubuntu-latest]
         rust: [stable, beta]
-        features: ["", " --features getblocktemplate-rpcs"]
+        # When vars.EXPERIMENTAL_FEATURES is empty this should become a single matrix item
+        features: ["", ${{ vars.EXPERIMENTAL_FEATURES }}]
         exclude:
-        # We're excluding macOS for the following reasons:
+        # We're excluding macOS beta for the following reasons:
         # - the concurrent macOS runner limit is much lower than the Linux limit
         # - macOS is slower than Linux, and shouldn't have a build or test difference with Linux
         # - macOS is a second-tier Zebra support platform
           - os: macos-latest
             rust: beta
-          - os: macos-latest
-            features: " --features getblocktemplate-rpcs"
-        # getblocktemplate-rpcs is an experimental feature, so we just need to test it on stable Rust
-        # beta is unlikely to fail just for this feature, and if it does, we can fix it when it reaches stable.
-          - os: ubuntu-latest
-            rust: beta
-            features: " --features getblocktemplate-rpcs"
 
     steps:
       - uses: actions/checkout@v4.1.0
@@ -119,7 +113,7 @@ jobs:
         #with:
         #  workspaces: ". -> C:\\zebra-target"
         with:
-          # Split the getblocktemplate-rpcs cache from the regular cache, to avoid linker errors.
+          # Split the experimental features cache from the regular cache, to avoid linker errors.
           # (These might be "disk full" errors, or they might be dependency resolution issues.)
           key: ${{ matrix.features }}
 
@@ -196,8 +190,6 @@ jobs:
 
       - name: Run zebrad large sync tests${{ matrix.features }}
         # Skip the entire step on Ubuntu and Windows, because the test would be skipped anyway due to ZEBRA_SKIP_NETWORK_TESTS
-        # Currently, this also skips large sync with `getblocktemplate-rpcs`,
-        # but that is already covered by the Docker tests.
         if: matrix.os == 'macos-latest'
         run: |
           cargo test ${{ matrix.features }} --release --verbose --package zebrad --test acceptance -- --nocapture --include-ignored sync_large_checkpoints_

From 698dd3333b7e4953d9d6ab7c26fc37660ab7eee2 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Tue, 17 Oct 2023 09:57:28 +1000
Subject: [PATCH 03/15] Rename experimental release build

---
 .github/workflows/release-binaries.yml | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
index 50c28acb7f8..95e87e23c6a 100644
--- a/.github/workflows/release-binaries.yml
+++ b/.github/workflows/release-binaries.yml
@@ -14,6 +14,12 @@ on:
     types:
       - released
 
+
+env:
+  EXPERIMENTAL_FEATURES: ${{ format('{0} {1}', vars.RUST_PROD_FEATURES, vars.RUST_EXPERIMENTAL_FEATURES) }}
+  RUST_LOG: ${{ vars.RUST_LOG }}
+  CARGO_INCREMENTAL: ${{ vars.CARGO_INCREMENTAL }}
+
 jobs:
   # Each time this workflow is executed, a build will be triggered to create a new image
   # with the corresponding tags using information from git
@@ -31,15 +37,15 @@ jobs:
     secrets: inherit
 
   # The image will be named `zebra:<semver>.experimental`
-  build-mining-testnet:
-    name: Build Release Testnet Mining Docker
+  build-experimental:
+    name: Build Experimental Features Release Docker
     uses: ./.github/workflows/sub-build-docker-image.yml
     with:
       dockerfile_path: ./docker/Dockerfile
       dockerfile_target: runtime
       image_name: zebra
       tag_suffix: .experimental
-      features: "default-release-binaries getblocktemplate-rpcs"
+      features: ${{ env.EXPERIMENTAL_FEATURES }}
       rust_log: info
     # This step needs access to Docker Hub secrets to run successfully
     secrets: inherit

From dc8dd163aad65cdbc604b6f81de136969440cbdf Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Tue, 17 Oct 2023 09:57:51 +1000
Subject: [PATCH 04/15] Make Rust features into GitHub repository variables

---
 .github/workflows/sub-build-docker-image.yml | 21 ++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/sub-build-docker-image.yml b/.github/workflows/sub-build-docker-image.yml
index c2988823cfd..81a9ac157a1 100644
--- a/.github/workflows/sub-build-docker-image.yml
+++ b/.github/workflows/sub-build-docker-image.yml
@@ -21,19 +21,17 @@ on:
       rust_lib_backtrace:
         required: false
         type: string
+      # defaults to: vars.RUST_LOG
       rust_log:
         required: false
         type: string
-        default: info
-      # keep these in sync with:
-      # https://github.com/ZcashFoundation/zebra/blob/main/docker/Dockerfile#L83
+      # defaults to: vars.RUST_PROD_FEATURES
       features:
         required: false
-        default: "default-release-binaries"
         type: string
+      # defaults to: vars.RUST_TEST_FEATURES and vars.RUST_PROD_FEATURES
       test_features:
         required: false
-        default: "lightwalletd-grpc-tests zebra-checkpoints"
         type: string
       tag_suffix:
         required: false
@@ -49,6 +47,13 @@ on:
         description: 'The image digest to be used on a caller workflow'
         value: ${{ jobs.build.outputs.image_digest }}
 
+
+env:
+  FEATURES: ${{ inputs.features || vars.RUST_PROD_FEATURES }}
+  TEST_FEATURES: ${{ inputs.test_features || vars.RUST_TEST_FEATURES }}
+  RUST_LOG: ${{ inputs.rust_log || vars.RUST_LOG }}
+  CARGO_INCREMENTAL: ${{ vars.CARGO_INCREMENTAL }}
+
 jobs:
   build:
     name: Build images
@@ -143,9 +148,9 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
             SHORT_SHA=${{ env.GITHUB_SHA_SHORT }}
-            RUST_LOG=${{ inputs.rust_log }}
-            FEATURES=${{ inputs.features }}
-            TEST_FEATURES=${{ inputs.test_features }}
+            RUST_LOG=${{ env.RUST_LOG }}
+            FEATURES=${{ env.FEATURES }}
+            TEST_FEATURES=${{ env.TEST_FEATURES }}
           push: true
           # Don't read from the cache if the caller disabled it.
           # https://docs.docker.com/engine/reference/commandline/buildx_build/#options

From a4e2c027f080fce270a928027a367dde04d3db3d Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Tue, 17 Oct 2023 10:07:41 +1000
Subject: [PATCH 05/15] Remove redundant features in entrypoint.sh

---
 docker/entrypoint.sh | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index 79dd0c3bbfa..1ffc0c0ba60 100755
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -43,15 +43,12 @@ set -o pipefail
 : "${TRACING_ENDPOINT_PORT:=3000}"
 # [rpc]
 : "${RPC_LISTEN_ADDR:=0.0.0.0}"
-# if ${RPC_PORT} is not set and ${FEATURES} contains getblocktemplate-rpcs,
-# set ${RPC_PORT} to the default value for the current network
+# if ${RPC_PORT} is not set, use the default value for the current network
 if [[ -z "${RPC_PORT}" ]]; then
-  if [[ " ${FEATURES} " =~ " getblocktemplate-rpcs " ]]; then
-    if [[ "${NETWORK}" = "Mainnet" ]]; then
-      : "${RPC_PORT:=8232}"
-    elif [[ "${NETWORK}" = "Testnet" ]]; then
-      : "${RPC_PORT:=18232}"
-    fi
+  if [[ "${NETWORK}" = "Mainnet" ]]; then
+    : "${RPC_PORT:=8232}"
+  elif [[ "${NETWORK}" = "Testnet" ]]; then
+    : "${RPC_PORT:=18232}"
   fi
 fi
 
@@ -319,16 +316,15 @@ case "$1" in
         run_cargo_test "${ENTRYPOINT_FEATURES}" "sending_transactions_using_lightwalletd"
 
       # These tests use mining code, but don't use gRPC.
-      # We add the mining feature here because our other code needs to pass tests without it.
       elif [[ "${TEST_GET_BLOCK_TEMPLATE}" -eq "1" ]]; then
         # Starting with a cached Zebra tip, test getting a block template from Zebra's RPC server.
         check_directory_files "${ZEBRA_CACHED_STATE_DIR}"
-        run_cargo_test "getblocktemplate-rpcs,${ENTRYPOINT_FEATURES}" "get_block_template"
+        run_cargo_test "${ENTRYPOINT_FEATURES}" "get_block_template"
 
       elif [[ "${TEST_SUBMIT_BLOCK}" -eq "1" ]]; then
         # Starting with a cached Zebra tip, test sending a block to Zebra's RPC port.
         check_directory_files "${ZEBRA_CACHED_STATE_DIR}"
-        run_cargo_test "getblocktemplate-rpcs,${ENTRYPOINT_FEATURES}" "submit_block"
+        run_cargo_test "${ENTRYPOINT_FEATURES}" "submit_block"
 
       else
           exec "$@"

From efb260b67eb6fbc8693354ce282b704bae154375 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Tue, 17 Oct 2023 10:09:20 +1000
Subject: [PATCH 06/15] Remove a dependency on a deleted job

---
 .github/workflows/ci-unit-tests-docker.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci-unit-tests-docker.yml b/.github/workflows/ci-unit-tests-docker.yml
index 002f434aa6e..cf802dc5d5d 100644
--- a/.github/workflows/ci-unit-tests-docker.yml
+++ b/.github/workflows/ci-unit-tests-docker.yml
@@ -241,7 +241,7 @@ jobs:
     #
     # This list is for reliable tests that are run on the `main` branch.
     # Testnet jobs are not in this list, because we expect testnet to fail occasionally.
-    needs: [ test-all, test-all-getblocktemplate-rpcs, test-fake-activation-heights, test-empty-sync, test-lightwalletd-integration, test-configuration-file, test-zebra-conf-path ]
+    needs: [ test-all, test-fake-activation-heights, test-empty-sync, test-lightwalletd-integration, test-configuration-file, test-zebra-conf-path ]
     # Only open tickets for failed scheduled jobs, manual workflow runs, or `main` branch merges.
     # (PR statuses are already reported in the PR jobs list, and checked by Mergify.)
     # TODO: if a job times out, we want to create a ticket. Does failure() do that? Or do we need cancelled()?

From fe38704c60ab27ba8a5378c53ce9823b3d8f7fa5 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Tue, 17 Oct 2023 10:11:56 +1000
Subject: [PATCH 07/15] Fix syntax of array

---
 .github/workflows/ci-unit-tests-os.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci-unit-tests-os.yml b/.github/workflows/ci-unit-tests-os.yml
index 5059a0377f5..595d5926e96 100644
--- a/.github/workflows/ci-unit-tests-os.yml
+++ b/.github/workflows/ci-unit-tests-os.yml
@@ -79,7 +79,7 @@ jobs:
         os: [ubuntu-latest]
         rust: [stable, beta]
         # When vars.EXPERIMENTAL_FEATURES is empty this should become a single matrix item
-        features: ["", ${{ vars.EXPERIMENTAL_FEATURES }}]
+        features: ${{ ["", vars.EXPERIMENTAL_FEATURES ] }}
         exclude:
         # We're excluding macOS beta for the following reasons:
         # - the concurrent macOS runner limit is much lower than the Linux limit

From a1323d47faa7b91212243803b9547ca74c8e93c7 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Tue, 17 Oct 2023 10:13:54 +1000
Subject: [PATCH 08/15] Another fix attempt

---
 .github/workflows/ci-unit-tests-os.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci-unit-tests-os.yml b/.github/workflows/ci-unit-tests-os.yml
index 595d5926e96..7953baf2fc8 100644
--- a/.github/workflows/ci-unit-tests-os.yml
+++ b/.github/workflows/ci-unit-tests-os.yml
@@ -79,7 +79,7 @@ jobs:
         os: [ubuntu-latest]
         rust: [stable, beta]
         # When vars.EXPERIMENTAL_FEATURES is empty this should become a single matrix item
-        features: ${{ ["", vars.EXPERIMENTAL_FEATURES ] }}
+        features: ${{ fromJSON(format('["", {0}]', vars.EXPERIMENTAL_FEATURES)) }}
         exclude:
         # We're excluding macOS beta for the following reasons:
         # - the concurrent macOS runner limit is much lower than the Linux limit

From 64cf41d55ab557935667370a99439734f50211a1 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Wed, 18 Oct 2023 18:16:34 +1000
Subject: [PATCH 09/15] Undo some accidental merge overwrites

---
 .github/workflows/ci-unit-tests-docker.yml | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/ci-unit-tests-docker.yml b/.github/workflows/ci-unit-tests-docker.yml
index cf802dc5d5d..68f708827ef 100644
--- a/.github/workflows/ci-unit-tests-docker.yml
+++ b/.github/workflows/ci-unit-tests-docker.yml
@@ -116,10 +116,10 @@ jobs:
       #       add a separate experimental workflow job if this job is slow
       - name: Run zebrad tests
         run: |
-          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
-          docker run -e NETWORK --name zebrad-tests --tty ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "${{ env.TEST_FEATURES }}" --workspace -- --include-ignored
+          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }}
+          docker run -e NETWORK --name zebrad-tests --tty${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} cargo test --locked --release --features "${{ env.TEST_FEATURES }}" --workspace -- --include-ignored
           if [[ -n "${{ vars.RUST_EXPERIMENTAL_FEATURES }}" ]]; then
-            docker run -e NETWORK --name zebrad-tests --tty ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "${{ env.EXPERIMENTAL_FEATURES }} " --workspace -- --include-ignored
+            docker run -e NETWORK --name zebrad-tests --tty ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} cargo test --locked --release --features "${{ env.EXPERIMENTAL_FEATURES }} " --workspace -- --include-ignored
           fi
         env:
           NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }}
@@ -148,8 +148,8 @@ jobs:
       # TODO: move this test command into entrypoint.sh
       - name: Run tests with fake activation heights
         run: |
-          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
-          docker run -e NETWORK -e TEST_FAKE_ACTIVATION_HEIGHTS --name zebrad-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "${{ env.TEST_FEATURES }}" --package zebra-state --lib -- --nocapture --include-ignored with_fake_activation_heights
+          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }}
+          docker run -e NETWORK -e TEST_FAKE_ACTIVATION_HEIGHTS --name zebrad-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} cargo test --locked --release --features "${{ env.TEST_FEATURES }}" --package zebra-state --lib -- --nocapture --include-ignored with_fake_activation_heights
         env:
           TEST_FAKE_ACTIVATION_HEIGHTS: '1'
           NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }}
@@ -170,8 +170,8 @@ jobs:
       # TODO: move this test command into entrypoint.sh
       - name: Run zebrad large sync tests
         run: |
-          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
-          docker run -e NETWORK --name zebrad-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "${{ env.TEST_FEATURES }}" --package zebrad --test acceptance -- --nocapture --include-ignored sync_large_checkpoints_
+          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }}
+          docker run -e NETWORK --name zebrad-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} cargo test --locked --release --features "${{ env.TEST_FEATURES }}" --package zebrad --test acceptance -- --nocapture --include-ignored sync_large_checkpoints_
         env:
           NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }}
 
@@ -191,8 +191,8 @@ jobs:
       # TODO: move this test command into entrypoint.sh
       - name: Run tests with empty lightwalletd launch
         run: |
-          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
-          docker run -e NETWORK -e ZEBRA_TEST_LIGHTWALLETD --name lightwalletd-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "${{ env.TEST_FEATURES }}" --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_integration
+          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }}
+          docker run -e NETWORK -e ZEBRA_TEST_LIGHTWALLETD --name lightwalletd-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} cargo test --locked --release --features "${{ env.TEST_FEATURES }}" --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_integration
         env:
           ZEBRA_TEST_LIGHTWALLETD: '1'
           NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }}

From 238395ffa943d05a5ce7c7f7b0b0ab7ba4219d13 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Wed, 18 Oct 2023 18:17:41 +1000
Subject: [PATCH 10/15] Add missing space

---
 .github/workflows/ci-unit-tests-docker.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci-unit-tests-docker.yml b/.github/workflows/ci-unit-tests-docker.yml
index 68f708827ef..c8395365963 100644
--- a/.github/workflows/ci-unit-tests-docker.yml
+++ b/.github/workflows/ci-unit-tests-docker.yml
@@ -117,7 +117,7 @@ jobs:
       - name: Run zebrad tests
         run: |
           docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }}
-          docker run -e NETWORK --name zebrad-tests --tty${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} cargo test --locked --release --features "${{ env.TEST_FEATURES }}" --workspace -- --include-ignored
+          docker run -e NETWORK --name zebrad-tests --tty ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} cargo test --locked --release --features "${{ env.TEST_FEATURES }}" --workspace -- --include-ignored
           if [[ -n "${{ vars.RUST_EXPERIMENTAL_FEATURES }}" ]]; then
             docker run -e NETWORK --name zebrad-tests --tty ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} cargo test --locked --release --features "${{ env.EXPERIMENTAL_FEATURES }} " --workspace -- --include-ignored
           fi

From 4b2c299c6f05f3ab98aae49ecbcc69d39baa7ef9 Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Thu, 19 Oct 2023 07:55:51 +1000
Subject: [PATCH 11/15] Explain how default is implemented

---
 .github/workflows/sub-build-docker-image.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sub-build-docker-image.yml b/.github/workflows/sub-build-docker-image.yml
index 81a9ac157a1..6356563a22e 100644
--- a/.github/workflows/sub-build-docker-image.yml
+++ b/.github/workflows/sub-build-docker-image.yml
@@ -29,7 +29,7 @@ on:
       features:
         required: false
         type: string
-      # defaults to: vars.RUST_TEST_FEATURES and vars.RUST_PROD_FEATURES
+      # defaults to: vars.RUST_TEST_FEATURES (and entrypoint.sh adds vars.RUST_PROD_FEATURES)
       test_features:
         required: false
         type: string

From 4497b185c0704059dbd3c676ec6753bf55bca53b Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Thu, 19 Oct 2023 07:57:10 +1000
Subject: [PATCH 12/15] Fix missing --features and quoting

---
 .github/workflows/ci-unit-tests-os.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci-unit-tests-os.yml b/.github/workflows/ci-unit-tests-os.yml
index 7953baf2fc8..a70121fbff4 100644
--- a/.github/workflows/ci-unit-tests-os.yml
+++ b/.github/workflows/ci-unit-tests-os.yml
@@ -79,7 +79,7 @@ jobs:
         os: [ubuntu-latest]
         rust: [stable, beta]
         # When vars.EXPERIMENTAL_FEATURES is empty this should become a single matrix item
-        features: ${{ fromJSON(format('["", {0}]', vars.EXPERIMENTAL_FEATURES)) }}
+        features: ${{ fromJSON(format('["", "--features {0}"]', vars.EXPERIMENTAL_FEATURES)) }}
         exclude:
         # We're excluding macOS beta for the following reasons:
         # - the concurrent macOS runner limit is much lower than the Linux limit

From d14f5912ead31111f933fea4cf0f8258887ab4cc Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Thu, 19 Oct 2023 08:32:18 +1000
Subject: [PATCH 13/15] We can fix this later

---
 .github/workflows/ci-unit-tests-os.yml | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci-unit-tests-os.yml b/.github/workflows/ci-unit-tests-os.yml
index a70121fbff4..695454e6090 100644
--- a/.github/workflows/ci-unit-tests-os.yml
+++ b/.github/workflows/ci-unit-tests-os.yml
@@ -78,8 +78,11 @@ jobs:
         # TODO: macOS tests were removed for now, see https://github.com/ZcashFoundation/zebra/issues/6824
         os: [ubuntu-latest]
         rust: [stable, beta]
-        # When vars.EXPERIMENTAL_FEATURES is empty this should become a single matrix item
-        features: ${{ fromJSON(format('["", "--features {0}"]', vars.EXPERIMENTAL_FEATURES)) }}
+        # TODO: When vars.EXPERIMENTAL_FEATURES has features in it, add it here.
+        #       Or work out a way to trim the space from the variable: GitHub doesn't allow empty variables.
+        #       Or use `default` for the empty feature set and EXPERIMENTAL_FEATURES, and update the branch protection rules.
+        #features: ${{ fromJSON(format('["", "{0}"]', vars.EXPERIMENTAL_FEATURES)) }}
+        features: [""]
         exclude:
         # We're excluding macOS beta for the following reasons:
         # - the concurrent macOS runner limit is much lower than the Linux limit
@@ -184,7 +187,7 @@ jobs:
       # If some tests hang, add "-- --nocapture" for just that test, or for all the tests.
       - name: Run tests${{ matrix.features }}
         run: |
-          cargo test ${{ matrix.features }} --release --verbose --workspace
+          cargo test --features "${{ matrix.features }}" --release --verbose --workspace
 
       # Explicitly run any tests that are usually #[ignored]
 
@@ -192,7 +195,7 @@ jobs:
         # Skip the entire step on Ubuntu and Windows, because the test would be skipped anyway due to ZEBRA_SKIP_NETWORK_TESTS
         if: matrix.os == 'macos-latest'
         run: |
-          cargo test ${{ matrix.features }} --release --verbose --package zebrad --test acceptance -- --nocapture --include-ignored sync_large_checkpoints_
+          cargo test --features "${{ matrix.features }}" --release --verbose --package zebrad --test acceptance -- --nocapture --include-ignored sync_large_checkpoints_
 
   # Install Zebra with lockfile dependencies, with no caching and default features
   install-from-lockfile-no-cache:

From 6b46230ee9dbc2781c725e49e781a2a0b3e0931e Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Thu, 19 Oct 2023 09:31:52 +1000
Subject: [PATCH 14/15] Use vars directly in with: blocks

---
 .github/workflows/release-binaries.yml | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
index 95e87e23c6a..f0b60acbad5 100644
--- a/.github/workflows/release-binaries.yml
+++ b/.github/workflows/release-binaries.yml
@@ -14,12 +14,6 @@ on:
     types:
       - released
 
-
-env:
-  EXPERIMENTAL_FEATURES: ${{ format('{0} {1}', vars.RUST_PROD_FEATURES, vars.RUST_EXPERIMENTAL_FEATURES) }}
-  RUST_LOG: ${{ vars.RUST_LOG }}
-  CARGO_INCREMENTAL: ${{ vars.CARGO_INCREMENTAL }}
-
 jobs:
   # Each time this workflow is executed, a build will be triggered to create a new image
   # with the corresponding tags using information from git
@@ -32,7 +26,8 @@ jobs:
       dockerfile_path: ./docker/Dockerfile
       dockerfile_target: runtime
       image_name: zebra
-      rust_log: info
+      features: ${{ vars.RUST_PROD_FEATURES }}
+      rust_log: ${{ vars.RUST_LOG }}
     # This step needs access to Docker Hub secrets to run successfully
     secrets: inherit
 
@@ -45,15 +40,15 @@ jobs:
       dockerfile_target: runtime
       image_name: zebra
       tag_suffix: .experimental
-      features: ${{ env.EXPERIMENTAL_FEATURES }}
-      rust_log: info
+      features: ${{ format('{0} {1}', vars.RUST_PROD_FEATURES, vars.RUST_EXPERIMENTAL_FEATURES) }}
+      rust_log: ${{ vars.RUST_LOG }}
     # This step needs access to Docker Hub secrets to run successfully
     secrets: inherit
 
   failure-issue:
     name: Open or update issues for release binaries failures
     # When a new job is added to this workflow, add it to this list.
-    needs: [ build, build-mining-testnet ]
+    needs: [ build, build-experimental ]
     # Open tickets for any failed build in this workflow.
     if: failure() || cancelled()
     runs-on: ubuntu-latest

From 3d25e519b8cff7483960e0091617b6e702e2b1ce Mon Sep 17 00:00:00 2001
From: teor <teor@riseup.net>
Date: Fri, 20 Oct 2023 07:07:34 +1000
Subject: [PATCH 15/15] Use correct features for fake activation heights test

---
 .github/workflows/ci-unit-tests-docker.yml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci-unit-tests-docker.yml b/.github/workflows/ci-unit-tests-docker.yml
index 5655fb6ad23..5f90fbf2184 100644
--- a/.github/workflows/ci-unit-tests-docker.yml
+++ b/.github/workflows/ci-unit-tests-docker.yml
@@ -119,8 +119,9 @@ jobs:
         run: |
           docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }}
           docker run -e NETWORK --name zebrad-tests --tty ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} cargo test --locked --release --features "${{ env.TEST_FEATURES }}" --workspace -- --include-ignored
-          if [[ -n "${{ vars.RUST_EXPERIMENTAL_FEATURES }}" ]]; then
-            docker run -e NETWORK --name zebrad-tests --tty ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} cargo test --locked --release --features "${{ env.EXPERIMENTAL_FEATURES }} " --workspace -- --include-ignored
+          # Currently GitHub doesn't allow empty variables
+          if [[ -n "${{ vars.RUST_EXPERIMENTAL_FEATURES }}" && "${{ vars.RUST_EXPERIMENTAL_FEATURES }}" != " " ]]; then
+            docker run -e NETWORK --name zebrad-tests-experimental --tty ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} cargo test --locked --release --features "${{ env.EXPERIMENTAL_FEATURES }} " --workspace -- --include-ignored
           fi
         env:
           NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }}
@@ -148,10 +149,11 @@ jobs:
           short-length: 7
 
       # TODO: move this test command into entrypoint.sh
+      #       make sure that at least one test runs, and that it doesn't skip itself due to the environmental variable
       - name: Run tests with fake activation heights
         run: |
           docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }}
-          docker run -e NETWORK -e TEST_FAKE_ACTIVATION_HEIGHTS --name zebrad-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} cargo test --locked --release --features "${{ env.TEST_FEATURES }}" --package zebra-state --lib -- --nocapture --include-ignored with_fake_activation_heights
+          docker run -e NETWORK -e TEST_FAKE_ACTIVATION_HEIGHTS --name zebrad-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} cargo test --locked --release --features "zebra-test" --package zebra-state --lib -- --nocapture --include-ignored with_fake_activation_heights
         env:
           TEST_FAKE_ACTIVATION_HEIGHTS: '1'
           NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }}