From b8b2c5f44a29c93a6acfd94d88594ff9833fc926 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 22 Mar 2024 16:35:28 +0100 Subject: [PATCH 1/6] Swift: fix local build on ARM macOS As pointed out in https://github.com/bazelbuild/bazel/issues/21768, bazel 7 moved multi-arch macOS toolchain support to `apple_support`, so building Swift (which must target `x86_64` for now) was broken. Internally in the codeql CLI we were unaffected as we use a custom bundled toolchain. This fixes building from this repository. --- .bazelrc | 2 +- MODULE.bazel | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bazelrc b/.bazelrc index 12232b4bbd68..341839d4b524 100644 --- a/.bazelrc +++ b/.bazelrc @@ -11,7 +11,7 @@ common --override_module=semmle_code=%workspace%/misc/bazel/semmle_code_stub build --repo_env=CC=clang --repo_env=CXX=clang++ build:linux --cxxopt=-std=c++20 -build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64 +build:macos --cxxopt=-std=c++20 --platforms=@apple_support//platforms:macos_x86_64 build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor try-import %workspace%/local.bazelrc diff --git a/MODULE.bazel b/MODULE.bazel index e8c79e8377fa..800d180ff8a0 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -12,7 +12,7 @@ local_path_override( ) # see https://registry.bazel.build/ for a list of available packages - +bazel_dep(name = "apple_support", version = "1.14.0") bazel_dep(name = "platforms", version = "0.0.8") bazel_dep(name = "rules_pkg", version = "0.9.1") bazel_dep(name = "rules_nodejs", version = "6.0.3") From 4ad597e9679fbb14ff8a7599a6b4f7365c475488 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 22 Mar 2024 17:00:11 +0100 Subject: [PATCH 2/6] Swift: add ARM build to CI, and other fixes Also: * removed the pretty much non-functioning bazel cache, will come back to it when we have a broader solution * replaced sharing the extractor pack via cache with uploading and downloading it --- .github/workflows/swift.yml | 25 ++++++--- swift/actions/build-and-test/action.yml | 53 +++---------------- .../download-extractor-pack/action.yml | 13 +++++ .../actions/run-integration-tests/action.yml | 2 +- swift/actions/run-ql-tests/action.yml | 2 +- swift/actions/share-extractor-pack/action.yml | 11 ---- .../actions/upload-extractor-pack/action.yml | 16 ++++++ 7 files changed, 56 insertions(+), 66 deletions(-) create mode 100644 swift/actions/download-extractor-pack/action.yml delete mode 100644 swift/actions/share-extractor-pack/action.yml create mode 100644 swift/actions/upload-extractor-pack/action.yml diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index 6956d31a3988..289f818367da 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -37,14 +37,23 @@ permissions: contents: read jobs: - # not using a matrix as you cannot depend on a specific job in a matrix, and we want to start linux checks - # without waiting for the macOS build - build-and-test-macos: + # not putting using a matrix as you cannot depend on a specific job in a matrix, and we want to start qltest and + # integration tests as soon as the corresponding build required has finished, without waiting for the slowest macOS + # build + build-and-test-macos-intel: if: github.repository_owner == 'github' - runs-on: macos-12-xl + runs-on: macos-13-large steps: - uses: actions/checkout@v4 - uses: ./swift/actions/build-and-test + build-and-test-macos-arm: + if: github.repository_owner == 'github' + runs-on: macos-13-xlarge + steps: + - uses: actions/checkout@v4 + - uses: ./swift/actions/build-and-test + with: + upload: false # use intel build for further tests build-and-test-linux: if: github.repository_owner == 'github' runs-on: ubuntu-latest-xl @@ -60,8 +69,8 @@ jobs: - uses: ./swift/actions/run-ql-tests qltests-macos: if: ${{ github.repository_owner == 'github' && github.event_name == 'pull_request' }} - needs: build-and-test-macos - runs-on: macos-12-xl + needs: build-and-test-macos-intel + runs-on: macos-13-xlarge steps: - uses: actions/checkout@v4 - uses: ./swift/actions/run-ql-tests @@ -74,8 +83,8 @@ jobs: - uses: ./swift/actions/run-integration-tests integration-tests-macos: if: ${{ github.repository_owner == 'github' && github.event_name == 'pull_request' }} - needs: build-and-test-macos - runs-on: macos-12-xl + needs: build-and-test-macos-intel + runs-on: macos-13-xlarge timeout-minutes: 60 steps: - uses: actions/checkout@v4 diff --git a/swift/actions/build-and-test/action.yml b/swift/actions/build-and-test/action.yml index 4553952f2c28..1dab9af7b595 100644 --- a/swift/actions/build-and-test/action.yml +++ b/swift/actions/build-and-test/action.yml @@ -1,5 +1,9 @@ name: Build Swift CodeQL pack description: Builds the Swift CodeQL pack +inputs: + upload: + description: Whether to upload the extractor pack + default: true runs: using: composite steps: @@ -7,48 +11,11 @@ runs: - uses: actions/setup-python@v4 with: python-version-file: 'swift/.python-version' - # FIXME: this is copy-pasted from .github/actions/cache-query-compilation, but we cannot factor it out to a common - # composite action because of https://github.com/actions/runner/issues/2009 (cache fails to save in the post action - # phase because its inputs were lost in the meantime) - # calculate the merge-base with main, in a way that works both on PRs and pushes to main. - - name: Calculate merge-base - shell: bash - if: ${{ github.event_name == 'pull_request' }} - env: - BASE_BRANCH: ${{ github.base_ref }} - run: | - MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ") - echo "merge_base=$MERGE_BASE" >> $GITHUB_ENV - - name: Restore read-only cache (PR) - if: ${{ github.event_name == 'pull_request' }} - uses: actions/cache/restore@v3 - with: - path: 'bazel-cache' - key: bazel-pr-${{ github.sha }} - restore-keys: | - bazel-${{ github.base_ref }}-${{ env.merge_base }} - bazel-${{ github.base_ref }}- - bazel-main- - - name: Fill cache (push) - if: ${{ github.event_name != 'pull_request' }} - uses: actions/cache@v3 - with: - path: 'bazel-cache' - key: bazel-${{ github.ref_name }}-${{ github.sha }} # just fill on main - restore-keys: | # restore the latest cache if the exact cache is unavailable, to speed up compilation. - bazel-${{ github.ref_name }}- - bazel-main- - name: Configure bazel shell: bash run: | - mkdir -p bazel-cache/{repository,disk} - echo build --repository_cache=bazel-cache/repository --disk_cache=bazel-cache/disk > local.bazelrc + echo > local.bazelrc echo test --test_output=errors >> local.bazelrc - # - name: Print unextracted entities - # shell: bash - # run: | - # bazel run //swift/extractor/print_unextracted - - uses: ./swift/actions/share-extractor-pack - name: Build Swift extractor shell: bash run: | @@ -63,10 +30,6 @@ runs: shell: bash run: | bazel test //swift/... - - name: Evict bazel cache - if: ${{ github.event_name != 'pull_request' }} - shell: bash - run: | - du -sh bazel-cache/* - find bazel-cache -atime +0 -type f -delete - du -sh bazel-cache/* + - name: Upload + if: inputs.upload == 'true' + uses: ./swift/actions/upload-extractor-pack diff --git a/swift/actions/download-extractor-pack/action.yml b/swift/actions/download-extractor-pack/action.yml new file mode 100644 index 000000000000..b0c9f5df42b4 --- /dev/null +++ b/swift/actions/download-extractor-pack/action.yml @@ -0,0 +1,13 @@ +name: Download Swift CodeQL pack +description: Downloads the Swift CodeQL pack +runs: + using: composite + steps: + - name: Download + uses: actions/download-artifact@v4 + with: + name: swift-extractor-pack-${{ runner.os }} + - name: Extract + shell: bash + run: | + zstd -cd swift-extractor-pack-${{ runner.os }}.tar.zst | tar xvf - diff --git a/swift/actions/run-integration-tests/action.yml b/swift/actions/run-integration-tests/action.yml index 2c6df4e2b513..d36b9565d232 100644 --- a/swift/actions/run-integration-tests/action.yml +++ b/swift/actions/run-integration-tests/action.yml @@ -3,7 +3,7 @@ description: Builds the Swift CodeQL pack runs: using: composite steps: - - uses: ./swift/actions/share-extractor-pack + - uses: ./swift/actions/download-extractor-pack - uses: actions/setup-python@v4 with: python-version-file: 'swift/.python-version' diff --git a/swift/actions/run-ql-tests/action.yml b/swift/actions/run-ql-tests/action.yml index da10997df448..ffa09faffb65 100644 --- a/swift/actions/run-ql-tests/action.yml +++ b/swift/actions/run-ql-tests/action.yml @@ -8,7 +8,7 @@ inputs: runs: using: composite steps: - - uses: ./swift/actions/share-extractor-pack + - uses: ./swift/actions/download-extractor-pack - uses: ./.github/actions/fetch-codeql - id: query-cache uses: ./.github/actions/cache-query-compilation diff --git a/swift/actions/share-extractor-pack/action.yml b/swift/actions/share-extractor-pack/action.yml deleted file mode 100644 index e9b7f9d5dacf..000000000000 --- a/swift/actions/share-extractor-pack/action.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: Build Swift CodeQL pack -description: Builds the Swift CodeQL pack -runs: - using: composite - steps: - # not using artifacts because of annoying https://github.com/actions/upload-artifact/issues/38 - - name: Mount cache for sharing extractor pack - uses: actions/cache@v3 - with: - path: swift/extractor-pack - key: extractor-pack-${{ github.run_id }}-${{ github.run_number }}-${{ runner.os }} diff --git a/swift/actions/upload-extractor-pack/action.yml b/swift/actions/upload-extractor-pack/action.yml new file mode 100644 index 000000000000..caaa6722c8cd --- /dev/null +++ b/swift/actions/upload-extractor-pack/action.yml @@ -0,0 +1,16 @@ +name: Upload Swift CodeQL pack +description: Uploads the Swift CodeQL pack +runs: + using: composite + steps: + - name: Compress + shell: bash + run: | + tar cf - swift/extractor-pack | zstd > swift-extractor-pack-${{ runner.os }}.tar.zst + - name: Upload + uses: actions/upload-artifact@v4 + with: + name: swift-extractor-pack-${{ runner.os }} + path: swift-extractor-pack-${{ runner.os }}.tar.zst + compression-level: 0 + retention-days: 3 From 246059160e35d1a1b02086c10d8b8a0b51797e83 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 25 Mar 2024 09:59:38 +0100 Subject: [PATCH 3/6] Swift: move running to intel The problems running on arm are concerning, but may be caused by not building with the internal macOS toolchain. We'll do separate work internally to investigate and possibly fix that. In the meantime, we can keep testing on intel as is currently done in main. --- .github/workflows/swift.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index 289f818367da..a6e0ad667e21 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -70,7 +70,7 @@ jobs: qltests-macos: if: ${{ github.repository_owner == 'github' && github.event_name == 'pull_request' }} needs: build-and-test-macos-intel - runs-on: macos-13-xlarge + runs-on: macos-13-large steps: - uses: actions/checkout@v4 - uses: ./swift/actions/run-ql-tests @@ -84,7 +84,7 @@ jobs: integration-tests-macos: if: ${{ github.repository_owner == 'github' && github.event_name == 'pull_request' }} needs: build-and-test-macos-intel - runs-on: macos-13-xlarge + runs-on: macos-13-large timeout-minutes: 60 steps: - uses: actions/checkout@v4 From 2b486311f698672fb5c11cedd7590b06cb5cc091 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 25 Mar 2024 11:51:44 +0100 Subject: [PATCH 4/6] Swift: move CI to macos 14 --- .github/workflows/swift.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index a6e0ad667e21..dce348884ac5 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -42,13 +42,13 @@ jobs: # build build-and-test-macos-intel: if: github.repository_owner == 'github' - runs-on: macos-13-large + runs-on: macos-14-large steps: - uses: actions/checkout@v4 - uses: ./swift/actions/build-and-test build-and-test-macos-arm: if: github.repository_owner == 'github' - runs-on: macos-13-xlarge + runs-on: macos-14-xlarge steps: - uses: actions/checkout@v4 - uses: ./swift/actions/build-and-test @@ -70,7 +70,7 @@ jobs: qltests-macos: if: ${{ github.repository_owner == 'github' && github.event_name == 'pull_request' }} needs: build-and-test-macos-intel - runs-on: macos-13-large + runs-on: macos-14-large steps: - uses: actions/checkout@v4 - uses: ./swift/actions/run-ql-tests @@ -84,7 +84,7 @@ jobs: integration-tests-macos: if: ${{ github.repository_owner == 'github' && github.event_name == 'pull_request' }} needs: build-and-test-macos-intel - runs-on: macos-13-large + runs-on: macos-14-large timeout-minutes: 60 steps: - uses: actions/checkout@v4 From e452ff678f58f3799e4084fc73d54d973d4a47c1 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 25 Mar 2024 17:25:46 +0100 Subject: [PATCH 5/6] Swift: update swift to 5.9 in integration tests --- swift/actions/run-integration-tests/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swift/actions/run-integration-tests/action.yml b/swift/actions/run-integration-tests/action.yml index d36b9565d232..31b84708494d 100644 --- a/swift/actions/run-integration-tests/action.yml +++ b/swift/actions/run-integration-tests/action.yml @@ -7,9 +7,9 @@ runs: - uses: actions/setup-python@v4 with: python-version-file: 'swift/.python-version' - - uses: swift-actions/setup-swift@65540b95f51493d65f5e59e97dcef9629ddf11bf + - uses: swift-actions/setup-swift@e1dca7c4a36344146bbc2803f0d538462477bb37 with: - swift-version: "5.8" + swift-version: 5.9 - uses: ./.github/actions/fetch-codeql - id: query-cache uses: ./.github/actions/cache-query-compilation From d0644d0b84aea113bdca83fa05266a03e1cea6e2 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 2 Apr 2024 10:51:42 +0200 Subject: [PATCH 6/6] Swift: remove running integration tests from CI They are now run internally by QLucie. --- .github/workflows/swift.yml | 30 ------------------ .../actions/run-integration-tests/action.yml | 31 ------------------- 2 files changed, 61 deletions(-) delete mode 100644 swift/actions/run-integration-tests/action.yml diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index dce348884ac5..3ae625608261 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -17,21 +17,6 @@ on: - main - rc/* - codeql-cli-* - push: - paths: - - "swift/**" - - "misc/bazel/**" - - "misc/codegen/**" - - "*.bazel*" - - .github/workflows/swift.yml - - .github/actions/** - - codeql-workspace.yml - - "!**/*.md" - - "!**/*.qhelp" - branches: - - main - - rc/* - - codeql-cli-* permissions: contents: read @@ -74,21 +59,6 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./swift/actions/run-ql-tests - integration-tests-linux: - if: github.repository_owner == 'github' - needs: build-and-test-linux - runs-on: ubuntu-latest-xl - steps: - - uses: actions/checkout@v4 - - uses: ./swift/actions/run-integration-tests - integration-tests-macos: - if: ${{ github.repository_owner == 'github' && github.event_name == 'pull_request' }} - needs: build-and-test-macos-intel - runs-on: macos-14-large - timeout-minutes: 60 - steps: - - uses: actions/checkout@v4 - - uses: ./swift/actions/run-integration-tests clang-format: if : ${{ github.event_name == 'pull_request' }} runs-on: ubuntu-latest diff --git a/swift/actions/run-integration-tests/action.yml b/swift/actions/run-integration-tests/action.yml deleted file mode 100644 index 31b84708494d..000000000000 --- a/swift/actions/run-integration-tests/action.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Build Swift CodeQL pack -description: Builds the Swift CodeQL pack -runs: - using: composite - steps: - - uses: ./swift/actions/download-extractor-pack - - uses: actions/setup-python@v4 - with: - python-version-file: 'swift/.python-version' - - uses: swift-actions/setup-swift@e1dca7c4a36344146bbc2803f0d538462477bb37 - with: - swift-version: 5.9 - - uses: ./.github/actions/fetch-codeql - - id: query-cache - uses: ./.github/actions/cache-query-compilation - with: - key: swift-integration - - name: Run integration tests - shell: bash - run: | - python swift/ql/integration-tests/runner.py --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" - env: - SEMMLE_DEBUG_TRACER: 10000 - - name: Upload test logs - if: ${{ always() }} - uses: actions/upload-artifact@v3 - with: - name: swift-integration-tests-logs-${{ runner.os }} - path: | - swift/ql/integration-tests/**/db/log - retention-days: 1