z

databendlabs · May 30, 2024 · f9cf4e4 · f9cf4e4
1 parent 24f3cdd
commit f9cf4e4
Show file tree

Hide file tree

Showing 3 changed files with 75 additions and 84 deletions.
diff --git a/.github/actions/artifact_download/action.yml b/.github/actions/artifact_download/action.yml
@@ -8,15 +8,15 @@ inputs:
     description: ""
     required: true
   category:
-    description: "default/hdfs"
+    description: "default/hdfs/udf"
     required: false
     default: default
   path:
     description: "default to ./target/${BUILD_PROFILE}/"
     required: false
     default: ""
   artifacts:
-    description: "Artifacts to download, only works with s3"
+    description: "Artifacts to download, only works with s3/gcs"
     required: false
     default: "meta,query"
 outputs:

diff --git a/.github/workflows/bindings.python.yml b/.github/workflows/bindings.python.yml
@@ -1,20 +1,18 @@
 name: Bindings Python
 
 on:
-  ## uncomment it when bendpy is enabled
-  workflow_dispatch:
-  # pull_request:
-  #   branches:
-  #     - main
-  #   paths:
-  #     - "src/**"
-  #     - ".github/workflows/bindings.python.yml"
-  # workflow_call:
-  #   inputs:
-  #     tag:
-  #       description: Tag to release
-  #       required: true
-  #       type: string
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "src/**"
+      - ".github/workflows/bindings.python.yml"
+  workflow_call:
+    inputs:
+      tag:
+        description: Tag to release
+        required: true
+        type: string
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -324,7 +324,7 @@ jobs:
         with:
           sha: ${{ github.sha }}
           target: ${{ matrix.target }}
-          category: defaulta
+          category: default
       - name: Pack Testsuite
         run: |
           target=${{ matrix.target }}
@@ -359,22 +359,22 @@ jobs:
         uses: actions/checkout@v4
         with:
           ref: ${{ needs.create_release.outputs.version }}
-      - name: Download binaries for usage
-        id: download_binaries
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          version="${{ needs.create_release.outputs.version }}"
-          declare -A platform_targets=( ["arm64"]="aarch64-unknown-linux-gnu" ["amd64"]="x86_64-unknown-linux-gnu")
-          mkdir -p ./distro/
-          for platform in ${!platform_targets[@]}; do
-              target=${platform_targets[$platform]}
-              gh release download ${version} --pattern "databend-${version}-${target}.tar.gz" --dir distro/
-              mkdir -p ./target/${target}/release
-              tar x -C ./target/${target}/release -f ./distro/databend-${version}-${target}.tar.gz --strip-components 1 bin/
-              mkdir -p ./distro/linux/${platform}
-              cp ./target/${target}/release/databend-* ./distro/linux/${platform}
-          done
+      - name: Download artifacts for x86_64
+        uses: ./.github/actions/artifact_download
+        with:
+          sha: ${{ github.sha }}
+          target: x86_64-unknown-linux-gnu
+          category: udf
+          artifacts: metactl,meta,query
+          path: distro/linux/amd64
+      - name: Download artifacts for aarch64
+        uses: ./.github/actions/artifact_download
+        with:
+          sha: ${{ github.sha }}
+          target: aarch64-unknown-linux-gnu
+          category: udf
+          artifacts: metactl,meta,query
+          path: distro/linux/arm64
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
       - uses: ./.github/actions/setup_docker
@@ -444,22 +444,22 @@ jobs:
         uses: actions/checkout@v4
         with:
           ref: ${{ needs.create_release.outputs.version }}
-      - name: Download binaries for usage
-        id: download_binaries
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          version="${{ needs.create_release.outputs.version }}"
-          declare -A platform_targets=( ["arm64"]="aarch64-unknown-linux-gnu" ["amd64"]="x86_64-unknown-linux-gnu")
-          mkdir -p ./distro/
-          for platform in ${!platform_targets[@]}; do
-              target=${platform_targets[$platform]}
-              gh release download ${version} --pattern "databend-${version}-${target}.tar.gz" --dir distro/
-              mkdir -p ./target/${target}/release
-              tar x -C ./target/${target}/release -f ./distro/databend-${version}-${target}.tar.gz --strip-components 1 bin/
-              mkdir -p ./distro/linux/${platform}
-              cp ./target/${target}/release/databend-* ./distro/linux/${platform}
-          done
+      - name: Download artifacts for x86_64
+        uses: ./.github/actions/artifact_download
+        with:
+          sha: ${{ github.sha }}
+          target: x86_64-unknown-linux-gnu
+          category: udf
+          artifacts: ${{ matrix.service }}
+          path: distro/linux/amd64
+      - name: Download artifacts for aarch64
+        uses: ./.github/actions/artifact_download
+        with:
+          sha: ${{ github.sha }}
+          target: aarch64-unknown-linux-gnu
+          category: udf
+          artifacts: ${{ matrix.service }}
+          path: distro/linux/arm64
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
       - uses: ./.github/actions/setup_docker
@@ -507,7 +507,7 @@ jobs:
           file: ./docker/${{ matrix.distro }}/${{ matrix.service }}.Dockerfile
 
   distribution:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, X64, Linux, 4c8g, gcp]
     needs: [create_release, build_default]
     strategy:
       matrix:
@@ -532,16 +532,14 @@ jobs:
         id: target
         run: |
           echo 'target=${{ matrix.arch }}-unknown-linux-gnu' >> $GITHUB_OUTPUT
-      - name: Download binaries for usage
-        id: download_binaries
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          target=${{ steps.target.outputs.target }}
-          version="${{ needs.create_release.outputs.version }}"
-          mkdir -p ./distro/
-          gh release download ${version} --pattern "databend-${version}-${target}.tar.gz" --dir distro/
-          tar x -C distro -f ./distro/databend-${version}-${target}.tar.gz
+      - name: Download artifacts
+        uses: ./.github/actions/artifact_download
+        with:
+          sha: ${{ github.sha }}
+          target: ${{ steps.target.outputs.target }}
+          category: udf
+          artifacts: metactl,meta,query
+          path: distro
       - name: Build Packages
         id: build_packages
         run: |
@@ -587,7 +585,7 @@ jobs:
           gpg_signing_key: ${{ secrets.GPG_KEY_DEB }}
 
   sha256sums:
-    needs: [create_release, linux, distribution]
+    needs: [create_release, publish, distribution]
     runs-on: ubuntu-latest
     steps:
       - name: checkout
@@ -627,25 +625,20 @@ jobs:
       target: all
 
   sqlsmith:
-    needs: [create_release, linux]
-    runs-on: ubuntu-latest
+    needs: [create_release, build_default]
+    runs-on: [self-hosted, X64, Linux, 4c8g, gcp]
     steps:
       - uses: actions/checkout@v4
         with:
           ref: ${{ needs.create_release.outputs.version }}
-      - name: Download artifact for release
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          version=${{ needs.create_release.outputs.version }}
-          target=x86_64-unknown-linux-gnu
-          mkdir -p ./distro/
-          mkdir -p ./target/release/
-          gh release download ${version} --pattern "databend-${version}-${target}.tar.gz" --dir distro/
-          gh release download ${version} --pattern "databend-testsuite-${version}-${target}.tar.gz" --dir distro/
-          tar x -C ./target/release -f ./distro/databend-${version}-${target}.tar.gz --strip-components 1 bin/
-          tar x -C ./target/release -f ./distro/databend-testsuite-${version}-${target}.tar.gz --strip-components 1 bin/
-          chmod +x ./target/release/databend-*
+      - name: Download artifacts
+        uses: ./.github/actions/artifact_download
+        with:
+          sha: ${{ github.sha }}
+          target: x86_64-unknown-linux-gnu
+          category: udf
+          artifacts: meta,query,sqlsmith
+          path: distro
       - name: Run sqlsmith
         timeout-minutes: 60
         shell: bash
@@ -657,20 +650,20 @@ jobs:
         with:
           name: test-sqlsmith
 
-  # bindings_python:
-  #   if: inputs.stable
-  #   needs: create_release
-  #   uses: ./.github/workflows/bindings.python.yml
-  #   secrets: inherit
-  #   with:
-  #     tag: ${{ needs.create_release.outputs.version }}
+  bindings_python:
+    if: inputs.stable
+    needs: create_release
+    uses: ./.github/workflows/bindings.python.yml
+    secrets: inherit
+    with:
+      tag: ${{ needs.create_release.outputs.version }}
 
   notify:
     runs-on: ubuntu-latest
     if: always()
     needs:
       - create_release
-      - linux
+      - publish
       - docker_all_in_one
       - docker_service
       - distribution