Merge branch 'main' of https://github.com/risingwavelabs/risingwave i…

…nto li0k/batch_predicate_pushdown
risingwavelabs · Dec 16, 2024 · 30d2aaf · 30d2aaf
2 parents 571e85e + 492e2d7
commit 30d2aaf
Show file tree

Hide file tree

Showing 39 changed files with 1,926 additions and 1,333 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -143,9 +143,9 @@ icelake = { git = "https://github.com/risingwavelabs/icelake.git", rev = "0ec44f
     "prometheus",
 ] }
 # branch dev-rebase-main-20241030
-iceberg = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "e28726443a57028f7c7df11d6d385470dc484d46" }
-iceberg-catalog-rest = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "e28726443a57028f7c7df11d6d385470dc484d46" }
-iceberg-catalog-glue = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "e28726443a57028f7c7df11d6d385470dc484d46" }
+iceberg = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "53f786fb2141b51d10a173cbcb5595edd5aa52a6" }
+iceberg-catalog-rest = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "53f786fb2141b51d10a173cbcb5595edd5aa52a6" }
+iceberg-catalog-glue = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "53f786fb2141b51d10a173cbcb5595edd5aa52a6" }
 opendal = "0.49"
 # used only by arrow-udf-flight
 arrow-flight = "53"

diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml
@@ -110,7 +110,8 @@ services:
     volumes:
       - ..:/risingwave
 
-  pg-mysql-backend-test-env:
+  # Standard environment for CI, including MySQL and Postgres for metadata.
+  ci-standard-env:
     image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240911
     depends_on:
       - mysql

diff --git a/ci/workflows/main-cron.yml b/ci/workflows/main-cron.yml
@@ -22,18 +22,19 @@ sql-backend: &sql-backend
           backend: "postgres"
           # PGPASSWORD=postgres psql -h db -p 5432 -U postgres -d rwmeta
           endpoint: "postgres://postgres:postgres@db:5432/rwmeta"
-      - with:
-          backend: "mysql"
-          # mysql -h mysql -P 3306 -u root -p123456 -D rwmeta
-          endpoint: "mysql://root:123456@mysql:3306/rwmeta"
+      # Temporarily disable tests for mysql backend as there are unresolved issues.
+      # - with:
+      #     backend: "mysql"
+      #     # mysql -h mysql-meta -P 3306 -u root -p123456 -D rwmeta
+      #     endpoint: "mysql://root:123456@mysql-meta:3306/rwmeta"
   env:
     RISEDEV_SQL_ENDPOINT: "{{matrix.endpoint}}"
 
 docker-compose-common: &docker-compose-common
     config: ci/docker-compose.yml
     mount-buildkite-agent: true
     propagate-environment: true
-    run: rw-build-env
+    run: ci-standard-env
 
 steps:
   - label: "build"
@@ -93,89 +94,65 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
-  - label: "end-to-end test (release, {{matrix.backend}} backend)"
-    key: "e2e-test-release"
-    <<: *sql-backend
-    command: "ci/scripts/cron-e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-e2e-test"
-      || build.env("CI_STEPS") =~ /(^|,)e2e-tests?(,|$$)/
-    depends_on:
-      - "build"
-      - "build-other"
-      - "docslt"
-    plugins:
-      - docker-compose#v5.5.0:
-          <<: *docker-compose-common
-          run: pg-mysql-backend-test-env
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 30
-    retry: *auto-retry
-
-  - label: "slow end-to-end test (release)"
-    key: "slow-e2e-test-release"
-    command: "ci/scripts/slow-e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-slow-e2e-tests"
-      || build.env("CI_STEPS") =~ /(^|,)slow-e2e-tests?(,|$$)/
-    depends_on:
-      - "build"
-      - "build-other"
-    plugins:
-      - docker-compose#v5.5.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 8
-    retry: *auto-retry
-
-  - label: "meta backup test (release)"
-    key: "e2e-meta-backup-test-release"
-    command: "ci/scripts/run-meta-backup-test.sh -p ci-release -m ci-3streaming-2serving-3fe"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-e2e-meta-backup-test"
-      || build.env("CI_STEPS") =~ /(^|,)e2e-tests?(,|$$)/
-    depends_on:
-      - "build"
-      - "build-other"
-      - "docslt"
-    plugins:
-      - docker-compose#v5.5.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 45
-    retry: *auto-retry
-
-  - label: "end-to-end test (parallel) (release)"
-    key: "e2e-test-release-parallel"
-    command: "ci/scripts/e2e-test-parallel.sh -p ci-release"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-e2e-parallel-tests"
-      || build.env("CI_STEPS") =~ /(^|,)e2e-parallel-tests?(,|$$)/
-    depends_on:
-      - "build"
-      - "docslt"
-    plugins:
-      - seek-oss/aws-sm#v2.3.2:
-          env:
-            BUILDKITE_ANALYTICS_TOKEN: buildkite-build-analytics-sqllogictest-token
-      - docker-compose#v5.5.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - test-collector#v1.0.0:
-          files: "*-junit.xml"
-          format: "junit"
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 13
-    retry: *auto-retry
+  - group: "end-to-end test (release)"
+    steps:
+    - label: "end-to-end test ({{matrix.backend}} backend)"
+      key: "e2e-test-release"
+      <<: *sql-backend
+      command: "ci/scripts/cron-e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe"
+      if: |
+        !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
+        || build.pull_request.labels includes "ci/run-e2e-test"
+        || build.env("CI_STEPS") =~ /(^|,)e2e-tests?(,|$$)/
+      depends_on:
+        - "build"
+        - "build-other"
+        - "docslt"
+      plugins:
+        - docker-compose#v5.5.0: *docker-compose-common
+        - ./ci/plugins/upload-failure-logs
+      timeout_in_minutes: 30
+      retry: *auto-retry
+
+    - label: "slow end-to-end test ({{matrix.backend}} backend)"
+      key: "slow-e2e-test-release"
+      <<: *sql-backend
+      command: "ci/scripts/slow-e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe"
+      if: |
+        !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
+        || build.pull_request.labels includes "ci/run-slow-e2e-tests"
+        || build.env("CI_STEPS") =~ /(^|,)slow-e2e-tests?(,|$$)/
+      depends_on:
+        - "build"
+        - "build-other"
+      plugins:
+        - docker-compose#v5.5.0: *docker-compose-common
+        - ./ci/plugins/upload-failure-logs
+      timeout_in_minutes: 8
+      retry: *auto-retry
+
+    - label: "end-to-end test (parallel, {{matrix.backend}} backend)"
+      key: "e2e-test-release-parallel"
+      <<: *sql-backend
+      command: "ci/scripts/e2e-test-parallel.sh -p ci-release"
+      if: |
+        !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
+        || build.pull_request.labels includes "ci/run-e2e-parallel-tests"
+        || build.env("CI_STEPS") =~ /(^|,)e2e-parallel-tests?(,|$$)/
+      depends_on:
+        - "build"
+        - "docslt"
+      plugins:
+        - seek-oss/aws-sm#v2.3.2:
+            env:
+              BUILDKITE_ANALYTICS_TOKEN: buildkite-build-analytics-sqllogictest-token
+        - docker-compose#v5.5.0: *docker-compose-common
+        - test-collector#v1.0.0:
+            files: "*-junit.xml"
+            format: "junit"
+        - ./ci/plugins/upload-failure-logs
+      timeout_in_minutes: 13
+      retry: *auto-retry
 
   - label: "end-to-end test (parallel, in-memory) (release)"
     key: "e2e-test-release-parallel-memory"
@@ -196,62 +173,84 @@ steps:
     timeout_in_minutes: 12
     retry: *auto-retry
 
-  - label: "end-to-end source test (release)"
-    key: "e2e-test-release-source"
-    command: "ci/scripts/e2e-source-test.sh -p ci-release"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-e2e-source-tests"
-      || build.env("CI_STEPS") =~ /(^|,)e2e-source-tests?(,|$$)/
-    depends_on:
-      - "build"
-      - "build-other"
-    plugins:
-      - docker-compose#v5.5.0:
-          run: source-test-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 15
-    retry: *auto-retry
+  - group: "end-to-end connector test (release)"
+    steps:
+    - label: "end-to-end source test ({{matrix.backend}} backend)"
+      key: "e2e-test-release-source"
+      <<: *sql-backend
+      command: "ci/scripts/e2e-source-test.sh -p ci-release"
+      if: |
+        !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
+        || build.pull_request.labels includes "ci/run-e2e-source-tests"
+        || build.env("CI_STEPS") =~ /(^|,)e2e-source-tests?(,|$$)/
+      depends_on:
+        - "build"
+        - "build-other"
+      plugins:
+        - docker-compose#v5.5.0:
+            <<: *docker-compose-common
+            run: source-test-env
+        - ./ci/plugins/upload-failure-logs
+      timeout_in_minutes: 15
+      retry: *auto-retry
+
+    - label: "end-to-end sink test ({{matrix.backend}} backend)"
+      key: "e2e-test-release-sink"
+      <<: *sql-backend
+      command: "ci/scripts/e2e-sink-test.sh -p ci-release"
+      if: |
+        !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
+        || build.pull_request.labels includes "ci/run-e2e-sink-tests"
+        || build.env("CI_STEPS") =~ /(^|,)e2e-sink-tests?(,|$$)/
+      depends_on:
+        - "build"
+        - "build-other"
+      plugins:
+        - docker-compose#v5.5.0:
+            <<: *docker-compose-common
+            run: sink-test-env
+        - ./ci/plugins/upload-failure-logs
+      timeout_in_minutes: 35
+      retry: *auto-retry
 
-  - label: "end-to-end sink test (release)"
-    key: "e2e-test-release-sink"
-    command: "ci/scripts/e2e-sink-test.sh -p ci-release"
+  - label: "fuzz test"
+    key: "fuzz-test"
+    command: "ci/scripts/cron-fuzz-test.sh -p ci-release"
     if: |
       !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-e2e-sink-tests"
-      || build.env("CI_STEPS") =~ /(^|,)e2e-sink-tests?(,|$$)/
+      || build.pull_request.labels includes "ci/run-sqlsmith-fuzzing-tests"
+      || build.env("CI_STEPS") =~ /(^|,)sqlsmith-fuzzing-tests?(,|$$)/
     depends_on:
       - "build"
-      - "build-other"
+      - "build-simulation"
     plugins:
+      - ./ci/plugins/swapfile
       - docker-compose#v5.5.0:
-          run: sink-test-env
+          run: rw-build-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 35
+    timeout_in_minutes: 20
     retry: *auto-retry
 
-  - label: "fuzz test"
-    key: "fuzz-test"
-    command: "ci/scripts/cron-fuzz-test.sh -p ci-release"
+  - label: "meta backup test (release)"
+    key: "e2e-meta-backup-test-release"
+    command: "ci/scripts/run-meta-backup-test.sh -p ci-release -m ci-3streaming-2serving-3fe"
     if: |
       !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-sqlsmith-fuzzing-tests"
-      || build.env("CI_STEPS") =~ /(^|,)sqlsmith-fuzzing-tests?(,|$$)/
+      || build.pull_request.labels includes "ci/run-e2e-meta-backup-test"
+      || build.env("CI_STEPS") =~ /(^|,)e2e-tests?(,|$$)/
     depends_on:
       - "build"
-      - "build-simulation"
+      - "build-other"
+      - "docslt"
     plugins:
-      - ./ci/plugins/swapfile
       - docker-compose#v5.5.0:
           run: rw-build-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 20
+    timeout_in_minutes: 45
     retry: *auto-retry
 
   # The timeout should be strictly more than timeout in `pull-request.yml`.