Squashed commit of the following:

commit b3c8e4e Author: xxchan <[email protected]> Date: Sat Apr 13 02:39:13 2024 +0800 fix Signed-off-by: xxchan <[email protected]> commit a575c3a Author: xxchan <[email protected]> Date: Sat Apr 13 02:38:15 2024 +0800 fix Signed-off-by: xxchan <[email protected]> commit df9f905 Author: xxchan <[email protected]> Date: Sat Apr 13 02:26:00 2024 +0800 update Signed-off-by: xxchan <[email protected]> commit b8859ca Author: xxchan <[email protected]> Date: Sat Apr 13 02:25:14 2024 +0800 fix Signed-off-by: xxchan <[email protected]> commit f001e31 Author: xxchan <[email protected]> Date: Sat Apr 13 02:23:06 2024 +0800 revert risedev Signed-off-by: xxchan <[email protected]> commit 33aa178 Merge: 530eb86 c98dfd5 Author: xxchan <[email protected]> Date: Sat Apr 13 02:22:13 2024 +0800 Merge branch 'xxchan/latin-tyrannosaurus' into xxchan/source-test commit 530eb86 Merge: 0b6f74c cf221e3 Author: xxchan <[email protected]> Date: Sat Apr 13 02:21:42 2024 +0800 Merge branch 'xxchan/latin-tyrannosaurus' into xxchan/source-test Signed-off-by: xxchan <[email protected]> commit 0b6f74c Author: xxchan <[email protected]> Date: Sat Apr 13 02:11:40 2024 +0800 sleep more? Signed-off-by: xxchan <[email protected]> commit 47c2c61 Author: xxchan <[email protected]> Date: Fri Apr 12 22:30:09 2024 +0800 sleep more Signed-off-by: xxchan <[email protected]> commit 1853adf Author: xxchan <[email protected]> Date: Fri Apr 12 21:59:31 2024 +0800 fix Signed-off-by: xxchan <[email protected]> commit 8711cf5 Merge: f1c0185 4f53f89 Author: xxchan <[email protected]> Date: Fri Apr 12 17:52:38 2024 +0800 Merge remote-tracking branch 'origin/main' into xxchan/source-test Signed-off-by: xxchan <[email protected]> commit f1c0185 Author: xxchan <[email protected]> Date: Fri Apr 12 17:52:12 2024 +0800 fix Signed-off-by: xxchan <[email protected]> commit e9e2dc4 Author: xxchan <[email protected]> Date: Fri Apr 12 16:49:20 2024 +0800 install rpk Signed-off-by: xxchan <[email protected]> commit e2f2a8e Author: xxchan <[email protected]> Date: Fri Apr 12 15:16:44 2024 +0800 f Signed-off-by: xxchan <[email protected]> commit bae6495 Merge: 58de398 4ac029c Author: xxchan <[email protected]> Date: Fri Apr 12 15:11:54 2024 +0800 Merge remote-tracking branch 'origin/main' into xxchan/source-test commit 58de398 Author: xxchan <[email protected]> Date: Fri Apr 12 15:11:49 2024 +0800 fix Signed-off-by: xxchan <[email protected]> commit 9ba9728 Author: xxchan <[email protected]> Date: Thu Apr 11 17:45:24 2024 +0800 fix Signed-off-by: xxchan <[email protected]> commit d8a2489 Author: xxchan <[email protected]> Date: Thu Apr 11 17:18:38 2024 +0800 fix Signed-off-by: xxchan <[email protected]> commit c4b4b16 Author: xxchan <[email protected]> Date: Thu Apr 11 17:16:25 2024 +0800 fix Signed-off-by: xxchan <[email protected]> commit 6dddbf3 Author: xxchan <[email protected]> Date: Thu Apr 11 17:15:00 2024 +0800 rename new to inline Signed-off-by: xxchan <[email protected]> commit cf50f5e Author: xxchan <[email protected]> Date: Thu Apr 11 17:04:54 2024 +0800 bump Signed-off-by: xxchan <[email protected]> commit 9ca9d55 Author: xxchan <[email protected]> Date: Thu Apr 11 16:00:07 2024 +0800 support user-managed kafka in risedev Signed-off-by: xxchan <[email protected]> commit d4d405d Author: xxchan <[email protected]> Date: Thu Apr 11 15:27:20 2024 +0800 update Signed-off-by: xxchan <[email protected]> commit 6ed6cfc Author: xxchan <[email protected]> Date: Thu Apr 11 10:21:25 2024 +0800 update commit f7a3dd5 Merge: 8b6c482 254ad0c Author: xxchan <[email protected]> Date: Thu Apr 11 09:45:50 2024 +0800 Merge remote-tracking branch 'origin/main' into xxchan/source-test commit 8b6c482 Author: xxchan <[email protected]> Date: Tue Apr 9 14:12:37 2024 +0800 update commit f280603 Author: xxchan <[email protected]> Date: Fri Apr 5 23:14:30 2024 +0800 add new source tests commit 0e3ace1 Author: xxchan <[email protected]> Date: Fri Apr 5 23:13:05 2024 +0800 revert unrelated change commit a3f3409 Author: xxchan <[email protected]> Date: Fri Apr 5 17:58:48 2024 +0800 fix commit 0a2ded0 Author: xxchan <[email protected]> Date: Fri Apr 5 17:27:45 2024 +0800 fix commit 8daa64d Author: xxchan <[email protected]> Date: Fri Apr 5 17:19:38 2024 +0800 debug commit 8628c1f Author: xxchan <[email protected]> Date: Fri Apr 5 17:15:32 2024 +0800 fix commit c8bde20 Author: xxchan <[email protected]> Date: Fri Apr 5 17:00:09 2024 +0800 ci: install risedev to ci image Signed-off-by: xxchan <[email protected]>
risingwavelabs · Apr 12, 2024 · 143d569 · 143d569
1 parent e11de32
commit 143d569
Show file tree

Hide file tree

Showing 17 changed files with 384 additions and 21 deletions.
diff --git a/Makefile.toml b/Makefile.toml
@@ -15,6 +15,7 @@ extend = [
   { path = "src/storage/backup/integration_tests/Makefile.toml" },
   { path = "src/java_binding/make-java-binding.toml" },
   { path = "src/stream/tests/integration_tests/integration_test.toml" },
+  { path = "e2e_test/source_inline/commands.toml" },
 ]
 
 env_files = ["./risedev-components.user.env"]

diff --git a/ci/Dockerfile b/ci/Dockerfile
@@ -34,8 +34,10 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-mo
 
 ENV PATH /root/.cargo/bin/:$PATH
 
-RUN rustup show
-RUN rustup default `rustup show active-toolchain | awk '{print $1}'`
+RUN rustup show && \
+    rustup default `rustup show active-toolchain | awk '{print $1}'` && \
+    rustup component add rustfmt llvm-tools-preview clippy && \
+    rustup target add wasm32-wasi
 
 RUN curl -sSL "https://github.com/bufbuild/buf/releases/download/v1.29.0/buf-$(uname -s)-$(uname -m).tar.gz" | \
     tar -xvzf - -C /usr/local --strip-components 1
@@ -45,18 +47,25 @@ RUN pip3 install pyarrow pytest
 
 # Install poetry
 RUN curl -sSL https://install.python-poetry.org | python3 -
-
-# add required rustup components
-RUN rustup component add rustfmt llvm-tools-preview clippy
-RUN rustup target add wasm32-wasi
+# Install rpk
+RUN if [ "$(uname -m)" = "amd64" ] || [ "$(uname -m)" = "x86_64" ]; then \
+        curl -LO https://github.com/redpanda-data/redpanda/releases/latest/download/rpk-linux-amd64.zip && \
+        unzip rpk-linux-amd64.zip -d ~/.local/bin/ && \
+        rm rpk-linux-amd64.zip; \
+    else \
+        curl -LO https://github.com/redpanda-data/redpanda/releases/latest/download/rpk-linux-arm64.zip && \
+        unzip rpk-linux-arm64.zip -d ~/.local/bin/ && \
+        rm rpk-linux-arm64.zip; \
+    fi
+ENV PATH /root/.local/bin:$PATH
 
 ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
 
 # install build tools
 RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
 RUN cargo binstall -y --no-symlinks cargo-llvm-cov cargo-nextest cargo-hakari cargo-sort cargo-cache cargo-audit \
     [email protected] \
-    sqllogictest-bin@0.19.1 \
+    sqllogictest-bin@0.20.0 \
     [email protected] \
     && cargo cache -a \
     && rm -rf "/root/.cargo/registry/index" \

diff --git a/ci/build-ci-image.sh b/ci/build-ci-image.sh
@@ -10,7 +10,7 @@ cat ../rust-toolchain
 # shellcheck disable=SC2155
 
 # REMEMBER TO ALSO UPDATE ci/docker-compose.yml
-export BUILD_ENV_VERSION=v20240405_1
+export BUILD_ENV_VERSION=v20240412
 
 export BUILD_TAG="public.ecr.aws/w1p7b4n3/rw-build-env:${BUILD_ENV_VERSION}"
 

diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml
@@ -71,7 +71,7 @@ services:
       retries: 5
 
   source-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240405_1
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240412
     depends_on:
       - mysql
       - db
@@ -84,7 +84,7 @@ services:
       - ..:/risingwave
 
   sink-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240405_1
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240412
     depends_on:
       - mysql
       - db
@@ -103,12 +103,12 @@ services:
 
 
   rw-build-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240405_1
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240412
     volumes:
       - ..:/risingwave
 
   ci-flamegraph-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240405_1
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240412
     # NOTE(kwannoel): This is used in order to permit
     # syscalls for `nperf` (perf_event_open),
     # so it can do CPU profiling.
@@ -119,7 +119,7 @@ services:
       - ..:/risingwave
 
   regress-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240405_1
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240412
     depends_on:
       db:
         condition: service_healthy

diff --git a/ci/scripts/e2e-source-test.sh b/ci/scripts/e2e-source-test.sh
@@ -168,6 +168,9 @@ echo "--- e2e, kafka alter source again"
 ./scripts/source/prepare_data_after_alter.sh 3
 risedev slt './e2e_test/source/basic/alter/kafka_after_new_data_2.slt'
 
+echo "--- e2e, inline test"
+risedev slt './e2e_test/source_inline/**/*.slt'
+
 echo "--- Run CH-benCHmark"
 risedev slt './e2e_test/ch_benchmark/batch/ch_benchmark.slt'
 risedev slt './e2e_test/ch_benchmark/streaming/*.slt'
diff --git a/e2e_test/source/README.md b/e2e_test/source/README.md
@@ -1,3 +1,7 @@
+> [!NOTE]
+>
+> Please write new tests according to the style in `e2e_test/source_inline`.
+
 Test in this directory needs some prior setup.
 
 See also `ci/scripts/e2e-source-test.sh`, and `scripts/source`

diff --git a/e2e_test/source_inline/README.md b/e2e_test/source_inline/README.md
@@ -0,0 +1,16 @@
+# "Inline" style source e2e tests
+
+Compared with prior source tests (`e2e_test/source`), tests in this directory are expected to be easy to run locally and easy to write.
+
+To run locally, use `risdev d` to start services (including external systems like Kafka and Postgres, or specify `user-managed` to use your own service).
+Then use `risedev slt` to run the tests, which will load the environment variables (ports, etc.)
+according to the services started by `risdev d`.
+
+```sh
+risedev slt 'e2e_test/source-inline/**/*.slt'
+```
+
+To write tests, please ensure each file is self-contained and does not depend on external scripts to setup the environment.
+Use `system` command to setup instead.
+
+Refer to https://github.com/risingwavelabs/risingwave/issues/12451#issuecomment-2051861048 for more details.
diff --git a/e2e_test/source_inline/commands.toml b/e2e_test/source_inline/commands.toml
@@ -0,0 +1,108 @@
+# This file contains commands used by the tests.
+
+[tasks.source-test-hook]
+private = true
+dependencies = ["check-risedev-env-file"]
+env_files = ["${PREFIX_CONFIG}/risedev-env"]
+
+# Note about the Kafka CLI tooling:
+# - Built-in Kafka console tools:
+#       Java based.
+#       Style example: kafka-topics.sh --bootstrap-server localhost:9092 --topic t --create
+#       Some limitations: cannot disable logging easily, cannot consume to end and then exit.
+# - kcat:
+#       C based (rdkafka)
+#       Some limitations: cannot do admin operations, only consume/produce.
+# - rpk:
+#       Golang based.
+#       Style example: RPK_BROKERS=localhost:9092 rpk topic create t
+[tasks.kafka-hook]
+private = true
+description = "Check if Kafka is started by RiseDev"
+dependencies = ["source-test-hook"]
+script = '''
+#!/usr/bin/env sh
+set -e
+
+if [ ! -d "${PREFIX_BIN}/kafka" ]; then
+    echo "Kafka is not installed in ${PREFIX_BIN}/kafka. Did you enable Kafka using $(tput setaf 4)\`./risedev configure\`$(tput sgr0)?"
+    exit 1
+fi
+
+# TODO: we may support risedev-env.override so that we can connect to a Kafka not started by risedev-dev.
+if [ -z "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" ]; then
+    echo "RISEDEV_KAFKA_BOOTSTRAP_SERVERS is not set in risedev-env file. Did you start Kafka using $(tput setaf 4)\`./risedev d\`$(tput sgr0)?"
+    exit 1
+fi
+'''
+
+[tasks.clean-kafka]
+category = "RiseDev - Test - Source Test - Kafka"
+description = "Delete all kafka topics."
+dependencies = ["kafka-hook"]
+command = "rpk"
+args = ["topic", "delete", "-r", "*"]
+
+[tasks.kafka-topics]
+category = "RiseDev - Test - Source Test - Kafka"
+dependencies = ["kafka-hook"]
+script = """
+#!/usr/bin/env sh
+set -e
+${PREFIX_BIN}/kafka/bin/kafka-topics.sh --bootstrap-server ${RISEDEV_KAFKA_BOOTSTRAP_SERVERS} "$@"
+"""
+
+[tasks.kafka-produce]
+category = "RiseDev - Test - Source Test - Kafka"
+dependencies = ["kafka-hook"]
+script = """
+#!/usr/bin/env sh
+set -e
+${PREFIX_BIN}/kafka/bin/kafka-console-producer.sh --bootstrap-server ${RISEDEV_KAFKA_BOOTSTRAP_SERVERS} "$@"
+"""
+
+[tasks.kafka-consume]
+category = "RiseDev - Test - Source Test - Kafka"
+dependencies = ["kafka-hook"]
+script = """
+#!/usr/bin/env sh
+set -e
+${PREFIX_BIN}/kafka/bin/kafka-console-consumer.sh --bootstrap-server ${RISEDEV_KAFKA_BOOTSTRAP_SERVERS} "$@"
+"""
+
+[tasks.kafka-consumer-groups]
+category = "RiseDev - Test - Source Test - Kafka"
+dependencies = ["kafka-hook"]
+script = """
+#!/usr/bin/env sh
+set -e
+${PREFIX_BIN}/kafka/bin/kafka-consumer-groups.sh --bootstrap-server ${RISEDEV_KAFKA_BOOTSTRAP_SERVERS} "$@"
+"""
+
+# rpk tools
+[tasks.rpk]
+category = "RiseDev - Test - Source Test - Kafka"
+dependencies = ["kafka-hook"]
+# check https://docs.redpanda.com/current/reference/rpk/rpk-x-options/ or rpk -X help/list for options
+script = """
+#!/usr/bin/env sh
+set -e
+
+if [ -z "$(which rpk)" ]; then
+    echo "rpk is not installed. Install it via https://docs.redpanda.com/current/get-started/rpk-install/"
+    exit 1
+fi
+
+rpk "$@"
+"""
+
+[tasks.redpanda-console]
+category = "RiseDev - Test - Source Test - Kafka"
+description = "Start Redpanda console (Kafka GUI) at localhost:8080."
+dependencies = ["kafka-hook"]
+script = '''
+#!/usr/bin/env sh
+set -e
+echo "$(tput setaf 2)Start Redpanda console at http://localhost:8080$(tput sgr0)"
+docker run --network host -e KAFKA_BROKERS=$RPK_BROKERS docker.redpanda.com/redpandadata/console:latest
+'''
diff --git a/e2e_test/source_inline/kafka/add_partition.slt b/e2e_test/source_inline/kafka/add_partition.slt
@@ -0,0 +1,74 @@
+# Note: control substitution on will force us to use "\\n" instead of "\n" in commands
+control substitution on
+
+system ok
+rpk topic create test_add_partition -p 3
+
+system ok
+cat <<EOF | rpk topic produce test_add_partition -f "%p %v\\n" -p 0
+0 {"x":"a"}
+1 {"x":"b"}
+2 {"x":"c"}
+EOF
+
+statement ok
+CREATE SOURCE s(x varchar)
+WITH(
+    ${RISEDEV_KAFKA_WITH_OPTIONS_COMMON},
+    topic = 'test_add_partition',
+    scan.startup.mode = 'earliest',
+) FORMAT PLAIN ENCODE JSON;
+
+statement ok
+CREATE MATERIALIZED VIEW mv AS SELECT * from s;
+
+query ?
+SELECT * FROM s order by x;
+----
+a
+b
+c
+
+sleep 2s
+
+query ?
+SELECT * FROM mv order by x;
+----
+a
+b
+c
+
+system ok
+rpk topic add-partitions test_add_partition --num 1
+
+system ok
+cat <<EOF | rpk topic produce test_add_partition -f "%p %v\\n" -p 0
+3 {"x":"d"}
+EOF
+
+
+query ?
+SELECT * FROM s order by x;
+----
+a
+b
+c
+d
+
+# It needs some time for the split change to be reflected in MV
+sleep 40s
+
+query ?
+SELECT * FROM mv order by x;
+----
+a
+b
+c
+d
+
+
+statement ok
+DROP SOURCE s CASCADE;
+
+system ok
+rpk topic delete test_add_partition