Merge pull request #61 from marsishandsome/feature/add-kafka

add kafka and ticdc in CI

.ci/integration_test.groovy

@@ -3,6 +3,9 @@ def call(ghprbActualCommit, ghprbPullId, ghprbPullTitle, ghprbPullLink, ghprbPul
     def TIDB_BRANCH = "release-4.0"
     def TIKV_BRANCH = "release-4.0"
     def PD_BRANCH = "release-4.0"
+    def TICDC_BRANCH = "release-4.0"
+
+    def kafka_version = "kafka_2.12-2.7.0"
 
     // parse tidb branch
     def m1 = ghprbCommentBody =~ /tidb\s*=\s*([^\s\\]+)(\s|\\|$)/
@@ -25,6 +28,13 @@ def call(ghprbActualCommit, ghprbPullId, ghprbPullTitle, ghprbPullLink, ghprbPul
     }
     println "TIKV_BRANCH=${TIKV_BRANCH}"
 
+    // parse ticdc branch
+    def m4 = ghprbCommentBody =~ /ticdc\s*=\s*([^\s\\]+)(\s|\\|$)/
+    if (m4) {
+        TICDC_BRANCH = "${m4[0][1]}"
+    }
+    println "TICDC_BRANCH=${TICDC_BRANCH}"
+
     catchError {
         node ('build') {
             container("java") {
@@ -45,6 +55,8 @@ def call(ghprbActualCommit, ghprbPullId, ghprbPullTitle, ghprbPullLink, ghprbPul
                     }
 
                     dir("/home/jenkins/agent/git/tibigdata/_run") {
+                        sh "rm -rf *"
+
                         // tidb
                         def tidb_sha1 = sh(returnStdout: true, script: "curl ${FILE_SERVER_URL}/download/refs/pingcap/tidb/${TIDB_BRANCH}/sha1").trim()
                         sh "curl ${FILE_SERVER_URL}/download/builds/pingcap/tidb/${tidb_sha1}/centos7/tidb-server.tar.gz | tar xz"
@@ -54,11 +66,18 @@ def call(ghprbActualCommit, ghprbPullId, ghprbPullTitle, ghprbPullLink, ghprbPul
                         // pd
                         def pd_sha1 = sh(returnStdout: true, script: "curl ${FILE_SERVER_URL}/download/refs/pingcap/pd/${PD_BRANCH}/sha1").trim()
                         sh "curl ${FILE_SERVER_URL}/download/builds/pingcap/pd/${pd_sha1}/centos7/pd-server.tar.gz | tar xz"
+                        //ticdc
+                        def ticdc_sha1 = sh(returnStdout: true, script: "curl ${FILE_SERVER_URL}/download/refs/pingcap/ticdc/${TICDC_BRANCH}/sha1").trim()
+                        sh "curl ${FILE_SERVER_URL}/download/builds/pingcap/ticdc/${ticdc_sha1}/centos7/ticdc-linux-amd64.tar.gz | tar xz"
+                        // kafka
+                        sh "curl ${FILE_SERVER_URL}/download/${kafka_version}.tgz | tar xz"
+                        sh "mv ${kafka_version} kafka/"
 
                         sh """
                         killall -9 tidb-server || true
                         killall -9 tikv-server || true
                         killall -9 pd-server || true
+                        killall -9 cdc || true
                         killall -9 java || true
                         sleep 10
                         bin/pd-server --name=pd --data-dir=pd --config=../.ci/config/pd.toml &>pd.log &
@@ -70,6 +89,24 @@ def call(ghprbActualCommit, ghprbPullId, ghprbPullTitle, ghprbPullLink, ghprbPul
                         bin/tidb-server --store=tikv --path="127.0.0.1:2379" --config=../.ci/config/tidb.toml &>tidb.log &
                         sleep 60
                         """
+
+                        sh """
+                        rm -rf /tmp/zookeeper
+                        rm -rf /tmp/kafka-logs
+                        kafka/bin/zookeeper-server-start.sh kafka/config/zookeeper.properties &
+                        sleep 10
+                        kafka/bin/kafka-server-start.sh kafka/config/server.properties &
+                        sleep 10
+                        kafka/bin/kafka-topics.sh --create --topic tidb_test --partitions 3 --replication-factor 1 --bootstrap-server localhost:9092
+                        kafka/bin/kafka-topics.sh --describe --topic tidb_test --bootstrap-server localhost:9092
+                        """
+
+                        sh """
+                        cd ticdc-linux-amd64
+                        ./bin/cdc server --pd="http://127.0.0.1:2379"  --log-file=ticdc.log --addr="0.0.0.0:8301" --advertise-addr="127.0.0.1:8301" &
+                        sleep 10
+                        ./bin/cdc cli changefeed create --pd="http://127.0.0.1:2379" --sink-uri="kafka://127.0.0.1:9092/tidb_test" --no-confirm
+                        """
                     }
                 }
 
@@ -87,6 +124,8 @@ def call(ghprbActualCommit, ghprbPullId, ghprbPullTitle, ghprbPullLink, ghprbPul
                             sh "cat _run/pd.log"
                             sh "cat _run/tikv.log"
                             sh "cat _run/tidb.log"
+                            sh "cat _run/kafka/logs/server.log"
+                            sh "cat _run/ticdc-linux-amd64/ticdc.log"
                             throw err
                         }
                     }