Sync 1.25.0

.circleci/config.yml

@@ -6,6 +6,7 @@ setup: true
 # the path of an updated fileset
 orbs:
   continuation: circleci/[email protected]
+  github-cli: circleci/[email protected]
 
 # optional parameter when triggering to
 # only run a particular type of integration
@@ -51,6 +52,7 @@ jobs:
       - image: cimg/python:3.8
     steps:
       - checkout
+      - github-cli/setup
       - run:
           name: Install yq
           command: |
@@ -150,26 +152,14 @@ jobs:
                 name: Remove approval steps if not pull from forks.
                 command: |
                   pip install pyyaml==6.0.1
-                  python -c  "import yaml
-                  d = yaml.safe_load(open('complete_config.yml'))
-                  for workflow_name, workflow_definition in d['workflows'].items():
-                      jobs = workflow_definition.get('jobs') if isinstance(workflow_definition, dict) else None
-                      if not jobs: continue
-
-                      # find all approvals
-                      approvals = list(filter(lambda x: isinstance(x, dict) and list(x.values())[0].get('type') == 'approval', jobs))
-                      for approval in approvals:
-                          approval_name = next(iter(approval))
-                          approval_upstreams = approval[approval_name].get('requires')
-                          approval_downstream = list(filter(lambda x: isinstance(x, dict) and approval_name in list(x.values())[0].get('requires', ''), jobs))
-                          # replace approval with its upstream jobs
-                          for job in approval_downstream:
-                              requires = next(iter(job.values()))['requires']
-                              requires.remove(approval_name)
-                              requires.extend(approval_upstreams)
-                          jobs.remove(approval)
-                  with open('complete_config.yml', 'w') as f:
-                      f.write(yaml.dump(d, sort_keys=False))"
+                  python dev/filter_approvals.py
+      - run: |
+          export IS_FULL_TESTS=$(gh pr view --json labels | jq 'any(.labels[]; .name == "full-tests")')
+          echo $IS_FULL_TESTS
+          if [ -z "$IS_FULL_TESTS" ] || [ "$IS_FULL_TESTS" == "0" ]; then
+            pip install pyyaml==6.0.1
+            python dev/filter_matrix.py 
+          fi
       - when:
           condition:
             or:
@@ -194,6 +184,7 @@ workflows:
   schedule_workflow:
     jobs:
       - determine_changed_modules:
+          context: pr
           filters:
             tags:
               only: /^[0-9]+(\.[0-9]+){2}(-rc\.[0-9]+)?$/

.circleci/continue_config.yml

@@ -75,7 +75,32 @@ commands:
             echo "Setting default Java to ${JAVA_BIN}"
             sudo update-alternatives --set java ${JAVA_BIN}
             sudo update-alternatives --set javac ${JAVAC_BIN}
-
+  set_java_spark_scala_version:
+    parameters:
+      env-variant:
+        type: string
+    description: "Set Java, Spark and Scala versions"
+    steps:
+      - run: |
+          set -eux
+          JAVA=$(echo << parameters.env-variant >> | cut -d '-' -f 1 | cut -d ':' -f 2)
+          SPARK=$(echo << parameters.env-variant >> | cut -d '-' -f 2 | cut -d ':' -f 2)
+          SCALA=$(echo << parameters.env-variant >> | cut -d '-' -f 3 | cut -d ':' -f 2)
+          echo spark=$SPARK java=$JAVA scala=$SCALA
+          JAVA8_HOME='/usr/lib/jvm/java-8-openjdk-amd64'
+          JAVA17_HOME='/usr/lib/jvm/java-17-openjdk-amd64'
+          JAVA_BIN=$([ "$JAVA" = "17" ] && echo "$JAVA17_HOME/bin/java" || echo "$JAVA8_HOME/jre/bin/java")
+          JAVAC_BIN=$([ "$JAVA" = "17" ] && echo "$JAVA17_HOME/bin/javac" || echo "$JAVA8_HOME/bin/javac")
+
+          echo 'export JAVA17_HOME="/usr/lib/jvm/java-17-openjdk-amd64"' >> "$BASH_ENV"
+          echo "export SPARK=\"${SPARK}\"" >> "$BASH_ENV"
+          echo "export JAVA=\"${JAVA}\"" >> "$BASH_ENV"
+          echo "export JAVA_BIN=\"${JAVA_BIN}\"" >> "$BASH_ENV"
+          echo "export JAVAC_BIN=\"${JAVAC_BIN}\"" >> "$BASH_ENV"
+          echo "export SCALA=\"${SCALA}\"" >> "$BASH_ENV"
+          echo "Setting default Java to ${JAVA_BIN}"
+          sudo update-alternatives --set java ${JAVA_BIN}
+          sudo update-alternatives --set javac ${JAVAC_BIN}
   store_submodule_tests:
     parameters:
       submodule:
@@ -462,7 +487,7 @@ jobs:
       - store_test_results:
           path: build/test-results/test
       - store_test_results:
-          path: transports-dataplex/build/test-results/test
+          path: transports-gcplineage/build/test-results/test
       - store_test_results:
           path: transports-gcs/build/test-results/test
       - store_test_results:
@@ -471,7 +496,7 @@ jobs:
           path: build/reports/tests/test
           destination: test-report
       - store_artifacts:
-          path: transports-dataplex/build/reports/tests/test
+          path: transports-gcplineage/build/reports/tests/test
           destination: test-report
       - store_artifacts:
           path: transports-gcs/build/reports/tests/test
@@ -486,7 +511,7 @@ jobs:
           path: build/libs
           destination: libs
       - store_artifacts:
-          path: transports-dataplex/build/libs
+          path: transports-gcplineage/build/libs
           destination: libs
       - store_artifacts:
           path: transports-gcs/build/libs
@@ -517,8 +542,8 @@ jobs:
           path: ./build/libs
           destination: java-client-artifacts
       - store_artifacts:
-          path: ./transports-dataplex/build/libs
-          destination: transports-dataplex-artifacts
+          path: ./transports-gcplineage/build/libs
+          destination: transports-gcplineage-artifacts
       - store_artifacts:
           path: ./transports-gcs/build/libs
           destination: transports-gcs-artifacts
@@ -532,8 +557,8 @@ jobs:
 
   release-integration-spark:
     working_directory: ~/openlineage/integration/spark
-    docker:
-      - image: cimg/openjdk:17.0
+    machine:
+      image: ubuntu-2404:current
     steps:
       - *checkout_project_root
       - run:
@@ -544,15 +569,16 @@ jobs:
             - v1-release-client-java-{{ checksum "/tmp/checksum.txt" }}
       - attach_workspace:
           at: ~/
+      - set_java_version
       - run: |
           # Get, then decode the GPG private key used to sign *.jar
           export ORG_GRADLE_PROJECT_signingKey=$(echo $GPG_SIGNING_KEY | base64 -d)
           export RELEASE_PASSWORD=$(echo $OSSRH_TOKEN_PASSWORD)
           export RELEASE_USERNAME=$(echo $OSSRH_TOKEN_USERNAME)
 
           # Publish *.jar
-          ./gradlew --no-daemon --console=plain clean publishToSonatype closeAndReleaseSonatypeStagingRepository --info -Pscala.binary.version=2.12 -Pjava.compile.home=/usr/local/jdk-17.0.11
-          ./gradlew --no-daemon --console=plain clean publishToSonatype closeAndReleaseSonatypeStagingRepository --info -Pscala.binary.version=2.13 -Pjava.compile.home=/usr/local/jdk-17.0.11
+          ./gradlew --no-daemon --console=plain clean publishToSonatype closeAndReleaseSonatypeStagingRepository --info -Pscala.binary.version=2.12 -Pjava.compile.home=${JAVA17_HOME}
+          ./gradlew --no-daemon --console=plain clean publishToSonatype closeAndReleaseSonatypeStagingRepository --info -Pscala.binary.version=2.13 -Pjava.compile.home=${JAVA17_HOME}
       - store_artifacts:
           path: ./build/libs
           destination: spark-client-artifacts
@@ -620,24 +646,8 @@ jobs:
           command: ./../../.circleci/checksum.sh /tmp/checksum.txt $CIRCLE_BRANCH
       - attach_workspace:
           at: ~/
-      - run:
-          name: Spark & Java version Variable
-          command: |
-            JAVA=$(echo << parameters.env-variant >> | cut -d '-' -f 1 | cut -d ':' -f 2)
-            SPARK=$(echo << parameters.env-variant >> | cut -d '-' -f 2 | cut -d ':' -f 2)
-            SCALA=$(echo << parameters.env-variant >> | cut -d '-' -f 3 | cut -d ':' -f 2)
-            echo spark=$SPARK java=$JAVA scala=$SCALA
-            JAVA8_HOME='/usr/lib/jvm/java-8-openjdk-amd64'
-            JAVA17_HOME='/usr/lib/jvm/java-17-openjdk-amd64'
-
-            echo 'export JAVA17_HOME=/usr/lib/jvm/java-17-openjdk-amd64' >> "$BASH_ENV"
-            echo 'export SPARK='${SPARK} >> "$BASH_ENV"
-            echo 'export JAVA_BIN='$([ "$JAVA" = "17" ] && echo "$JAVA17_HOME/bin/java" || echo "$JAVA8_HOME/jre/bin/java") >> "$BASH_ENV"
-            echo 'export JAVAC_BIN='$([ "$JAVA" = "17" ] && echo "$JAVA17_HOME/bin/javac" || echo "$JAVA8_HOME/bin/javac") >> "$BASH_ENV"
-            echo 'export SCALA='${SCALA} >> "$BASH_ENV"
-            echo "${JAVA}"
-            echo "${JAVA_BIN}"
-            echo "${JAVAC_BIN}"
+      - set_java_spark_scala_version:
+            env-variant: << parameters.env-variant >>
       - restore_cache:
           keys:
             - v1-integration-spark-{{ checksum "/tmp/checksum.txt" }}
@@ -689,33 +699,15 @@ jobs:
       - run:
           name: Generate cache key
           command: ./../../.circleci/checksum.sh /tmp/checksum.txt $CIRCLE_BRANCH
-      - run:
-          name: Spark & Java version Variable
-          command: |
-            JAVA=$(echo << parameters.env-variant >> | cut -d '-' -f 1 | cut -d ':' -f 2)
-            SPARK=$(echo << parameters.env-variant >> | cut -d '-' -f 2 | cut -d ':' -f 2)
-            SCALA=$(echo << parameters.env-variant >> | cut -d '-' -f 3 | cut -d ':' -f 2)
-            echo spark=$SPARK java=$JAVA scala=$SCALA
-            JAVA8_HOME='/usr/lib/jvm/java-8-openjdk-amd64'
-            JAVA17_HOME='/usr/lib/jvm/java-17-openjdk-amd64'
-
-            echo 'export JAVA17_HOME=/usr/lib/jvm/java-17-openjdk-amd64' >> "$BASH_ENV"
-            echo 'export SPARK_VERSION_VAR='${SPARK} >> "$BASH_ENV"
-            echo 'export SCALA='${SCALA} >> "$BASH_ENV"
-            echo 'export JAVA_BIN='$([ "$JAVA" = "17" ] && echo "$JAVA17_HOME/bin/java" || echo "$JAVA8_HOME/jre/bin/java") >> "$BASH_ENV"
-            echo 'export JAVAC_BIN='$([ "$JAVA" = "17" ] && echo "$JAVA17_HOME/bin/javac" || echo "$JAVA8_HOME/bin/javac") >> "$BASH_ENV"
-            echo $JAVA_BIN
+      - set_java_spark_scala_version:
+            env-variant: << parameters.env-variant >>
       - run: mkdir -p app/build/gcloud && echo $GCLOUD_SERVICE_KEY > app/build/gcloud/gcloud-service-key.json && chmod 644 app/build/gcloud/gcloud-service-key.json
       - restore_cache:
           keys:
             - v1-integration-spark-{{ checksum "/tmp/checksum.txt" }}
       - attach_workspace:
           at: ~/
-      - run: |
-          echo "Setting default Java to ${JAVA_BIN}"
-          sudo update-alternatives --set java ${JAVA_BIN}
-          sudo update-alternatives --set javac ${JAVAC_BIN}
-      - run: ./gradlew --no-daemon --console=plain integrationTest -x test -Pspark.version=${SPARK_VERSION_VAR} -Pscala.binary.version=${SCALA} -Pjava.compile.home=${JAVA17_HOME}
+      - run: ./gradlew --no-daemon --console=plain integrationTest -x test -Pspark.version=${SPARK} -Pscala.binary.version=${SCALA} -Pjava.compile.home=${JAVA17_HOME}
       - run: ./gradlew --no-daemon --console=plain jacocoTestReport -Pscala.binary.version=${SCALA} -Pjava.compile.home=${JAVA17_HOME}
       - store_test_results:
           path: app/build/test-results/integrationTest
@@ -846,7 +838,7 @@ jobs:
 
   integration-test-databricks-integration-spark:
     parameters:
-      spark-version:
+      env-variant:
         type: string
     working_directory: ~/openlineage/integration/spark
     machine:
@@ -871,20 +863,23 @@ jobs:
                   - v1-integration-spark-{{ checksum "/tmp/checksum.txt" }}
             - attach_workspace:
                 at: ~/
-            - set_java_version
-            - run: |
-                sudo update-alternatives --set java ${JAVA_BIN}
-                sudo update-alternatives --set javac ${JAVAC_BIN}
-            - run: ./gradlew --console=plain shadowJar -x test -Pjava.compile.home=${JAVA17_HOME}
-            - run: ./gradlew --no-daemon --console=plain databricksIntegrationTest -x test   -Pspark.version=<< parameters.spark-version >> -PdatabricksHost=$DATABRICKS_HOST -PdatabricksToken=$DATABRICKS_TOKEN -Pjava.compile.home=${JAVA17_HOME}
+            - set_java_spark_scala_version:
+                  env-variant: << parameters.env-variant >>
+            - run: mkdir -p app/build/logs
+            - run: mkdir -p app/build/events
+            - run: ./gradlew --console=plain clean shadowJar -x test -Pjava.compile.home=${JAVA17_HOME}
+            - run: ./gradlew --no-daemon --console=plain databricksIntegrationTest -x test -Pspark.version=${SPARK} -Pscala.binary.version=${SCALA} -Pjava.compile.home=${JAVA17_HOME} -Dopenlineage.tests.databricks.workspace.host=$DATABRICKS_HOST -Dopenlineage.tests.databricks.workspace.token=$DATABRICKS_TOKEN
             - store_test_results:
                 path: app/build/test-results/databricksIntegrationTest
             - store_artifacts:
                 path: app/build/reports/tests/databricksIntegrationTest
                 destination: test-report
             - store_artifacts:
-                path: app/build/cluster-log4j.log
-                destination: cluster-log4j.log
+                path: app/build/logs
+                destination: cluster-logs
+            - store_artifacts:
+                path: app/build/events
+                destination: events
             - save_cache:
                 key: v1-databricks-integration-spark-{{ checksum "/tmp/checksum.txt" }}
                 paths:
@@ -992,8 +987,12 @@ jobs:
           at: ~/
       - set_java_version
       - run: chmod -R 777 data/iceberg/db
-      - run: ./gradlew --console=plain examples:stateful:build -Pflink.version=<< parameters.flink-version >>
-      - run: ./gradlew --no-daemon --console=plain integrationTest --i -Pflink.version=<< parameters.flink-version >>
+      - run: |
+          # Get, then decode the GPG private key used to sign *.jar
+          export ORG_GRADLE_PROJECT_signingKey=$(echo $GPG_SIGNING_KEY | base64 -d)
+          export RELEASE_PASSWORD=$(echo $OSSRH_TOKEN_PASSWORD)
+          export RELEASE_USERNAME=$(echo $OSSRH_TOKEN_USERNAME)
+          ./gradlew --no-daemon --console=plain integrationTest --i -Pflink.version=<< parameters.flink-version >>
       - run:
           when: on_fail
           command: cat app/build/test-results/integrationTest/TEST-*.xml

.circleci/workflows/openlineage-flink.yml

@@ -23,6 +23,7 @@ workflows:
           filters:
             tags:
               only: /^[0-9]+(\.[0-9]+){2}(-rc\.[0-9]+)?$/
+          context: << pipeline.parameters.build-context >>
           matrix:
             parameters:
               flink-version: [ '1.15.4', '1.16.2', '1.17.1', '1.18.1', '1.19.0' ]

.circleci/workflows/openlineage-spark.yml

@@ -23,17 +23,17 @@ workflows:
             parameters:
               env-variant: [
                 'java:8-spark:2.4.8-scala:2.12',
-                'java:8-spark:3.2.4-scala:2.12',
-                'java:8-spark:3.2.4-scala:2.13',
-                'java:8-spark:3.3.4-scala:2.12',
-                'java:8-spark:3.3.4-scala:2.13',
-                'java:17-spark:3.3.4-scala:2.12',
-                'java:17-spark:3.3.4-scala:2.13',
-                'java:8-spark:3.4.3-scala:2.12',
-                'java:8-spark:3.4.3-scala:2.13',
-                'java:8-spark:3.5.2-scala:2.12',
-                'java:8-spark:3.5.2-scala:2.13',
-                'java:17-spark:3.5.2-scala:2.12',
+                'java:8-spark:3.2.4-scala:2.12-full-tests',
+                'java:8-spark:3.2.4-scala:2.13-full-tests',
+                'java:8-spark:3.3.4-scala:2.12-full-tests',
+                'java:8-spark:3.3.4-scala:2.13-full-tests',
+                'java:17-spark:3.3.4-scala:2.12-full-tests',
+                'java:17-spark:3.3.4-scala:2.13-full-tests',
+                'java:8-spark:3.4.3-scala:2.12-full-tests',
+                'java:8-spark:3.4.3-scala:2.13-full-tests',
+                'java:8-spark:3.5.2-scala:2.12-full-tests',
+                'java:8-spark:3.5.2-scala:2.13-full-tests',
+                'java:17-spark:3.5.2-scala:2.12-full-tests',
                 'java:17-spark:3.5.2-scala:2.13',
                 'java:17-spark:4.0.0-scala:2.13'
               ]
@@ -92,7 +92,10 @@ workflows:
           context: integration-tests
           matrix:
             parameters:
-              spark-version: [ '3.4.2', '3.5.2' ]
+              env-variant: [
+                'java:8-spark:3.4.1-scala:2.12-full-tests',
+                'java:17-spark:3.5.0-scala:2.12-full-tests'
+              ]
           requires:
             - approval-integration-spark
           post-steps:
@@ -112,17 +115,17 @@ workflows:
             parameters:
               env-variant: [
                 'java:8-spark:2.4.8-scala:2.12',
-                'java:8-spark:3.2.4-scala:2.12',
-                'java:8-spark:3.2.4-scala:2.13',
-                'java:8-spark:3.3.4-scala:2.12',
-                'java:8-spark:3.3.4-scala:2.13',
-                'java:17-spark:3.3.4-scala:2.12',
-                'java:17-spark:3.3.4-scala:2.13',
-                'java:8-spark:3.4.3-scala:2.12',
-                'java:8-spark:3.4.3-scala:2.13',
-                'java:8-spark:3.5.2-scala:2.12',
-                'java:8-spark:3.5.2-scala:2.13',
-                'java:17-spark:3.5.2-scala:2.12',
+                'java:8-spark:3.2.4-scala:2.12-full-tests',
+                'java:8-spark:3.2.4-scala:2.13-full-tests',
+                'java:8-spark:3.3.4-scala:2.12-full-tests',
+                'java:8-spark:3.3.4-scala:2.13-full-tests',
+                'java:17-spark:3.3.4-scala:2.12-full-tests',
+                'java:17-spark:3.3.4-scala:2.13-full-tests',
+                'java:8-spark:3.4.3-scala:2.12-full-tests',
+                'java:8-spark:3.4.3-scala:2.13-full-tests',
+                'java:8-spark:3.5.2-scala:2.12-full-tests',
+                'java:8-spark:3.5.2-scala:2.13-full-tests',
+                'java:17-spark:3.5.2-scala:2.12-full-tests',
                 'java:17-spark:3.5.2-scala:2.13',
                 'java:17-spark:4.0.0-scala:2.13'
               ]