Skip to content

Commit

Permalink
Downgrade hive to work on Spark 3.4.1, apache#3274
Browse files Browse the repository at this point in the history
  • Loading branch information
@hansva
hansva committed Oct 11, 2023
1 parent bc2a0d3 commit 4907ef2
Show file tree
Hide file tree
Showing 8 changed files with 148 additions and 33 deletions.
28 changes: 27 additions & 1 deletion assemblies/plugins/databases/hive-assemblies/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@

<properties>
<!-- Version information : https://github.com/apache/hive/blob/master/pom.xml-->
<libthrift.version>0.16.0</libthrift.version>
<libthrift.version>0.9.3</libthrift.version>
</properties>

<dependencies>
Expand All @@ -62,6 +62,10 @@
<groupId>org.glassfish.web</groupId>
<artifactId>javax.servlet.jsp</artifactId>
</exclusion>
<exclusion>
<artifactId>jdk.tools</artifactId>
<groupId>jdk.tools</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
Expand All @@ -81,6 +85,10 @@
<groupId>org.glassfish.web</groupId>
<artifactId>javax.servlet.jsp</artifactId>
</exclusion>
<exclusion>
<artifactId>jdk.tools</artifactId>
<groupId>jdk.tools</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
Expand All @@ -105,6 +113,10 @@
<groupId>org.glassfish.web</groupId>
<artifactId>javax.servlet.jsp</artifactId>
</exclusion>
<exclusion>
<artifactId>jdk.tools</artifactId>
<groupId>jdk.tools</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
Expand All @@ -116,6 +128,12 @@
<groupId>org.apache.hive</groupId>
<artifactId>hive-service</artifactId>
<version>${hive.version}</version>
<exclusions>
<exclusion>
<groupId>org.pentaho</groupId>
<artifactId>pentaho-aggdesigner-algorithm</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
Expand All @@ -134,6 +152,10 @@
<groupId>org.glassfish.web</groupId>
<artifactId>javax.servlet.jsp</artifactId>
</exclusion>
<exclusion>
<artifactId>jdk.tools</artifactId>
<groupId>jdk.tools</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
Expand All @@ -153,6 +175,10 @@
<groupId>org.glassfish.web</groupId>
<artifactId>javax.servlet.jsp</artifactId>
</exclusion>
<exclusion>
<artifactId>jdk.tools</artifactId>
<groupId>jdk.tools</groupId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
Expand Down
36 changes: 36 additions & 0 deletions assemblies/plugins/engines/beam/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -462,6 +462,12 @@
<groupId>org.apache.hive.hcatalog</groupId>
<artifactId>hive-hcatalog-core</artifactId>
<version>${hive.version}</version>
<exclusions>
<exclusion>
<groupId>org.pentaho</groupId>
<artifactId>pentaho-aggdesigner-algorithm</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
Expand All @@ -473,16 +479,46 @@
<artifactId>hive-exec</artifactId>
<classifier>core</classifier>
<version>${hive.version}</version>
<exclusions>
<exclusion>
<groupId>org.pentaho</groupId>
<artifactId>pentaho-aggdesigner-algorithm</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-metastore</artifactId>
<version>${hive.version}</version>
<exclusions>
<exclusion>
<groupId>org.glassfish.web</groupId>
<artifactId>javax.servlet.jsp</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.web</groupId>
<artifactId>javax.servlet.jsp</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.web</groupId>
<artifactId>javax.servlet.jsp</artifactId>
</exclusion>
<exclusion>
<artifactId>jdk.tools</artifactId>
<groupId>jdk.tools</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-cli</artifactId>
<version>${hive.version}</version>
<exclusions>
<exclusion>
<groupId>org.pentaho</groupId>
<artifactId>pentaho-aggdesigner-algorithm</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
Expand Down
38 changes: 19 additions & 19 deletions docker/integration-tests/integration-tests-spark.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,25 +17,25 @@

version: '2.4'
services:
# spark:
# build:
# context: ../../docker/integration-tests/spark/.
# dockerfile: Dockerfile.master
# environment:
# - INIT_DAEMON_STEP=setup_spark
# ports:
# - 8080:8080
# - 7077:7077
# spark-worker:
# build:
# context: ../../docker/integration-tests/spark/.
# dockerfile: Dockerfile.worker
# depends_on:
# - spark
# environment:
# - "SPARK_MASTER=spark://spark:7077"
# ports:
# - 8081:8081
spark:
build:
context: ../../docker/integration-tests/spark/.
dockerfile: Dockerfile.master
environment:
- INIT_DAEMON_STEP=setup_spark
ports:
- 8080:8080
- 7077:7077
spark-worker:
build:
context: ../../docker/integration-tests/spark/.
dockerfile: Dockerfile.worker
depends_on:
- spark
environment:
- "SPARK_MASTER=spark://spark:7077"
ports:
- 8081:8081
namenode:
build:
dockerfile: Dockerfile.hadoop
Expand Down
4 changes: 2 additions & 2 deletions docker/integration-tests/spark/Dockerfile.master
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ ENV ENABLE_INIT_DAEMON false
ENV INIT_DAEMON_BASE_URI http://identifier/init-daemon
ENV INIT_DAEMON_STEP spark_master_init

ENV BASE_URL=https://archive.apache.org/dist/spark/
ENV SPARK_VERSION=3.4.0
ENV BASE_URL=https://dlcdn.apache.org/spark/
ENV SPARK_VERSION=3.4.1
ENV HADOOP_VERSION=3
ENV SPARK_MASTER_PORT 7077
ENV SPARK_MASTER_WEBUI_PORT 8080
Expand Down
4 changes: 2 additions & 2 deletions docker/integration-tests/spark/Dockerfile.worker
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ ENV ENABLE_INIT_DAEMON false
ENV INIT_DAEMON_BASE_URI http://identifier/init-daemon
ENV INIT_DAEMON_STEP spark_master_init

ENV BASE_URL=https://archive.apache.org/dist/spark/
ENV SPARK_VERSION=3.4.0
ENV BASE_URL=https://dlcdn.apache.org/spark/
ENV SPARK_VERSION=3.4.1
ENV HADOOP_VERSION=3
ENV SPARK_WORKER_WEBUI_PORT 8081
ENV SPARK_WORKER_LOG /spark/logs
Expand Down
6 changes: 5 additions & 1 deletion plugins/databases/hive/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
<name>Hop Plugins Databases Apache Hive</name>

<properties>
<hive.version>3.1.3</hive.version>
<hive.version>2.3.9</hive.version>
</properties>

<dependencies>
Expand Down Expand Up @@ -62,6 +62,10 @@
<groupId>org.glassfish.web</groupId>
<artifactId>javax.servlet.jsp</artifactId>
</exclusion>
<exclusion>
<artifactId>jdk.tools</artifactId>
<groupId>jdk.tools</groupId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
Expand Down
61 changes: 55 additions & 6 deletions plugins/engines/beam/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -400,6 +400,12 @@
<groupId>org.apache.hive.hcatalog</groupId>
<artifactId>hive-hcatalog-core</artifactId>
<version>${hive.version}</version>
<exclusions>
<exclusion>
<groupId>org.pentaho</groupId>
<artifactId>pentaho-aggdesigner-algorithm</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
Expand All @@ -411,11 +417,23 @@
<artifactId>hive-exec</artifactId>
<classifier>core</classifier>
<version>${hive.version}</version>
<exclusions>
<exclusion>
<groupId>org.pentaho</groupId>
<artifactId>pentaho-aggdesigner-algorithm</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-cli</artifactId>
<version>${hive.version}</version>
<exclusions>
<exclusion>
<groupId>org.pentaho</groupId>
<artifactId>pentaho-aggdesigner-algorithm</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
Expand All @@ -427,20 +445,51 @@
<artifactId>hive-service-rpc</artifactId>
<version>${hive.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-standalone-metastore</artifactId>
<version>${hive.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-metastore</artifactId>
<version>${hive.version}</version>
<exclusions>
<exclusion>
<groupId>org.glassfish.web</groupId>
<artifactId>javax.servlet.jsp</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.web</groupId>
<artifactId>javax.servlet.jsp</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.web</groupId>
<artifactId>javax.servlet.jsp</artifactId>
</exclusion>
<exclusion>
<artifactId>jdk.tools</artifactId>
<groupId>jdk.tools</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-spark-client</artifactId>
<artifactId>spark-client</artifactId>
<version>${hive.version}</version>
<exclusions>
<exclusion>
<groupId>org.glassfish.web</groupId>
<artifactId>javax.servlet.jsp</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.web</groupId>
<artifactId>javax.servlet.jsp</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.web</groupId>
<artifactId>javax.servlet.jsp</artifactId>
</exclusion>
<exclusion>
<artifactId>jdk.tools</artifactId>
<groupId>jdk.tools</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
Expand Down
4 changes: 2 additions & 2 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@
<juniversalchardet.version>2.4.0</juniversalchardet.version>
<kafka-clients.version>3.3.1</kafka-clients.version>
<kryo.version>5.3.0</kryo.version>
<libthrift.version>0.17.0</libthrift.version>
<libthrift.version>0.9.3</libthrift.version>
<log4j.version>2.19.0</log4j.version>
<lz4.version>1.8.0</lz4.version>
<metrics.version>4.2.12</metrics.version>
Expand All @@ -184,7 +184,7 @@
<woodstox.version>6.4.0</woodstox.version>
<zookeeper.version>3.8.0</zookeeper.version>
<xerces.version>2.12.2</xerces.version>
<hive.version>3.1.3</hive.version>
<hive.version>2.3.9</hive.version>

<!-- Test dependencies -->
<byte-buddy.version>1.12.19</byte-buddy.version>
Expand Down

0 comments on commit 4907ef2

Please sign in to comment.