You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2021/07/05 07:21:12 UTC
[camel] branch main updated: CAMEL-15650: Upgrade camel-spark to
Apache Spark 3.x
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new a53f052 CAMEL-15650: Upgrade camel-spark to Apache Spark 3.x
a53f052 is described below
commit a53f052e5946a3d8085b793377cfb28fe0814aa5
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Mon Jul 5 09:20:35 2021 +0200
CAMEL-15650: Upgrade camel-spark to Apache Spark 3.x
---
camel-dependencies/pom.xml | 5 +-
components/camel-spark/pom.xml | 81 +++++++++++++++++++---
.../ROOT/pages/camel-3x-upgrade-guide-3_12.adoc | 6 +-
parent/pom.xml | 7 +-
4 files changed, 81 insertions(+), 18 deletions(-)
diff --git a/camel-dependencies/pom.xml b/camel-dependencies/pom.xml
index 062b50e..50016a1 100644
--- a/camel-dependencies/pom.xml
+++ b/camel-dependencies/pom.xml
@@ -240,9 +240,8 @@
<gson-version>2.8.7</gson-version>
<guava-eventbus-version>28.2-jre</guava-eventbus-version>
<guice3-version>3.0</guice3-version>
- <hadoop2-version>2.10.0</hadoop2-version>
<hadoop3-guava-version>29.0-jre</hadoop3-guava-version>
- <hadoop3-version>3.3.0</hadoop3-version>
+ <hadoop3-version>3.3.1</hadoop3-version>
<hamcrest-version>2.2</hamcrest-version>
<hapi-fhir-version>4.1.0</hapi-fhir-version>
<hapi-version>2.3</hapi-version>
@@ -511,7 +510,7 @@
<solr-version>8.8.2</solr-version>
<solr-version-range>[8,9)</solr-version-range>
<solr-zookeeper-version>3.6.2</solr-zookeeper-version>
- <spark-version>2.4.8</spark-version>
+ <spark-version>3.1.2</spark-version>
<splunk-version>1.6.5.0_1</splunk-version>
<spock-version>2.0-M4-groovy-3.0</spock-version>
<spring-batch-version>4.3.3</spring-batch-version>
diff --git a/components/camel-spark/pom.xml b/components/camel-spark/pom.xml
index dd20546..07ba0df 100644
--- a/components/camel-spark/pom.xml
+++ b/components/camel-spark/pom.xml
@@ -43,13 +43,36 @@
<artifactId>camel-support</artifactId>
</dependency>
+ <!-- scala 2.12.x -->
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ <version>2.12.14</version>
+ </dependency>
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-reflect</artifactId>
+ <version>2.12.14</version>
+ </dependency>
+
+ <!-- guava -->
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${hadoop3-guava-version}</version>
+ </dependency>
+
<!-- spark -->
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.11</artifactId>
- <version>${spark-version}</version>
+ <artifactId>spark-core_2.12</artifactId>
+ <version>3.1.2</version>
<exclusions>
<exclusion>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
@@ -69,10 +92,14 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-hive_2.11</artifactId>
- <version>${spark-version}</version>
+ <artifactId>spark-hive_2.12</artifactId>
+ <version>3.1.2</version>
<exclusions>
<exclusion>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>log4j</groupId>
<artifactId>apache-log4j-extras</artifactId>
</exclusion>
@@ -101,17 +128,31 @@
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
- <artifactId>jackson-module-scala_2.11</artifactId>
+ <artifactId>jackson-module-scala_2.12</artifactId>
<version>${jackson2-version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- hadoop -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
- <version>${hadoop2-version}</version>
+ <version>${hadoop3-version}</version>
<exclusions>
<exclusion>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
@@ -124,9 +165,13 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>${hadoop2-version}</version>
+ <version>${hadoop3-version}</version>
<exclusions>
<exclusion>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
@@ -143,9 +188,17 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>${hadoop2-version}</version>
+ <version>${hadoop3-version}</version>
<exclusions>
<exclusion>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
@@ -158,9 +211,17 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- <version>${hadoop2-version}</version>
+ <version>${hadoop3-version}</version>
<exclusions>
<exclusion>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
@@ -175,7 +236,7 @@
<dependency>
<groupId>org.mongodb.mongo-hadoop</groupId>
<artifactId>mongo-hadoop-core</artifactId>
- <version>${mongo-hadoop-version}</version>
+ <version>2.0.2</version>
<optional>true</optional>
</dependency>
diff --git a/docs/user-manual/modules/ROOT/pages/camel-3x-upgrade-guide-3_12.adoc b/docs/user-manual/modules/ROOT/pages/camel-3x-upgrade-guide-3_12.adoc
index a212ae0..2d75a3f 100644
--- a/docs/user-manual/modules/ROOT/pages/camel-3x-upgrade-guide-3_12.adoc
+++ b/docs/user-manual/modules/ROOT/pages/camel-3x-upgrade-guide-3_12.adoc
@@ -23,4 +23,8 @@ The `camel-json` data format has renamed the option `unmarshalTypeName` to `unma
The `camel-protobuf` data format has renamed the option `collectionTypeName` to `collectionType`.
The `camel-protobuf` data format has renamed the option `jsonViewTypeName` to `jsonView`.
The `camel-protobuf` data format has renamed the option `unmarshalTypeName` to `unmarshalType`.
-The `camel-yaml` data format has renamed the option `unmarshalTypeName` to `unmarshalType`.
\ No newline at end of file
+The `camel-yaml` data format has renamed the option `unmarshalTypeName` to `unmarshalType`.
+
+=== camel-spark
+
+The `camel-spark` component has been upgraded from Spark 2.x to 3.x.
\ No newline at end of file
diff --git a/parent/pom.xml b/parent/pom.xml
index 6cef0dd..53a4bf9 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -219,8 +219,7 @@
<gson-version>2.8.7</gson-version>
<guava-eventbus-version>28.2-jre</guava-eventbus-version>
<guice3-version>3.0</guice3-version>
- <hadoop2-version>2.10.0</hadoop2-version>
- <hadoop3-version>3.3.0</hadoop3-version>
+ <hadoop3-version>3.3.1</hadoop3-version>
<hadoop3-guava-version>29.0-jre</hadoop3-guava-version>
<hamcrest-version>2.2</hamcrest-version>
<hapi-version>2.3</hapi-version>
@@ -239,7 +238,7 @@
<httpcore4-version>4.4.14</httpcore4-version>
<httpclient4-version>4.5.13</httpclient4-version>
<httpasyncclient-version>4.1.4</httpasyncclient-version>
- <huaweicloud-obs-version>3.21.4.1</huaweicloud-obs-version>
+ <huaweicloud-obs-version>3.21.4.1</huaweicloud-obs-version>
<huaweicloud-sdk-version>3.0.32-rc</huaweicloud-sdk-version>
<hystrix-version>1.5.18</hystrix-version>
<ical4j-version>1.0.7</ical4j-version>
@@ -498,7 +497,7 @@
<solr-version>8.8.2</solr-version>
<solr-version-range>[8,9)</solr-version-range>
<solr-zookeeper-version>3.6.2</solr-zookeeper-version>
- <spark-version>2.4.8</spark-version>
+ <spark-version>3.1.2</spark-version>
<splunk-version>1.6.5.0_1</splunk-version>
<spock-version>2.0-M4-groovy-3.0</spock-version>
<spring-batch-version>4.3.3</spring-batch-version>