You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by zb...@apache.org on 2021/07/06 13:30:23 UTC
[camel-quarkus] 02/02: Upgrade spark and hadoop versions
This is an automated email from the ASF dual-hosted git repository.
zbendhiba pushed a commit to branch camel-main
in repository https://gitbox.apache.org/repos/asf/camel-quarkus.git
commit ff9aa5ee212bdff54250a75cb4a63fe576b8903d
Author: Zineb Bendhiba <be...@gmail.com>
AuthorDate: Tue Jul 6 14:30:31 2021 +0200
Upgrade spark and hadoop versions
---
.../ROOT/pages/reference/extensions/aws2-sqs.adoc | 4 +-
.../reference/extensions/elasticsearch-rest.adoc | 4 +-
.../ROOT/pages/reference/extensions/hdfs.adoc | 4 +-
.../reference/extensions/huaweicloud-smn.adoc | 2 +-
.../reference/extensions/microprofile-health.adoc | 4 +-
.../ROOT/pages/reference/extensions/netty.adoc | 4 +-
.../partials/reference/components/aws2-sqs.adoc | 2 +-
.../reference/components/elasticsearch-rest.adoc | 2 +-
.../partials/reference/components/hwcloud-smn.adoc | 2 +-
.../ROOT/partials/reference/components/netty.adoc | 2 +-
.../reference/languages/exchangeProperty.adoc | 2 +-
.../ROOT/partials/reference/languages/file.adoc | 2 +-
.../ROOT/partials/reference/languages/header.adoc | 2 +-
.../partials/reference/languages/tokenize.adoc | 2 +-
.../reference/others/microprofile-health.adoc | 2 +-
extensions-jvm/hdfs/runtime/pom.xml | 2 +-
extensions-jvm/spark/bom/pom.xml | 51 ++++++++++++++++++----
pom.xml | 9 +++-
18 files changed, 71 insertions(+), 31 deletions(-)
diff --git a/docs/modules/ROOT/pages/reference/extensions/aws2-sqs.adoc b/docs/modules/ROOT/pages/reference/extensions/aws2-sqs.adoc
index 6dbebf4..be179d2 100644
--- a/docs/modules/ROOT/pages/reference/extensions/aws2-sqs.adoc
+++ b/docs/modules/ROOT/pages/reference/extensions/aws2-sqs.adoc
@@ -7,7 +7,7 @@
:cq-native-supported: true
:cq-status: Stable
:cq-status-deprecation: Stable
-:cq-description: Sending and receive messages to/from AWS SQS service using AWS SDK version 2.x.
+:cq-description: Send and receive messages to/from AWS SQS service using AWS SDK version 2.x.
:cq-deprecated: false
:cq-jvm-since: 1.0.0
:cq-native-since: 1.0.0
@@ -15,7 +15,7 @@
[.badges]
[.badge-key]##JVM since##[.badge-supported]##1.0.0## [.badge-key]##Native since##[.badge-supported]##1.0.0##
-Sending and receive messages to/from AWS SQS service using AWS SDK version 2.x.
+Send and receive messages to/from AWS SQS service using AWS SDK version 2.x.
== What's inside
diff --git a/docs/modules/ROOT/pages/reference/extensions/elasticsearch-rest.adoc b/docs/modules/ROOT/pages/reference/extensions/elasticsearch-rest.adoc
index 854aa27..faa258a 100644
--- a/docs/modules/ROOT/pages/reference/extensions/elasticsearch-rest.adoc
+++ b/docs/modules/ROOT/pages/reference/extensions/elasticsearch-rest.adoc
@@ -7,7 +7,7 @@
:cq-native-supported: true
:cq-status: Stable
:cq-status-deprecation: Stable
-:cq-description: Send requests to with an ElasticSearch via REST API.
+:cq-description: Send requests to ElasticSearch via REST API
:cq-deprecated: false
:cq-jvm-since: 1.0.0
:cq-native-since: 1.0.0
@@ -15,7 +15,7 @@
[.badges]
[.badge-key]##JVM since##[.badge-supported]##1.0.0## [.badge-key]##Native since##[.badge-supported]##1.0.0##
-Send requests to with an ElasticSearch via REST API.
+Send requests to ElasticSearch via REST API
== What's inside
diff --git a/docs/modules/ROOT/pages/reference/extensions/hdfs.adoc b/docs/modules/ROOT/pages/reference/extensions/hdfs.adoc
index 48bd5e5..d366cae 100644
--- a/docs/modules/ROOT/pages/reference/extensions/hdfs.adoc
+++ b/docs/modules/ROOT/pages/reference/extensions/hdfs.adoc
@@ -6,7 +6,7 @@
:cq-native-supported: false
:cq-status: Preview
:cq-status-deprecation: Preview
-:cq-description: Read and write from/to an HDFS filesystem using Hadoop 2.x.
+:cq-description: Read and write from/to an HDFS filesystem using Hadoop 3.x.
:cq-deprecated: false
:cq-jvm-since: 1.1.0
:cq-native-since: n/a
@@ -14,7 +14,7 @@
[.badges]
[.badge-key]##JVM since##[.badge-supported]##1.1.0## [.badge-key]##Native##[.badge-unsupported]##unsupported##
-Read and write from/to an HDFS filesystem using Hadoop 2.x.
+Read and write from/to an HDFS filesystem using Hadoop 3.x.
== What's inside
diff --git a/docs/modules/ROOT/pages/reference/extensions/huaweicloud-smn.adoc b/docs/modules/ROOT/pages/reference/extensions/huaweicloud-smn.adoc
index be42673..98e05f9 100644
--- a/docs/modules/ROOT/pages/reference/extensions/huaweicloud-smn.adoc
+++ b/docs/modules/ROOT/pages/reference/extensions/huaweicloud-smn.adoc
@@ -18,7 +18,7 @@ Huawei Cloud component to integrate with SimpleNotification services
== What's inside
-* xref:{cq-camel-components}::hwcloud-smn-component.adoc[Huawei Cloud Simple Message Notification (SMN) component], URI syntax: `hwcloud-smn:smnService`
+* xref:{cq-camel-components}::hwcloud-smn-component.adoc[Huawei Simple Message Notification (SMN) component], URI syntax: `hwcloud-smn:smnService`
Please refer to the above link for usage and configuration details.
diff --git a/docs/modules/ROOT/pages/reference/extensions/microprofile-health.adoc b/docs/modules/ROOT/pages/reference/extensions/microprofile-health.adoc
index a25153a..92c5e6c 100644
--- a/docs/modules/ROOT/pages/reference/extensions/microprofile-health.adoc
+++ b/docs/modules/ROOT/pages/reference/extensions/microprofile-health.adoc
@@ -7,7 +7,7 @@
:cq-native-supported: true
:cq-status: Stable
:cq-status-deprecation: Stable
-:cq-description: Bridging Eclipse MicroProfile Health with Camel health checks
+:cq-description: Expose Camel health checks via MicroProfile Health
:cq-deprecated: false
:cq-jvm-since: 0.3.0
:cq-native-since: 0.3.0
@@ -15,7 +15,7 @@
[.badges]
[.badge-key]##JVM since##[.badge-supported]##0.3.0## [.badge-key]##Native since##[.badge-supported]##0.3.0##
-Bridging Eclipse MicroProfile Health with Camel health checks
+Expose Camel health checks via MicroProfile Health
== What's inside
diff --git a/docs/modules/ROOT/pages/reference/extensions/netty.adoc b/docs/modules/ROOT/pages/reference/extensions/netty.adoc
index 30a781b..3d42861 100644
--- a/docs/modules/ROOT/pages/reference/extensions/netty.adoc
+++ b/docs/modules/ROOT/pages/reference/extensions/netty.adoc
@@ -7,7 +7,7 @@
:cq-native-supported: true
:cq-status: Stable
:cq-status-deprecation: Stable
-:cq-description: Socket level networking using TCP or UDP with the Netty 4.x.
+:cq-description: Socket level networking using TCP or UDP with Netty 4.x.
:cq-deprecated: false
:cq-jvm-since: 0.4.0
:cq-native-since: 0.4.0
@@ -15,7 +15,7 @@
[.badges]
[.badge-key]##JVM since##[.badge-supported]##0.4.0## [.badge-key]##Native since##[.badge-supported]##0.4.0##
-Socket level networking using TCP or UDP with the Netty 4.x.
+Socket level networking using TCP or UDP with Netty 4.x.
== What's inside
diff --git a/docs/modules/ROOT/partials/reference/components/aws2-sqs.adoc b/docs/modules/ROOT/partials/reference/components/aws2-sqs.adoc
index 2a953b3..548e3ad 100644
--- a/docs/modules/ROOT/partials/reference/components/aws2-sqs.adoc
+++ b/docs/modules/ROOT/partials/reference/components/aws2-sqs.adoc
@@ -9,5 +9,5 @@
:cq-native-since: 1.0.0
:cq-camel-part-name: aws2-sqs
:cq-camel-part-title: AWS Simple Queue Service (SQS)
-:cq-camel-part-description: Sending and receive messages to/from AWS SQS service using AWS SDK version 2.x.
+:cq-camel-part-description: Send and receive messages to/from AWS SQS service using AWS SDK version 2.x.
:cq-extension-page-title: AWS 2 Simple Queue Service (SQS)
diff --git a/docs/modules/ROOT/partials/reference/components/elasticsearch-rest.adoc b/docs/modules/ROOT/partials/reference/components/elasticsearch-rest.adoc
index 3daddf0..5c492ca 100644
--- a/docs/modules/ROOT/partials/reference/components/elasticsearch-rest.adoc
+++ b/docs/modules/ROOT/partials/reference/components/elasticsearch-rest.adoc
@@ -9,5 +9,5 @@
:cq-native-since: 1.0.0
:cq-camel-part-name: elasticsearch-rest
:cq-camel-part-title: Elasticsearch Rest
-:cq-camel-part-description: Send requests to with an ElasticSearch via REST API.
+:cq-camel-part-description: Send requests to ElasticSearch via REST API
:cq-extension-page-title: Elasticsearch Rest
diff --git a/docs/modules/ROOT/partials/reference/components/hwcloud-smn.adoc b/docs/modules/ROOT/partials/reference/components/hwcloud-smn.adoc
index 06732e9..abc460f 100644
--- a/docs/modules/ROOT/partials/reference/components/hwcloud-smn.adoc
+++ b/docs/modules/ROOT/partials/reference/components/hwcloud-smn.adoc
@@ -8,6 +8,6 @@
:cq-jvm-since: 1.8.0
:cq-native-since: n/a
:cq-camel-part-name: hwcloud-smn
-:cq-camel-part-title: Huawei Cloud Simple Message Notification (SMN)
+:cq-camel-part-title: Huawei Simple Message Notification (SMN)
:cq-camel-part-description: To broadcast messages and connect cloud services through notifications on Huawei Cloud
:cq-extension-page-title: SimpleNotification
diff --git a/docs/modules/ROOT/partials/reference/components/netty.adoc b/docs/modules/ROOT/partials/reference/components/netty.adoc
index 559ee24..46dea2e 100644
--- a/docs/modules/ROOT/partials/reference/components/netty.adoc
+++ b/docs/modules/ROOT/partials/reference/components/netty.adoc
@@ -9,5 +9,5 @@
:cq-native-since: 0.4.0
:cq-camel-part-name: netty
:cq-camel-part-title: Netty
-:cq-camel-part-description: Socket level networking using TCP or UDP with the Netty 4.x.
+:cq-camel-part-description: Socket level networking using TCP or UDP with Netty 4.x.
:cq-extension-page-title: Netty
diff --git a/docs/modules/ROOT/partials/reference/languages/exchangeProperty.adoc b/docs/modules/ROOT/partials/reference/languages/exchangeProperty.adoc
index 55b63d2..9533c59 100644
--- a/docs/modules/ROOT/partials/reference/languages/exchangeProperty.adoc
+++ b/docs/modules/ROOT/partials/reference/languages/exchangeProperty.adoc
@@ -9,5 +9,5 @@
:cq-native-since: 0.0.1
:cq-camel-part-name: exchangeProperty
:cq-camel-part-title: ExchangeProperty
-:cq-camel-part-description: Get the value of named Camel Exchange property.
+:cq-camel-part-description: Get values of named Camel Exchange properties.
:cq-extension-page-title: Core
diff --git a/docs/modules/ROOT/partials/reference/languages/file.adoc b/docs/modules/ROOT/partials/reference/languages/file.adoc
index 51b37b3..4f56872 100644
--- a/docs/modules/ROOT/partials/reference/languages/file.adoc
+++ b/docs/modules/ROOT/partials/reference/languages/file.adoc
@@ -9,5 +9,5 @@
:cq-native-since: 0.0.1
:cq-camel-part-name: file
:cq-camel-part-title: File
-:cq-camel-part-description: For expressions and predicates using the file/simple language.
+:cq-camel-part-description: File related capabilities for the Simle language
:cq-extension-page-title: Core
diff --git a/docs/modules/ROOT/partials/reference/languages/header.adoc b/docs/modules/ROOT/partials/reference/languages/header.adoc
index ba1066e..5611593 100644
--- a/docs/modules/ROOT/partials/reference/languages/header.adoc
+++ b/docs/modules/ROOT/partials/reference/languages/header.adoc
@@ -9,5 +9,5 @@
:cq-native-since: 0.0.1
:cq-camel-part-name: header
:cq-camel-part-title: Header
-:cq-camel-part-description: Get the value of the named Camel Message header.
+:cq-camel-part-description: Get values of Camel Message headers.
:cq-extension-page-title: Core
diff --git a/docs/modules/ROOT/partials/reference/languages/tokenize.adoc b/docs/modules/ROOT/partials/reference/languages/tokenize.adoc
index a3b2751..fe9aa69 100644
--- a/docs/modules/ROOT/partials/reference/languages/tokenize.adoc
+++ b/docs/modules/ROOT/partials/reference/languages/tokenize.adoc
@@ -9,5 +9,5 @@
:cq-native-since: 0.0.1
:cq-camel-part-name: tokenize
:cq-camel-part-title: Tokenize
-:cq-camel-part-description: Tokenize text payloads using the specified delimiter patterns.
+:cq-camel-part-description: Tokenize text payloads using delimiter patterns.
:cq-extension-page-title: Core
diff --git a/docs/modules/ROOT/partials/reference/others/microprofile-health.adoc b/docs/modules/ROOT/partials/reference/others/microprofile-health.adoc
index 89dc0a6..ed35e3c 100644
--- a/docs/modules/ROOT/partials/reference/others/microprofile-health.adoc
+++ b/docs/modules/ROOT/partials/reference/others/microprofile-health.adoc
@@ -9,5 +9,5 @@
:cq-native-since: 0.3.0
:cq-camel-part-name: microprofile-health
:cq-camel-part-title: Microprofile Health
-:cq-camel-part-description: Bridging Eclipse MicroProfile Health with Camel health checks
+:cq-camel-part-description: Expose Camel health checks via MicroProfile Health
:cq-extension-page-title: MicroProfile Health
diff --git a/extensions-jvm/hdfs/runtime/pom.xml b/extensions-jvm/hdfs/runtime/pom.xml
index 498ec47..d204919 100644
--- a/extensions-jvm/hdfs/runtime/pom.xml
+++ b/extensions-jvm/hdfs/runtime/pom.xml
@@ -28,7 +28,7 @@
<artifactId>camel-quarkus-hdfs</artifactId>
<name>Camel Quarkus :: HDFS :: Runtime</name>
- <description>Read and write from/to an HDFS filesystem using Hadoop 2.x.</description>
+ <description>Read and write from/to an HDFS filesystem using Hadoop 3.x.</description>
<properties>
<camel.quarkus.jvmSince>1.1.0</camel.quarkus.jvmSince>
diff --git a/extensions-jvm/spark/bom/pom.xml b/extensions-jvm/spark/bom/pom.xml
index 32a237e..9e6540a 100644
--- a/extensions-jvm/spark/bom/pom.xml
+++ b/extensions-jvm/spark/bom/pom.xml
@@ -29,7 +29,7 @@
<artifactId>camel-quarkus-spark-bom</artifactId>
<packaging>pom</packaging>
<name>Camel Quarkus :: Spark :: BOM</name>
- <description>Camel Quarkus Spark specific dependency managament</description>
+ <description>Camel Quarkus Spark specific dependency management</description>
<dependencyManagement>
<dependencies>
@@ -73,6 +73,11 @@
</dependency>
<dependency>
+ <groupId>com.google.inject.extensions</groupId>
+ <artifactId>guice-servlet</artifactId>
+ <version>${guice-servlet.version}</version>
+ </dependency>
+ <dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
<version>${jersey-sun.version}</version><!-- Replace com.sun.jersey:jersey-core:1.9 that contains old and incompatible javax.ws.rs.* classes -->
@@ -98,11 +103,36 @@
<version>${jersey-sun.version}</version><!-- Replace com.sun.jersey:jersey-core:1.9 that contains old and incompatible javax.ws.rs.* classes -->
</dependency>
<dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-servlet</artifactId>
+ <version>${jersey-sun.version}</version> <!-- Spark -->
+ </dependency>
+ <dependency>
+ <groupId>commons-net</groupId>
+ <artifactId>commons-net</artifactId>
+ <version>${commons-net.version}</version>
+ </dependency>
+ <dependency>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
<version>${netty3.version}</version><!-- Managed here because it would conflict with netty 4 used elsewhere -->
</dependency>
<dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-json</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-jvm</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-text</artifactId>
+ <version>${commons-text.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derby</artifactId>
<version>${derby.version}</version><!-- Managed here because it is not used elsewhere -->
@@ -110,37 +140,42 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
- <version>${hadoop2.version}</version><!-- Managed here because it would conflict with Hadoop 3 used elsewhere -->
+ <version>${hadoop3.version}</version><!-- Managed here because it would conflict with Hadoop 3 used elsewhere -->
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>${hadoop2.version}</version><!-- Managed here because it would conflict with Hadoop 3 used elsewhere -->
+ <version>${hadoop3.version}</version><!-- Managed here because it would conflict with Hadoop 3 used elsewhere -->
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>${hadoop2.version}</version><!-- Managed here because it would conflict with Hadoop 3 used elsewhere -->
+ <version>${hadoop3.version}</version><!-- Managed here because it would conflict with Hadoop 3 used elsewhere -->
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- <version>${hadoop2.version}</version><!-- Managed here because it would conflict with Hadoop 3 used elsewhere -->
+ <version>${hadoop3.version}</version><!-- Managed here because it would conflict with Hadoop 3 used elsewhere -->
+ </dependency>
+ <dependency>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
+ <version>${libthrift-version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
- <version>${scala-2.11.version}</version>
+ <version>${scala-2.12.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
- <version>${scala-2.11.version}</version>
+ <version>${scala-2.12.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
- <version>${scala-2.11.version}</version>
+ <version>${scala-2.12.version}</version>
</dependency>
<dependency>
<groupId>org.xerial.snappy</groupId>
diff --git a/pom.xml b/pom.xml
index 8fb74b7..25aca9a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -67,6 +67,8 @@
<commons-collections.version>3.2.2</commons-collections.version><!-- used by hbase, should be pretty stable as commons-collections are not developed actively anymore -->
<commons-lang.version>2.6</commons-lang.version><!-- used by hbase, should be pretty stable as commons-lang is not developed actively anymore -->
<commons-math3.version>3.6.1</commons-math3.version><!-- Mess in the transitive dependencies of Spark and hbase-testing-util -->
+ <commons-net.version>${commons-net-version}</commons-net.version> <!-- Spark -->
+ <commons-text.version>${commons-text-version}</commons-text.version> <!-- Spark -->
<curator.version>4.3.0</curator.version><!-- Mess in the transitive dependencies of Spark, Zookeeper and other hadoop related components -->
<derby.version>10.15.2.0</derby.version><!-- Spark -->
<eddsa.version>${eddsa-version}</eddsa.version>
@@ -77,7 +79,8 @@
<guava.version>29.0-jre</guava.version>
<graalvm.version>21.1.0</graalvm.version><!-- @sync io.quarkus:quarkus-bom:${quarkus.version} dep:org.graalvm.nativeimage:svm -->
<gson.version>2.8.6</gson.version><!-- @sync com.ibm.jsonata4java:JSONata4Java:${jsonata4java-version} dep:com.google.code.gson:gson -->
- <hadoop2.version>${hadoop2-version}</hadoop2.version><!-- Spark -->
+ <guice-servlet.version>4.0</guice-servlet.version><!-- Spark -->
+ <hadoop3.version>${hadoop3-version}</hadoop3.version><!-- Spark -->
<hapi.version>${hapi-version}</hapi.version>
<hapi-fhir.version>${hapi-fhir-version}</hapi-fhir.version>
<hbase.version>${hbase-version}</hbase.version>
@@ -98,7 +101,9 @@
<kafka.version>2.8.0</kafka.version><!-- @sync io.quarkus:quarkus-bom:${quarkus.version} dep:org.apache.kafka:kafka-clients -->
<kudu.version>${kudu-version}</kudu.version>
<kotlin.version>1.4.32</kotlin.version><!-- @sync io.quarkus:quarkus-bom:${quarkus.version} dep:org.jetbrains.kotlin:kotlin-stdlib -->
+ <libthrift.version>${libthrift-version}</libthrift.version> <!-- Spark -->
<netty3.version>${netty3-version}</netty3.version><!-- Spark -->
+ <metrics.version>${metrics-version}</metrics.version><!-- Spark -->
<minio.version>${minio-version}</minio.version><!-- Keep in sync with quarkiverse-minio and Camel, preferring Camel assuming that it is more likely to be up to date -->
<mvel2.version>${mvel-version}</mvel2.version>
<nimbus-jose-jwt.version>9.10</nimbus-jose-jwt.version><!-- It's placed here because of mess in hdfs transitive deps, please upgrade when it's possible -->
@@ -107,7 +112,7 @@
<protobuf.version>${protobuf-version}</protobuf.version>
<retrofit.version>2.5.0</retrofit.version>
<saaj.impl.version>1.5.3</saaj.impl.version>
- <scala-2.11.version>2.11.12</scala-2.11.version><!-- Spark -->
+ <scala-2.12.version>2.12.14</scala-2.12.version><!-- Spark -->
<smallrye.reactive.messaging.camel.version>3.5.0</smallrye.reactive.messaging.camel.version><!-- @sync io.quarkus:quarkus-bom:${quarkus.version} dep:io.smallrye.reactive:smallrye-reactive-messaging-provider -->
<spring.version>${spring5-version}</spring.version>
<snakeyaml.version>${snakeyaml-version}</snakeyaml.version>