You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by pp...@apache.org on 2020/10/26 12:23:55 UTC
[camel-quarkus] branch master updated: Document Spark peculiarities
#1928
This is an automated email from the ASF dual-hosted git repository.
ppalaga pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel-quarkus.git
The following commit(s) were added to refs/heads/master by this push:
new 57cb62b Document Spark peculiarities #1928
57cb62b is described below
commit 57cb62bb98a847ec6cb4315407e81a6256805432
Author: Peter Palaga <pp...@redhat.com>
AuthorDate: Mon Oct 26 12:20:08 2020 +0100
Document Spark peculiarities #1928
---
.../ROOT/pages/reference/extensions/spark.adoc | 51 ++++++++++++++++++++++
extensions-jvm/spark/bom/pom.xml | 39 +++++++++++++++++
.../quarkus/component/spark/it/SparkTest.java | 4 +-
.../spark/runtime/src/main/doc/limitations.adoc | 3 ++
.../spark/runtime/src/main/doc/usage.adoc | 40 +++++++++++++++++
poms/bom/pom.xml | 37 ----------------
6 files changed, 135 insertions(+), 39 deletions(-)
diff --git a/docs/modules/ROOT/pages/reference/extensions/spark.adoc b/docs/modules/ROOT/pages/reference/extensions/spark.adoc
index 2133cd7..55ff0c1 100644
--- a/docs/modules/ROOT/pages/reference/extensions/spark.adoc
+++ b/docs/modules/ROOT/pages/reference/extensions/spark.adoc
@@ -31,3 +31,54 @@ Please refer to the above link for usage and configuration details.
----
Check the xref:user-guide/index.adoc[User guide] for more information about writing Camel Quarkus applications.
+
+== Usage
+
+=== Special BOM
+
+`camel-quarkus-spark` is special compared to other Camel Quarkus extensions which are managed in
+`camel-quarkus-bom` together with their important dependencies. This is because the Camel Spark extension depends
+on rather old versions of various libraries, that may conflict with their newer versions required by other Quarkus and
+Camel Quarkus extensions. Therefore, `camel-quarkus-spark` extension and its dependencies are managed in a special
+BOM called `camel-quarkus-spark-bom`.
+
+For `camel-quarkus-spark` to work properly in end user applications, `camel-quarkus-spark-bom` need to be
+imported before any other Quarkus or Camel Quarkus BOMs.
+
+[source,xml]
+----
+<dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.camel.quarkus</groupId>
+ <artifactId>camel-quarkus-spark-bom</artifactId>
+ <version>${camel-quarkus.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+
+ <!-- Other BOMs -->
+ <dependency>
+ <groupId>org.apache.camel.quarkus</groupId>
+ <artifactId>camel-quarkus-bom</artifactId>
+ <version>${camel-quarkus.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+</dependencyManagement>
+----
+
+[WARNING]
+====
+The above setup will make `camel-quarkus-spark` work properly but it may break other extensions which are
+sensitive to downgrades of the transitive dependencies done in `camel-quarkus-spark-bom`.
+====
+
+
+== Camel Quarkus limitations
+
+* `spark:dataframe` does not work on Java 9+ because Spark 2.x does not support Java 9+. This will be solved when
+ Spark is upgraded to 3.x in Camel, see https://github.com/apache/camel-quarkus/issues/1955[issue #1955].
+* `spark:hive` is does not work, see https://github.com/apache/camel-quarkus/issues/1956[issue #1956].
+
diff --git a/extensions-jvm/spark/bom/pom.xml b/extensions-jvm/spark/bom/pom.xml
index 0455dda..32d3cb6 100644
--- a/extensions-jvm/spark/bom/pom.xml
+++ b/extensions-jvm/spark/bom/pom.xml
@@ -35,6 +35,45 @@
<dependencyManagement>
<dependencies>
+
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-spark</artifactId>
+ <version>${camel.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId><!-- Should always be available via Quarkus tooling -->
+ </exclusion>
+ <exclusion>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>javax.ws.rs-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>jsr311-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel.quarkus</groupId>
+ <artifactId>camel-quarkus-spark</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel.quarkus</groupId>
+ <artifactId>camel-quarkus-spark-deployment</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
diff --git a/extensions-jvm/spark/integration-test/src/test/java/org/apache/camel/quarkus/component/spark/it/SparkTest.java b/extensions-jvm/spark/integration-test/src/test/java/org/apache/camel/quarkus/component/spark/it/SparkTest.java
index ef794fd..c6cd512 100644
--- a/extensions-jvm/spark/integration-test/src/test/java/org/apache/camel/quarkus/component/spark/it/SparkTest.java
+++ b/extensions-jvm/spark/integration-test/src/test/java/org/apache/camel/quarkus/component/spark/it/SparkTest.java
@@ -49,7 +49,7 @@ class SparkTest {
}
@Test
- // TODO: Spark 2.x does not support Java 9+. The upgrade to Spark 3.x will solve that, see https://issues.apache.org/jira/browse/CAMEL-15650
+ // TODO: Spark 2.x does not support Java 9+ https://github.com/apache/camel-quarkus/issues/1955
@EnabledForJreRange(max = JRE.JAVA_8)
public void conditionalDataframe() throws IOException {
RestAssured.get("/spark/dataframe/Micra/count")
@@ -59,7 +59,7 @@ class SparkTest {
}
@Test
- @Disabled // TODO this does not work on plain Camel either; see https://issues.apache.org/jira/browse/CAMEL-15696
+ @Disabled // TODO this does not work on plain Camel either https://github.com/apache/camel-quarkus/issues/1956
public void hiveCount() throws IOException {
RestAssured.get("/spark/hive/count")
.then()
diff --git a/extensions-jvm/spark/runtime/src/main/doc/limitations.adoc b/extensions-jvm/spark/runtime/src/main/doc/limitations.adoc
new file mode 100644
index 0000000..a73f266
--- /dev/null
+++ b/extensions-jvm/spark/runtime/src/main/doc/limitations.adoc
@@ -0,0 +1,3 @@
+* `spark:dataframe` does not work on Java 9+ because Spark 2.x does not support Java 9+. This will be solved when
+ Spark is upgraded to 3.x in Camel, see https://github.com/apache/camel-quarkus/issues/1955[issue #1955].
+* `spark:hive` is does not work, see https://github.com/apache/camel-quarkus/issues/1956[issue #1956].
diff --git a/extensions-jvm/spark/runtime/src/main/doc/usage.adoc b/extensions-jvm/spark/runtime/src/main/doc/usage.adoc
new file mode 100644
index 0000000..316bbe5
--- /dev/null
+++ b/extensions-jvm/spark/runtime/src/main/doc/usage.adoc
@@ -0,0 +1,40 @@
+=== Special BOM
+
+`camel-quarkus-spark` is special compared to other Camel Quarkus extensions which are managed in
+`camel-quarkus-bom` together with their important dependencies. This is because the Camel Spark extension depends
+on rather old versions of various libraries, that may conflict with their newer versions required by other Quarkus and
+Camel Quarkus extensions. Therefore, `camel-quarkus-spark` extension and its dependencies are managed in a special
+BOM called `camel-quarkus-spark-bom`.
+
+For `camel-quarkus-spark` to work properly in end user applications, `camel-quarkus-spark-bom` need to be
+imported before any other Quarkus or Camel Quarkus BOMs.
+
+[source,xml]
+----
+<dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.camel.quarkus</groupId>
+ <artifactId>camel-quarkus-spark-bom</artifactId>
+ <version>${camel-quarkus.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+
+ <!-- Other BOMs -->
+ <dependency>
+ <groupId>org.apache.camel.quarkus</groupId>
+ <artifactId>camel-quarkus-bom</artifactId>
+ <version>${camel-quarkus.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+</dependencyManagement>
+----
+
+[WARNING]
+====
+The above setup will make `camel-quarkus-spark` work properly but it may break other extensions which are
+sensitive to downgrades of the transitive dependencies done in `camel-quarkus-spark-bom`.
+====
diff --git a/poms/bom/pom.xml b/poms/bom/pom.xml
index b07e266..cc3b74e 100644
--- a/poms/bom/pom.xml
+++ b/poms/bom/pom.xml
@@ -1761,33 +1761,6 @@
</dependency>
<dependency>
<groupId>org.apache.camel</groupId>
- <artifactId>camel-spark</artifactId>
- <version>${camel.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId><!-- Should always be available via Quarkus tooling -->
- </exclusion>
- <exclusion>
- <groupId>javax.ws.rs</groupId>
- <artifactId>javax.ws.rs-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.ws.rs</groupId>
- <artifactId>jsr311-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.camel</groupId>
<artifactId>camel-splunk</artifactId>
<version>${camel.version}</version>
</dependency>
@@ -4574,16 +4547,6 @@
</dependency>
<dependency>
<groupId>org.apache.camel.quarkus</groupId>
- <artifactId>camel-quarkus-spark</artifactId>
- <version>${camel-quarkus.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.camel.quarkus</groupId>
- <artifactId>camel-quarkus-spark-deployment</artifactId>
- <version>${camel-quarkus.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.camel.quarkus</groupId>
<artifactId>camel-quarkus-splunk</artifactId>
<version>${camel-quarkus.version}</version>
</dependency>