You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by rz...@apache.org on 2024/04/11 13:26:59 UTC
(storm) 01/01: STORM-4053 - Fix ambiguity in hadoop dependencies and include missing Client API / Runtime
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch 4053
in repository https://gitbox.apache.org/repos/asf/storm.git
commit eaca0a0850174ff7b9e788342e6362e6971f9a5c
Author: Richard Zowalla <ri...@hs-heilbronn.de>
AuthorDate: Thu Apr 11 14:32:52 2024 +0200
STORM-4053 - Fix ambiguity in hadoop dependencies and include missing Client API / Runtime
---
DEPENDENCY-LICENSES | 3 +++
external/storm-hdfs-blobstore/pom.xml | 33 ++++++++++++++++----------------
external/storm-hdfs-oci/pom.xml | 25 ++++++++----------------
external/storm-hdfs/pom.xml | 36 ++++++++++++-----------------------
4 files changed, 39 insertions(+), 58 deletions(-)
diff --git a/DEPENDENCY-LICENSES b/DEPENDENCY-LICENSES
index e34b9b115..6394648e9 100644
--- a/DEPENDENCY-LICENSES
+++ b/DEPENDENCY-LICENSES
@@ -54,6 +54,8 @@ List of third-party dependencies grouped by their license type.
* Apache Hadoop Annotations (org.apache.hadoop:hadoop-annotations:3.3.6 - no url defined)
* Apache Hadoop Archives (org.apache.hadoop:hadoop-archives:3.3.6 - no url defined)
* Apache Hadoop Auth (org.apache.hadoop:hadoop-auth:3.3.6 - no url defined)
+ * Apache Hadoop Client API (org.apache.hadoop:hadoop-client-api:3.3.6 - no url defined)
+ * Apache Hadoop Client Runtime (org.apache.hadoop:hadoop-client-runtime:3.3.6 - no url defined)
* Apache Hadoop Common (org.apache.hadoop:hadoop-common:3.3.6 - no url defined)
* Apache Hadoop HDFS (org.apache.hadoop:hadoop-hdfs:3.3.6 - no url defined)
* Apache Hadoop HDFS Client (org.apache.hadoop:hadoop-hdfs-client:3.3.6 - no url defined)
@@ -511,6 +513,7 @@ List of third-party dependencies grouped by their license type.
Common Development and Distribution License (CDDL) v1.1
* Java Transaction API (javax.transaction:jta:1.1 - http://java.sun.com/products/jta)
+ * jsp-api (javax.servlet.jsp:jsp-api:2.1 - no url defined)
* transaction-api (javax.transaction:transaction-api:1.1 - no url defined)
Common Development and Distribution License (CDDL) v1.1, The GNU General Public License (GPL), Version 2, With Classpath Exception
diff --git a/external/storm-hdfs-blobstore/pom.xml b/external/storm-hdfs-blobstore/pom.xml
index 88d73dba9..1e566433f 100644
--- a/external/storm-hdfs-blobstore/pom.xml
+++ b/external/storm-hdfs-blobstore/pom.xml
@@ -28,6 +28,10 @@
<artifactId>storm-hdfs-blobstore</artifactId>
<dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-client</artifactId>
@@ -44,25 +48,13 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.woodstox</groupId>
- <artifactId>woodstox-core</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-collections</groupId>
- <artifactId>commons-collections</artifactId>
+ <artifactId>hadoop-client-api</artifactId>
+ <version>${hadoop.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-configuration2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
+ <version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
@@ -102,6 +94,13 @@
<configuration>
<reuseForks>false</reuseForks>
<forkCount>1</forkCount>
+ <!-- We exclude these dependencies from the test classpath as they have some ambiguity with classes from the
+ HDFS mini cluster used for testing. However, users need these dependencies, so we declare it in our POM
+ -->
+ <classpathDependencyExcludes>
+ <classpathDependencyExcludes>org.apache.hadoop:hadoop-client-api</classpathDependencyExcludes>
+ <classpathDependencyExcludes>org.apache.hadoop:hadoop-client-runtime</classpathDependencyExcludes>
+ </classpathDependencyExcludes>
</configuration>
</plugin>
<plugin>
diff --git a/external/storm-hdfs-oci/pom.xml b/external/storm-hdfs-oci/pom.xml
index fee84ee7c..449d1ec0e 100644
--- a/external/storm-hdfs-oci/pom.xml
+++ b/external/storm-hdfs-oci/pom.xml
@@ -27,6 +27,10 @@
<artifactId>storm-hdfs-oci</artifactId>
<dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-server</artifactId>
@@ -43,26 +47,13 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <artifactId>hadoop-client-api</artifactId>
<version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <artifactId>*</artifactId>
- <groupId>*</groupId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
- <groupId>com.fasterxml.woodstox</groupId>
- <artifactId>woodstox-core</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-collections</groupId>
- <artifactId>commons-collections</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-configuration2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
+ <version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
diff --git a/external/storm-hdfs/pom.xml b/external/storm-hdfs/pom.xml
index 22c777467..0f11308b1 100644
--- a/external/storm-hdfs/pom.xml
+++ b/external/storm-hdfs/pom.xml
@@ -28,6 +28,10 @@
<artifactId>storm-hdfs</artifactId>
<dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-client</artifactId>
@@ -69,31 +73,8 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <artifactId>hadoop-client-runtime</artifactId>
<version>${hadoop.version}</version>
- <exclusions>
- <!-- bring in a lot of stuff which isn't needed here -->
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.woodstox</groupId>
- <artifactId>woodstox-core</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-collections</groupId>
- <artifactId>commons-collections</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-configuration2</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
@@ -133,6 +114,13 @@
<configuration>
<reuseForks>false</reuseForks>
<forkCount>1</forkCount>
+ <!-- We exclude these dependencies from the test classpath as they have some ambiguity with classes from the
+ HDFS mini cluster used for testing. However, users need these dependencies, so we declare it in our POM
+ -->
+ <classpathDependencyExcludes>
+ <classpathDependencyExcludes>org.apache.hadoop:hadoop-client-api</classpathDependencyExcludes>
+ <classpathDependencyExcludes>org.apache.hadoop:hadoop-client-runtime</classpathDependencyExcludes>
+ </classpathDependencyExcludes>
</configuration>
</plugin>
<plugin>