You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2024/10/17 14:27:52 UTC
(spark) branch master updated: [SPARK-49988][BUILD] Remove unused Hadoop dependency management
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new f1f04b097d2f [SPARK-49988][BUILD] Remove unused Hadoop dependency management
f1f04b097d2f is described below
commit f1f04b097d2ff8850426640c77dd43a53f449ad6
Author: Cheng Pan <ch...@apache.org>
AuthorDate: Thu Oct 17 07:27:40 2024 -0700
[SPARK-49988][BUILD] Remove unused Hadoop dependency management
### What changes were proposed in this pull request?
Remove unused vanilla hadoop dependency(and transitive deps) management, i.e. `hadoop-client`, `xerces:xercesImpl`, and inline deps defined in `hadoop3` because it's the only supported hadoop profile.
### Why are the changes needed?
Simplify pom.xml.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass CI and verified runtime jars are not affected by running`dev/test-dependencies.sh`.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #48491 from pan3793/SPARK-49988.
Authored-by: Cheng Pan <ch...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
LICENSE-binary | 1 -
NOTICE-binary | 21 --
pom.xml | 297 ---------------------
project/SparkBuild.scala | 1 -
.../kubernetes/integration-tests/pom.xml | 20 +-
5 files changed, 6 insertions(+), 334 deletions(-)
diff --git a/LICENSE-binary b/LICENSE-binary
index 89826482d363..40d28fbe71e6 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -402,7 +402,6 @@ org.xerial.snappy:snappy-java
org.yaml:snakeyaml
oro:oro
stax:stax-api
-xerces:xercesImpl
core/src/main/java/org/apache/spark/util/collection/TimSort.java
core/src/main/resources/org/apache/spark/ui/static/bootstrap*
diff --git a/NOTICE-binary b/NOTICE-binary
index c4cfe0e9f8b3..3f36596b9d6d 100644
--- a/NOTICE-binary
+++ b/NOTICE-binary
@@ -448,27 +448,6 @@ which has the following notices:
* Alec Wysoker
* Performance and memory usage improvement
-The binary distribution of this product bundles binaries of
-Xerces2 Java Parser 2.9.1,
-which has the following notices:
- * =========================================================================
- == NOTICE file corresponding to section 4(d) of the Apache License, ==
- == Version 2.0, in this case for the Apache Xerces Java distribution. ==
- =========================================================================
-
- Apache Xerces Java
- Copyright 1999-2007 The Apache Software Foundation
-
- This product includes software developed at
- The Apache Software Foundation (http://www.apache.org/).
-
- Portions of this software were originally based on the following:
- - software copyright (c) 1999, IBM Corporation., http://www.ibm.com.
- - software copyright (c) 1999, Sun Microsystems., http://www.sun.com.
- - voluntary contributions made by Paul Eng on behalf of the
- Apache Software Foundation that were originally developed at iClick, Inc.,
- software copyright (c) 1999.
-
Apache Commons Collections
Copyright 2001-2015 The Apache Software Foundation
diff --git a/pom.xml b/pom.xml
index ff15f200e2bb..2e169df7201c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1418,92 +1418,6 @@
<scope>test</scope>
</dependency>
<!-- End of Hadoop 3.x dependencies -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <version>${hadoop.version}</version>
- <scope>${hadoop.deps.scope}</scope>
- <exclusions>
- <exclusion>
- <groupId>org.fusesource.leveldbjni</groupId>
- <artifactId>leveldbjni-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.jboss.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <!-- BeanUtils >= 1.9.0 no longer splits out -core; exclude it -->
- <groupId>commons-beanutils</groupId>
- <artifactId>commons-beanutils-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>servlet-api-2.5</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.jersey-test-framework</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.contribs</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>net.java.dev.jets3t</groupId>
- <artifactId>jets3t</artifactId>
- </exclusion>
- <!-- Hadoop-3.x -->
- <exclusion>
- <groupId>javax.ws.rs</groupId>
- <artifactId>jsr311-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-webapp</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
@@ -1544,16 +1458,6 @@
<version>${bouncycastle.version}</version>
<scope>test</scope>
</dependency>
- <!-- Managed up to match Hadoop in HADOOP-16530 -->
- <!--
- When upgrading `xercesImpl` version, also need to change
- the version definition in `SparkBuild#DependencyOverrides`.
- -->
- <dependency>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- <version>2.12.2</version>
- </dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
@@ -1636,207 +1540,6 @@
<version>1.1.1</version>
<scope>${hadoop.deps.scope}</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
- <version>${yarn.version}</version>
- <scope>${hadoop.deps.scope}</scope>
- <exclusions>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.jboss.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.jersey-test-framework</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.contribs</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>jdk.tools</groupId>
- <artifactId>jdk.tools</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-common</artifactId>
- <version>${yarn.version}</version>
- <scope>${hadoop.deps.scope}</scope>
- <exclusions>
- <exclusion>
- <groupId>asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.jboss.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.jersey-test-framework</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.contribs</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-tests</artifactId>
- <version>${yarn.version}</version>
- <classifier>tests</classifier>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.fusesource.leveldbjni</groupId>
- <artifactId>leveldbjni-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.jboss.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.jersey-test-framework</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.contribs</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <!--
- Hack to exclude org.apache.hadoop:hadoop-yarn-server-resourcemanager:jar:tests.
- For some reasons, SBT starts to pull the dependencies of 'hadoop-yarn-server-tests' above
- with 'tests' classifier after upgrading SBT 1.3 (SPARK-21708). Otherwise, some tests might
- fail, see also SPARK-33104.
- -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
- <version>${yarn.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
- <version>${yarn.version}</version>
- <scope>${hadoop.deps.scope}</scope>
- <exclusions>
- <exclusion>
- <groupId>asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.jboss.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.jersey-test-framework</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.contribs</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index a87e0af0b542..e7f7d68e9848 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -1060,7 +1060,6 @@ object DependencyOverrides {
lazy val guavaVersion = sys.props.get("guava.version").getOrElse("33.1.0-jre")
lazy val settings = Seq(
dependencyOverrides += "com.google.guava" % "guava" % guavaVersion,
- dependencyOverrides += "xerces" % "xercesImpl" % "2.12.2",
dependencyOverrides += "jline" % "jline" % "2.14.6",
dependencyOverrides += "org.apache.avro" % "avro" % "1.11.3")
}
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index 45ce25b8e037..cebef07821f3 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -79,6 +79,12 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>bundle</artifactId>
+ <version>${aws.java.sdk.v2.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
@@ -189,20 +195,6 @@
</build>
<profiles>
- <profile>
- <id>hadoop-3</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
- <dependencies>
- <dependency>
- <groupId>software.amazon.awssdk</groupId>
- <artifactId>bundle</artifactId>
- <version>${aws.java.sdk.v2.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
- </profile>
<profile>
<id>volcano</id>
<dependencies>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org