You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sa...@apache.org on 2026/09/24 17:11:23 UTC
(spark) branch branch-4.x updated: [SPARK-57897][4.X][BUILD] Use AWS SDK bundle in credential-aws
This is an automated email from the ASF dual-hosted git repository.
sarutak pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 71c9c5bbe67c [SPARK-57897][4.X][BUILD] Use AWS SDK bundle in credential-aws
71c9c5bbe67c is described below
commit 71c9c5bbe67c9fbc88375e7c1d3002abd4e756b2
Author: Wenjun <we...@apache.org>
AuthorDate: Fri Sep 25 02:11:08 2026 +0900
[SPARK-57897][4.X][BUILD] Use AWS SDK bundle in credential-aws
### What changes were proposed in this pull request?
Backport #59000 to `branch-4.x`: use the AWS SDK bundle in `credential-aws`.
### Why are the changes needed?
Fix S3A `NoSuchMethodError` caused by mixing modular SDK jars with the relocated bundle, even at the same version.
### Does this PR introduce _any_ user-facing change?
Yes. Fixes S3A linkage failures when both profiles are enabled. Builds with only `credential-aws` use the larger bundle.
### How was this patch tested?
Verified that added/removed lines match the original commit and `git diff --check` passes. Resolved manifest conflicts while preserving `branch-4.x` dependency versions. Tests and manifest generation were not rerun for this backport; original validation is documented in #59000.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-6)
Closes #59011 from ruanwenjun/backport-59000-branch-4.x.
Authored-by: Wenjun <we...@apache.org>
Signed-off-by: Kousuke Saruta <sa...@apache.org>
---
connector/credential-aws/pom.xml | 12 ++++--------
dev/deps/spark-deps-hadoop-3-hive-2.3 | 30 +-----------------------------
2 files changed, 5 insertions(+), 37 deletions(-)
diff --git a/connector/credential-aws/pom.xml b/connector/credential-aws/pom.xml
index fc79304ecaff..1303a805cf14 100644
--- a/connector/credential-aws/pom.xml
+++ b/connector/credential-aws/pom.xml
@@ -49,17 +49,13 @@
<artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
<!--
- Keep pinned to ${aws.java.sdk.v2.version}. When this module is built into the
- assembly alongside the hadoop-cloud profile, this `sts` jar co-exists with
- hadoop-cloud's `bundle` jar and they share the same software.amazon.awssdk.*
- classes (e.g. StsClient). This is safe only because both resolve to the same
- version, making the duplicated classes byte-identical. A divergent version here
- would place duplicate classes at DIFFERENT versions on the classpath and risk
- NoSuchMethodError / LinkageError at runtime.
+ Use the same bundle as hadoop-cloud to avoid mixing shaded and unshaded SDK
+ classes. Even at the same version, modular SDK jars are not binary-compatible
+ with the bundle's relocated third-party dependencies.
-->
<dependency>
<groupId>software.amazon.awssdk</groupId>
- <artifactId>sts</artifactId>
+ <artifactId>bundle</artifactId>
<version>${aws.java.sdk.v2.version}</version>
</dependency>
diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3
index 8c7903be014e..fbf8f06a3404 100644
--- a/dev/deps/spark-deps-hadoop-3-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -12,11 +12,9 @@ aliyun-java-sdk-kms/2.11.0//aliyun-java-sdk-kms-2.11.0.jar
aliyun-java-sdk-ram/3.1.0//aliyun-java-sdk-ram-3.1.0.jar
aliyun-sdk-oss/3.18.1//aliyun-sdk-oss-3.18.1.jar
analyticsaccelerator-s3/1.3.1//analyticsaccelerator-s3-1.3.1.jar
-annotations/2.35.4//annotations-2.35.4.jar
antlr-runtime/3.5.2//antlr-runtime-3.5.2.jar
antlr4-runtime/4.13.1//antlr4-runtime-4.13.1.jar
aopalliance-repackaged/3.0.6//aopalliance-repackaged-3.0.6.jar
-apache-client/2.35.4//apache-client-2.35.4.jar
arpack/3.2.0//arpack-3.2.0.jar
arpack_combined_all/0.1//arpack_combined_all-0.1.jar
arrow-compression/19.0.0//arrow-compression-19.0.0.jar
@@ -26,12 +24,9 @@ arrow-memory-netty-buffer-patch/19.0.0//arrow-memory-netty-buffer-patch-19.0.0.j
arrow-memory-netty/19.0.0//arrow-memory-netty-19.0.0.jar
arrow-vector/19.0.0//arrow-vector-19.0.0.jar
audience-annotations/0.12.0//audience-annotations-0.12.0.jar
-auth/2.35.4//auth-2.35.4.jar
avro-ipc/1.12.1//avro-ipc-1.12.1.jar
avro-mapred/1.12.1//avro-mapred-1.12.1.jar
avro/1.12.1//avro-1.12.1.jar
-aws-core/2.35.4//aws-core-2.35.4.jar
-aws-query-protocol/2.35.4//aws-query-protocol-2.35.4.jar
azure-data-lake-store-sdk/2.3.9//azure-data-lake-store-sdk-2.3.9.jar
azure-keyvault-core/1.0.0//azure-keyvault-core-1.0.0.jar
azure-storage/7.0.1//azure-storage-7.0.1.jar
@@ -40,8 +35,6 @@ breeze-macros_2.13/2.1.0//breeze-macros_2.13-2.1.0.jar
breeze_2.13/2.1.0//breeze_2.13-2.1.0.jar
bundle/2.35.4//bundle-2.35.4.jar
cats-kernel_2.13/2.8.0//cats-kernel_2.13-2.8.0.jar
-checksums-spi/2.35.4//checksums-spi-2.35.4.jar
-checksums/2.35.4//checksums-2.35.4.jar
chill-java/0.10.0//chill-java-0.10.0.jar
chill_2.13/0.10.0//chill_2.13-0.10.0.jar
commons-cli/1.11.0//commons-cli-1.11.0.jar
@@ -71,9 +64,7 @@ derbyshared/10.16.1.1//derbyshared-10.16.1.1.jar
derbytools/10.16.1.1//derbytools-10.16.1.1.jar
dom4j/2.1.4//dom4j-2.1.4.jar
dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar
-endpoints-spi/2.35.4//endpoints-spi-2.35.4.jar
esdk-obs-java/3.20.4.2//esdk-obs-java-3.20.4.2.jar
-eventstream/1.0.1//eventstream-1.0.1.jar
failureaccess/1.0.3//failureaccess-1.0.3.jar
flatbuffers-java/25.2.10//flatbuffers-java-25.2.10.jar
gmetric4j/1.0.10//gmetric4j-1.0.10.jar
@@ -106,15 +97,9 @@ hive-storage-api/2.8.1//hive-storage-api-2.8.1.jar
hk2-api/3.0.6//hk2-api-3.0.6.jar
hk2-locator/3.0.6//hk2-locator-3.0.6.jar
hk2-utils/3.0.6//hk2-utils-3.0.6.jar
-http-auth-aws-eventstream/2.35.4//http-auth-aws-eventstream-2.35.4.jar
-http-auth-aws/2.35.4//http-auth-aws-2.35.4.jar
-http-auth-spi/2.35.4//http-auth-spi-2.35.4.jar
-http-auth/2.35.4//http-auth-2.35.4.jar
-http-client-spi/2.35.4//http-client-spi-2.35.4.jar
httpclient/4.5.14//httpclient-4.5.14.jar
httpcore/4.4.16//httpcore-4.4.16.jar
icu4j/78.3//icu4j-78.3.jar
-identity-spi/2.35.4//identity-spi-2.35.4.jar
ini4j/0.5.4//ini4j-0.5.4.jar
istack-commons-runtime/4.1.2//istack-commons-runtime-4.1.2.jar
ivy/2.5.3//ivy-2.5.3.jar
@@ -159,7 +144,6 @@ jline/2.14.6//jline-2.14.6.jar
jline/3.29.0/jdk8/jline-3.29.0-jdk8.jar
joda-time/2.14.1//joda-time-2.14.1.jar
jpam/1.1//jpam-1.1.jar
-json-utils/2.35.4//json-utils-2.35.4.jar
json/1.8//json-1.8.jar
json4s-ast_2.13/4.0.7//json4s-ast_2.13-4.0.7.jar
json4s-core_2.13/4.0.7//json4s-core_2.13-4.0.7.jar
@@ -211,7 +195,6 @@ metrics-graphite/4.2.37//metrics-graphite-4.2.37.jar
metrics-jmx/4.2.37//metrics-jmx-4.2.37.jar
metrics-json/4.2.37//metrics-json-4.2.37.jar
metrics-jvm/4.2.37//metrics-jvm-4.2.37.jar
-metrics-spi/2.35.4//metrics-spi-2.35.4.jar
minlog/1.3.0//minlog-1.3.0.jar
netty-all/4.2.13.Final//netty-all-4.2.13.Final.jar
netty-buffer/4.2.13.Final//netty-buffer-4.2.13.Final.jar
@@ -225,7 +208,6 @@ netty-codec/4.2.13.Final//netty-codec-4.2.13.Final.jar
netty-common/4.2.13.Final//netty-common-4.2.13.Final.jar
netty-handler-proxy/4.2.13.Final//netty-handler-proxy-4.2.13.Final.jar
netty-handler/4.2.13.Final//netty-handler-4.2.13.Final.jar
-netty-nio-client/2.35.4//netty-nio-client-2.35.4.jar
netty-resolver-dns/4.2.13.Final//netty-resolver-dns-4.2.13.Final.jar
netty-resolver/4.2.13.Final//netty-resolver-4.2.13.Final.jar
netty-tcnative-boringssl-static/2.0.76.Final/linux-aarch_64/netty-tcnative-boringssl-static-2.0.76.Final-linux-aarch_64.jar
@@ -266,14 +248,9 @@ parquet-format-structures/1.17.0//parquet-format-structures-1.17.0.jar
parquet-hadoop/1.17.0//parquet-hadoop-1.17.0.jar
parquet-jackson/1.17.0//parquet-jackson-1.17.0.jar
pickle/1.5//pickle-1.5.jar
-profiles/2.35.4//profiles-2.35.4.jar
-protocol-core/2.35.4//protocol-core-2.35.4.jar
py4j/0.10.9.9//py4j-0.10.9.9.jar
-reactive-streams/1.0.4//reactive-streams-1.0.4.jar
-regions/2.35.4//regions-2.35.4.jar
+reactive-streams/1.0.3//reactive-streams-1.0.3.jar
remotetea-oncrpc/1.1.2//remotetea-oncrpc-1.1.2.jar
-retries-spi/2.35.4//retries-spi-2.35.4.jar
-retries/2.35.4//retries-2.35.4.jar
rocksdbjni/9.8.4//rocksdbjni-9.8.4.jar
scala-compiler/2.13.18//scala-compiler-2.13.18.jar
scala-library/2.13.18//scala-library-2.13.18.jar
@@ -281,7 +258,6 @@ scala-parallel-collections_2.13/1.2.0//scala-parallel-collections_2.13-1.2.0.jar
scala-parser-combinators_2.13/2.4.0//scala-parser-combinators_2.13-2.4.0.jar
scala-reflect/2.13.18//scala-reflect-2.13.18.jar
scala-xml_2.13/2.4.0//scala-xml_2.13-2.4.0.jar
-sdk-core/2.35.4//sdk-core-2.35.4.jar
slf4j-api/2.0.17//slf4j-api-2.0.17.jar
snakeyaml-engine/3.0.1//snakeyaml-engine-3.0.1.jar
snakeyaml/2.5//snakeyaml-2.5.jar
@@ -292,15 +268,11 @@ spire-util_2.13/0.18.0//spire-util_2.13-0.18.0.jar
spire_2.13/0.18.0//spire_2.13-0.18.0.jar
stax-api/1.0.1//stax-api-1.0.1.jar
stream/2.9.8//stream-2.9.8.jar
-sts/2.35.4//sts-2.35.4.jar
super-csv/2.2.0//super-csv-2.2.0.jar
-third-party-jackson-core/2.35.4//third-party-jackson-core-2.35.4.jar
threeten-extra/1.8.0//threeten-extra-1.8.0.jar
tink/1.20.0//tink-1.20.0.jar
transaction-api/1.1//transaction-api-1.1.jar
univocity-parsers/2.9.1//univocity-parsers-2.9.1.jar
-utils-lite/2.35.4//utils-lite-2.35.4.jar
-utils/2.35.4//utils-2.35.4.jar
vertx-auth-common/4.5.26//vertx-auth-common-4.5.26.jar
vertx-core/4.5.26//vertx-core-4.5.26.jar
vertx-uri-template/4.5.26//vertx-uri-template-4.5.26.jar
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org