You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2026/03/09 11:05:06 UTC
(hadoop) branch branch-3.5.0 updated (bebee55f0ad -> 1786816ea7d)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a change to branch branch-3.5.0
in repository https://gitbox.apache.org/repos/asf/hadoop.git
from bebee55f0ad YARN-11897: Make NMWebservices API backward-compatible after Jersey 2… (#8227) Contributed by Hean-Chhinling.
new 489ffb0ccd4 HADOOP-19733. S3A: load credentials providers using configured classloader (#8048)
new 1786816ea7d HADOOP-19831. Upgrade to Jackson 2.18.6 due to security fixes. (#8296)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
LICENSE-binary | 12 ++--
hadoop-project/pom.xml | 4 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 17 ++++-
.../apache/hadoop/fs/s3a/auth/SignerFactory.java | 2 +-
.../s3a/ITestS3AFileSystemIsolatedClassloader.java | 82 ++++++++++++++++++++--
5 files changed, 103 insertions(+), 14 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 01/02: HADOOP-19733. S3A: load credentials providers using configured classloader (#8048)
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.5.0
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 489ffb0ccd4e601d68cea6ac4a3a76a148fd7ba6
Author: Brandon Vincent <br...@gmail.com>
AuthorDate: Thu Mar 5 05:28:27 2026 -0800
HADOOP-19733. S3A: load credentials providers using configured classloader (#8048)
Follow-up to HADOOP-17372 and HADOOP-18993.
Contributed by Brandon Vincent
---
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 17 ++++-
.../apache/hadoop/fs/s3a/auth/SignerFactory.java | 2 +-
.../s3a/ITestS3AFileSystemIsolatedClassloader.java | 82 ++++++++++++++++++++--
3 files changed, 95 insertions(+), 6 deletions(-)
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index af470812021..850f13e30bf 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -639,6 +639,10 @@ public static long dateToLong(final Date date) {
* <li>a public default constructor.</li>
* </ol>
*
+ * Uses the configuration's class loader, to respect the setting of
+ * {@link Constants#AWS_S3_CLASSLOADER_ISOLATION}. For backwards
+ * compatibility, if configuration is null, a default classloader is used.
+ *
* @param className name of class for which instance is to be created
* @param conf configuration
* @param uri URI of the FS
@@ -657,7 +661,18 @@ public static <InstanceT> InstanceT getInstanceFromReflection(String className,
String methodName,
String configKey) throws IOException {
try {
- Class<?> instanceClass = S3AUtils.class.getClassLoader().loadClass(className);
+ ClassLoader classLoader;
+ if (conf != null) {
+ classLoader = conf.getClassLoader();
+ LOG.debug("Loading class {} with Configuration classloader {}",
+ className, classLoader);
+ } else {
+ classLoader = S3AUtils.class.getClassLoader();
+ LOG.debug("Loading class {} with default classloader {}",
+ className, classLoader);
+ }
+
+ Class<?> instanceClass = classLoader.loadClass(className);
if (Modifier.isAbstract(instanceClass.getModifiers())) {
throw isAbstract(uri, className, configKey);
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
index 153ce1c8f0b..739490941f4 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
@@ -126,7 +126,7 @@ public static Signer createSigner(String signerType, final Configuration conf, S
LOG.debug("Signer class from {} and key {} is {}", signerType, configKey, className);
Signer signer =
- S3AUtils.getInstanceFromReflection(className, null, null, Signer.class, "create",
+ S3AUtils.getInstanceFromReflection(className, conf, null, Signer.class, "create",
configKey);
requireNonNull(conf);
if (signer instanceof Configurable sc) {
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemIsolatedClassloader.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemIsolatedClassloader.java
index 947bd8ab560..98e924d8a00 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemIsolatedClassloader.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemIsolatedClassloader.java
@@ -20,14 +20,22 @@
import java.io.IOException;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.s3a.impl.InstantiationIOException;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
/**
* Checks that classloader isolation for loading extension classes is applied
@@ -37,10 +45,33 @@
*/
public class ITestS3AFileSystemIsolatedClassloader extends AbstractS3ATestBase {
+ private static String customClassName = "custom.class.name";
+
+ private static class CustomCredentialsProvider implements AwsCredentialsProvider {
+
+ CustomCredentialsProvider() {
+ }
+
+ @Override
+ public AwsCredentials resolveCredentials() {
+ return null;
+ }
+
+ }
+
private static class CustomClassLoader extends ClassLoader {
}
- private final ClassLoader customClassLoader = new CustomClassLoader();
+ private final ClassLoader customClassLoader = spy(new CustomClassLoader());
+ {
+ try {
+ doReturn(CustomCredentialsProvider.class)
+ .when(customClassLoader)
+ .loadClass(customClassName);
+ } catch (ClassNotFoundException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
private S3AFileSystem createNewTestFs(Configuration conf) throws IOException {
S3AFileSystem fs = new S3AFileSystem();
@@ -87,8 +118,15 @@ private Map<String, String> mapOf(String key, String value) {
return m;
}
+ private Map<String, String> mapOf(String key1, String value1, String key2, String value2) {
+ HashMap<String, String> m = new HashMap<>();
+ m.put(key1, value1);
+ m.put(key2, value2);
+ return m;
+ }
+
@Test
- public void defaultIsolatedClassloader() throws IOException {
+ public void defaultIsolatedClassloader() throws Exception {
assertInNewFilesystem(mapOf(), (fs) -> {
Assertions.assertThat(fs.getConf().getClassLoader())
.describedAs("The classloader used to load s3a fs extensions")
@@ -100,10 +138,21 @@ public void defaultIsolatedClassloader() throws IOException {
.isEqualTo(fs.getClass().getClassLoader())
.describedAs("the classloader that loaded the fs");
});
+
+ InstantiationIOException ex = intercept(
+ InstantiationIOException.class,
+ () -> assertInNewFilesystem(
+ mapOf(Constants.AWS_CREDENTIALS_PROVIDER, customClassName),
+ (fs) -> {}));
+
+ Assertions.assertThat(ex.getCause())
+ .describedAs("cause")
+ .isInstanceOf(ClassNotFoundException.class)
+ .hasMessageContaining(customClassName);
}
@Test
- public void isolatedClassloader() throws IOException {
+ public void isolatedClassloader() throws Exception {
assertInNewFilesystem(mapOf(Constants.AWS_S3_CLASSLOADER_ISOLATION, "true"), (fs) -> {
Assertions.assertThat(fs.getConf().getClassLoader())
.describedAs("The classloader used to load s3a fs extensions")
@@ -115,11 +164,26 @@ public void isolatedClassloader() throws IOException {
.isEqualTo(fs.getClass().getClassLoader())
.describedAs("the classloader that loaded the fs");
});
+
+ InstantiationIOException ex = intercept(
+ InstantiationIOException.class,
+ () -> assertInNewFilesystem(
+ mapOf(Constants.AWS_S3_CLASSLOADER_ISOLATION, "true",
+ Constants.AWS_CREDENTIALS_PROVIDER, customClassName),
+ (fs) -> {}));
+
+ Assertions.assertThat(ex.getCause())
+ .describedAs("cause")
+ .isInstanceOf(ClassNotFoundException.class)
+ .hasMessageContaining(customClassName);
}
@Test
public void notIsolatedClassloader() throws IOException {
- assertInNewFilesystem(mapOf(Constants.AWS_S3_CLASSLOADER_ISOLATION, "false"), (fs) -> {
+ Map<String, String> confToSet =
+ mapOf(Constants.AWS_S3_CLASSLOADER_ISOLATION, "false",
+ Constants.AWS_CREDENTIALS_PROVIDER, customClassName);
+ assertInNewFilesystem(confToSet, (fs) -> {
Assertions.assertThat(fs.getConf().getClassLoader())
.describedAs("The classloader used to load s3a fs extensions")
.isEqualTo(Thread.currentThread().getContextClassLoader())
@@ -129,6 +193,16 @@ public void notIsolatedClassloader() throws IOException {
.describedAs("The classloader used to load s3a fs extensions")
.isNotEqualTo(fs.getClass().getClassLoader())
.describedAs("the classloader that loaded the fs");
+
+ S3AFileSystem s3a = (S3AFileSystem) fs;
+ List<AwsCredentialsProvider> providers =
+ s3a.getS3AInternals().shareCredentials("test").getProviders();
+ Assertions.assertThat(providers)
+ .describedAs("providers")
+ .hasSize(1);
+ Assertions.assertThat(providers.get(0))
+ .describedAs("provider")
+ .isInstanceOf(CustomCredentialsProvider.class);
});
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 02/02: HADOOP-19831. Upgrade to Jackson 2.18.6 due to security fixes. (#8296)
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.5.0
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 1786816ea7d8491b3a868c775bfd06dd2706503d
Author: PJ Fanning <pj...@users.noreply.github.com>
AuthorDate: Mon Mar 9 12:00:43 2026 +0100
HADOOP-19831. Upgrade to Jackson 2.18.6 due to security fixes. (#8296)
Contributed by PJ Fanning
---
LICENSE-binary | 12 ++++++------
hadoop-project/pom.xml | 4 ++--
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/LICENSE-binary b/LICENSE-binary
index c1328d40e73..3effc6d6fb9 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -224,12 +224,12 @@ com.aliyun:java-trace-api:0.2.11-beta
com.aliyun.oss:aliyun-sdk-oss:3.13.2
com.cedarsoftware:java-util:1.9.0
com.cedarsoftware:json-io:2.5.1
-com.fasterxml.jackson.core:jackson-annotations:2.18.5
-com.fasterxml.jackson.core:jackson-core:2.18.5
-com.fasterxml.jackson.core:jackson-databind:2.18.5
-com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.18.5
-com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.18.5
-com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.18.5
+com.fasterxml.jackson.core:jackson-annotations:2.18.6
+com.fasterxml.jackson.core:jackson-core:2.18.6
+com.fasterxml.jackson.core:jackson-databind:2.18.6
+com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.18.6
+com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.18.6
+com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.18.6
com.fasterxml.uuid:java-uuid-generator:3.1.4
com.fasterxml.woodstox:woodstox-core:5.4.0
com.github.ben-manes.caffeine:caffeine:2.9.3
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 39f9206fbf8..e07af8f56e6 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -76,8 +76,8 @@
<jaxb.version>2.3.9</jaxb.version>
<!-- jackson versions -->
- <jackson2.version>2.18.5</jackson2.version>
- <jackson2.databind.version>2.18.5</jackson2.databind.version>
+ <jackson2.version>2.18.6</jackson2.version>
+ <jackson2.databind.version>2.18.6</jackson2.databind.version>
<!-- httpcomponents versions -->
<httpclient.version>4.5.13</httpclient.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org