You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2026/03/05 13:28:34 UTC
(hadoop) branch trunk updated: HADOOP-19733. S3A: load credentials providers using configured classloader (#8048)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2adc485f145 HADOOP-19733. S3A: load credentials providers using configured classloader (#8048)
2adc485f145 is described below
commit 2adc485f14549c7f4e53dfc6894f34d71eef4f46
Author: Brandon Vincent <br...@gmail.com>
AuthorDate: Thu Mar 5 05:28:27 2026 -0800
HADOOP-19733. S3A: load credentials providers using configured classloader (#8048)
Follow-up to HADOOP-17372 and HADOOP-18993.
Contributed by Brandon Vincent
---
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 17 ++++-
.../apache/hadoop/fs/s3a/auth/SignerFactory.java | 2 +-
.../s3a/ITestS3AFileSystemIsolatedClassloader.java | 82 ++++++++++++++++++++--
3 files changed, 95 insertions(+), 6 deletions(-)
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index af470812021..850f13e30bf 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -639,6 +639,10 @@ public static long dateToLong(final Date date) {
* <li>a public default constructor.</li>
* </ol>
*
+ * Uses the configuration's class loader, to respect the setting of
+ * {@link Constants#AWS_S3_CLASSLOADER_ISOLATION}. For backwards
+ * compatibility, if configuration is null, a default classloader is used.
+ *
* @param className name of class for which instance is to be created
* @param conf configuration
* @param uri URI of the FS
@@ -657,7 +661,18 @@ public static <InstanceT> InstanceT getInstanceFromReflection(String className,
String methodName,
String configKey) throws IOException {
try {
- Class<?> instanceClass = S3AUtils.class.getClassLoader().loadClass(className);
+ ClassLoader classLoader;
+ if (conf != null) {
+ classLoader = conf.getClassLoader();
+ LOG.debug("Loading class {} with Configuration classloader {}",
+ className, classLoader);
+ } else {
+ classLoader = S3AUtils.class.getClassLoader();
+ LOG.debug("Loading class {} with default classloader {}",
+ className, classLoader);
+ }
+
+ Class<?> instanceClass = classLoader.loadClass(className);
if (Modifier.isAbstract(instanceClass.getModifiers())) {
throw isAbstract(uri, className, configKey);
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
index 153ce1c8f0b..739490941f4 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
@@ -126,7 +126,7 @@ public static Signer createSigner(String signerType, final Configuration conf, S
LOG.debug("Signer class from {} and key {} is {}", signerType, configKey, className);
Signer signer =
- S3AUtils.getInstanceFromReflection(className, null, null, Signer.class, "create",
+ S3AUtils.getInstanceFromReflection(className, conf, null, Signer.class, "create",
configKey);
requireNonNull(conf);
if (signer instanceof Configurable sc) {
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemIsolatedClassloader.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemIsolatedClassloader.java
index 947bd8ab560..98e924d8a00 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemIsolatedClassloader.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemIsolatedClassloader.java
@@ -20,14 +20,22 @@
import java.io.IOException;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.s3a.impl.InstantiationIOException;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
/**
* Checks that classloader isolation for loading extension classes is applied
@@ -37,10 +45,33 @@
*/
public class ITestS3AFileSystemIsolatedClassloader extends AbstractS3ATestBase {
+ private static String customClassName = "custom.class.name";
+
+ private static class CustomCredentialsProvider implements AwsCredentialsProvider {
+
+ CustomCredentialsProvider() {
+ }
+
+ @Override
+ public AwsCredentials resolveCredentials() {
+ return null;
+ }
+
+ }
+
private static class CustomClassLoader extends ClassLoader {
}
- private final ClassLoader customClassLoader = new CustomClassLoader();
+ private final ClassLoader customClassLoader = spy(new CustomClassLoader());
+ {
+ try {
+ doReturn(CustomCredentialsProvider.class)
+ .when(customClassLoader)
+ .loadClass(customClassName);
+ } catch (ClassNotFoundException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
private S3AFileSystem createNewTestFs(Configuration conf) throws IOException {
S3AFileSystem fs = new S3AFileSystem();
@@ -87,8 +118,15 @@ private Map<String, String> mapOf(String key, String value) {
return m;
}
+ private Map<String, String> mapOf(String key1, String value1, String key2, String value2) {
+ HashMap<String, String> m = new HashMap<>();
+ m.put(key1, value1);
+ m.put(key2, value2);
+ return m;
+ }
+
@Test
- public void defaultIsolatedClassloader() throws IOException {
+ public void defaultIsolatedClassloader() throws Exception {
assertInNewFilesystem(mapOf(), (fs) -> {
Assertions.assertThat(fs.getConf().getClassLoader())
.describedAs("The classloader used to load s3a fs extensions")
@@ -100,10 +138,21 @@ public void defaultIsolatedClassloader() throws IOException {
.isEqualTo(fs.getClass().getClassLoader())
.describedAs("the classloader that loaded the fs");
});
+
+ InstantiationIOException ex = intercept(
+ InstantiationIOException.class,
+ () -> assertInNewFilesystem(
+ mapOf(Constants.AWS_CREDENTIALS_PROVIDER, customClassName),
+ (fs) -> {}));
+
+ Assertions.assertThat(ex.getCause())
+ .describedAs("cause")
+ .isInstanceOf(ClassNotFoundException.class)
+ .hasMessageContaining(customClassName);
}
@Test
- public void isolatedClassloader() throws IOException {
+ public void isolatedClassloader() throws Exception {
assertInNewFilesystem(mapOf(Constants.AWS_S3_CLASSLOADER_ISOLATION, "true"), (fs) -> {
Assertions.assertThat(fs.getConf().getClassLoader())
.describedAs("The classloader used to load s3a fs extensions")
@@ -115,11 +164,26 @@ public void isolatedClassloader() throws IOException {
.isEqualTo(fs.getClass().getClassLoader())
.describedAs("the classloader that loaded the fs");
});
+
+ InstantiationIOException ex = intercept(
+ InstantiationIOException.class,
+ () -> assertInNewFilesystem(
+ mapOf(Constants.AWS_S3_CLASSLOADER_ISOLATION, "true",
+ Constants.AWS_CREDENTIALS_PROVIDER, customClassName),
+ (fs) -> {}));
+
+ Assertions.assertThat(ex.getCause())
+ .describedAs("cause")
+ .isInstanceOf(ClassNotFoundException.class)
+ .hasMessageContaining(customClassName);
}
@Test
public void notIsolatedClassloader() throws IOException {
- assertInNewFilesystem(mapOf(Constants.AWS_S3_CLASSLOADER_ISOLATION, "false"), (fs) -> {
+ Map<String, String> confToSet =
+ mapOf(Constants.AWS_S3_CLASSLOADER_ISOLATION, "false",
+ Constants.AWS_CREDENTIALS_PROVIDER, customClassName);
+ assertInNewFilesystem(confToSet, (fs) -> {
Assertions.assertThat(fs.getConf().getClassLoader())
.describedAs("The classloader used to load s3a fs extensions")
.isEqualTo(Thread.currentThread().getContextClassLoader())
@@ -129,6 +193,16 @@ public void notIsolatedClassloader() throws IOException {
.describedAs("The classloader used to load s3a fs extensions")
.isNotEqualTo(fs.getClass().getClassLoader())
.describedAs("the classloader that loaded the fs");
+
+ S3AFileSystem s3a = (S3AFileSystem) fs;
+ List<AwsCredentialsProvider> providers =
+ s3a.getS3AInternals().shareCredentials("test").getProviders();
+ Assertions.assertThat(providers)
+ .describedAs("providers")
+ .hasSize(1);
+ Assertions.assertThat(providers.get(0))
+ .describedAs("provider")
+ .isInstanceOf(CustomCredentialsProvider.class);
});
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org