You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2025/07/08 15:07:20 UTC
(hadoop) branch branch-3.4.2 updated (8c27442888f -> e5a851d378a)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a change to branch branch-3.4.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
from 8c27442888f HADOOP-19576. S3A: Disable Purging Pending MPUs Before Directory Purge (#7722)
new 09a4950b653 HADOOP-19225. Upgrade Jetty to 9.4.57.v20241219 due to CVE-2024-8184 (#7116)
new e5a851d378a HADOOP-19384. S3A: Add support for ProfileCredentialsProvider (#7284)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
LICENSE-binary | 28 +++---
.../src/main/resources/core-default.xml | 1 +
hadoop-project/pom.xml | 2 +-
.../fs/s3a/auth/ProfileAWSCredentialsProvider.java | 106 +++++++++++++++++++++
.../fs/s3a/TestS3AAWSCredentialsProvider.java | 35 ++++++-
5 files changed, 156 insertions(+), 16 deletions(-)
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 01/02: HADOOP-19225. Upgrade Jetty to 9.4.57.v20241219 due to CVE-2024-8184 (#7116)
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.4.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 09a4950b6538e2d9906d423e74c70589688f7ad7
Author: PJ Fanning <pj...@users.noreply.github.com>
AuthorDate: Tue Jul 8 15:52:52 2025 +0100
HADOOP-19225. Upgrade Jetty to 9.4.57.v20241219 due to CVE-2024-8184 (#7116)
Contributed by PJ Fanning.
---
LICENSE-binary | 28 ++++++++++++++--------------
hadoop-project/pom.xml | 2 +-
2 files changed, 15 insertions(+), 15 deletions(-)
diff --git a/LICENSE-binary b/LICENSE-binary
index 5af73093b36..3ad4b30045d 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -339,20 +339,20 @@ org.apache.solr:solr-solrj:8.11.2
org.apache.yetus:audience-annotations:0.5.0
org.apache.zookeeper:zookeeper:3.8.4
org.codehaus.jettison:jettison:1.5.4
-org.eclipse.jetty:jetty-annotations:9.4.53.v20231009
-org.eclipse.jetty:jetty-http:9.4.53.v20231009
-org.eclipse.jetty:jetty-io:9.4.53.v20231009
-org.eclipse.jetty:jetty-jndi:9.4.53.v20231009
-org.eclipse.jetty:jetty-plus:9.4.53.v20231009
-org.eclipse.jetty:jetty-security:9.4.53.v20231009
-org.eclipse.jetty:jetty-server:9.4.53.v20231009
-org.eclipse.jetty:jetty-servlet:9.4.53.v20231009
-org.eclipse.jetty:jetty-util:9.4.53.v20231009
-org.eclipse.jetty:jetty-util-ajax:9.4.53.v20231009
-org.eclipse.jetty:jetty-webapp:9.4.53.v20231009
-org.eclipse.jetty:jetty-xml:9.4.53.v20231009
-org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.53.v20231009
-org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.53.v20231009
+org.eclipse.jetty:jetty-annotations:9.4.57.v20241219
+org.eclipse.jetty:jetty-http:9.4.57.v20241219
+org.eclipse.jetty:jetty-io:9.4.57.v20241219
+org.eclipse.jetty:jetty-jndi:9.4.57.v20241219
+org.eclipse.jetty:jetty-plus:9.4.57.v20241219
+org.eclipse.jetty:jetty-security:9.4.57.v20241219
+org.eclipse.jetty:jetty-server:9.4.57.v20241219
+org.eclipse.jetty:jetty-servlet:9.4.57.v20241219
+org.eclipse.jetty:jetty-util:9.4.57.v20241219
+org.eclipse.jetty:jetty-util-ajax:9.4.57.v20241219
+org.eclipse.jetty:jetty-webapp:9.4.57.v20241219
+org.eclipse.jetty:jetty-xml:9.4.57.v20241219
+org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.57.v20241219
+org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.57.v20241219
org.ehcache:ehcache:3.8.2
org.ini4j:ini4j:0.5.4
org.lz4:lz4-java:1.7.1
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 7ddca73055c..7c4e5d78fdd 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -37,7 +37,7 @@
<!--Whether to proceed to next module if any test failures exist-->
<maven.test.failure.ignore>true</maven.test.failure.ignore>
<maven.test.redirectTestOutputToFile>true</maven.test.redirectTestOutputToFile>
- <jetty.version>9.4.53.v20231009</jetty.version>
+ <jetty.version>9.4.57.v20241219</jetty.version>
<test.exclude>_</test.exclude>
<test.exclude.pattern>_</test.exclude.pattern>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 02/02: HADOOP-19384. S3A: Add support for ProfileCredentialsProvider (#7284)
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.4.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit e5a851d378aa7bfbaf13a005f9a5e284111f19e4
Author: Venkatasubrahmanian Narayanan <10...@users.noreply.github.com>
AuthorDate: Tue Jul 8 08:06:27 2025 -0700
HADOOP-19384. S3A: Add support for ProfileCredentialsProvider (#7284)
Contributed by Venkatasubrahmanian Narayanan
---
.../src/main/resources/core-default.xml | 1 +
.../fs/s3a/auth/ProfileAWSCredentialsProvider.java | 106 +++++++++++++++++++++
.../fs/s3a/TestS3AAWSCredentialsProvider.java | 35 ++++++-
3 files changed, 141 insertions(+), 1 deletion(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 4104e304314..12ccac3ee65 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1425,6 +1425,7 @@
token binding it may be used
to communicate wih the STS endpoint to request session/role
credentials.
+ org.apache.hadoop.fs.s3a.auth.ProfileAWSCredentialsProvider is also supported, but is not enabled by default.
</description>
</property>
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java
new file mode 100644
index 00000000000..09eca499b0e
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.auth;
+
+import java.net.URI;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
+import software.amazon.awssdk.profiles.ProfileFile;
+
+import org.apache.commons.lang3.SystemUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class ProfileAWSCredentialsProvider extends AbstractAWSCredentialProvider {
+ private static final Logger LOG = LoggerFactory.getLogger(ProfileAWSCredentialsProvider.class);
+
+ public static final String NAME
+ = "org.apache.hadoop.fs.s3a.auth.ProfileAWSCredentialsProvider";
+
+ /** Conf setting for credentials file path.*/
+ public static final String PROFILE_FILE = "fs.s3a.auth.profile.file";
+
+ /** Conf setting for profile name.*/
+ public static final String PROFILE_NAME = "fs.s3a.auth.profile.name";
+
+ /** Environment variable for credentials file path.*/
+ public static final String CREDENTIALS_FILE_ENV = "AWS_SHARED_CREDENTIALS_FILE";
+ /** Environment variable for profile name.*/
+ public static final String PROFILE_ENV = "AWS_PROFILE";
+
+ private final ProfileCredentialsProvider pcp;
+
+ private static Path getCredentialsPath(Configuration conf) {
+ String credentialsFile = conf.get(PROFILE_FILE, null);
+ if (credentialsFile == null) {
+ credentialsFile = SystemUtils.getEnvironmentVariable(CREDENTIALS_FILE_ENV, null);
+ if (credentialsFile != null) {
+ LOG.debug("Fetched credentials file path from environment variable");
+ }
+ } else {
+ LOG.debug("Fetched credentials file path from conf");
+ }
+ if (credentialsFile == null) {
+ LOG.debug("Using default credentials file path");
+ return FileSystems.getDefault().getPath(SystemUtils.getUserHome().getPath(),
+ ".aws", "credentials");
+ } else {
+ return FileSystems.getDefault().getPath(credentialsFile);
+ }
+ }
+
+ private static String getCredentialsName(Configuration conf) {
+ String profileName = conf.get(PROFILE_NAME, null);
+ if (profileName == null) {
+ profileName = SystemUtils.getEnvironmentVariable(PROFILE_ENV, null);
+ if (profileName == null) {
+ profileName = "default";
+ LOG.debug("Using default profile name");
+ } else {
+ LOG.debug("Fetched profile name from environment variable");
+ }
+ } else {
+ LOG.debug("Fetched profile name from conf");
+ }
+ return profileName;
+ }
+
+ public ProfileAWSCredentialsProvider(URI uri, Configuration conf) {
+ super(uri, conf);
+ ProfileCredentialsProvider.Builder builder = ProfileCredentialsProvider.builder();
+ builder.profileName(getCredentialsName(conf))
+ .profileFile(ProfileFile.builder()
+ .content(getCredentialsPath(conf))
+ .type(ProfileFile.Type.CREDENTIALS)
+ .build());
+ pcp = builder.build();
+ }
+
+ public AwsCredentials resolveCredentials() {
+ return pcp.resolveCredentials();
+ }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
index d51bc954a63..c2d82624878 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
@@ -18,8 +18,11 @@
package org.apache.hadoop.fs.s3a;
-import java.io.IOException;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
import java.io.InterruptedIOException;
+import java.io.IOException;
import java.net.URI;
import java.nio.file.AccessDeniedException;
import java.util.ArrayList;
@@ -52,6 +55,7 @@
import org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory;
import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider;
import org.apache.hadoop.fs.s3a.auth.NoAuthWithAWSException;
+import org.apache.hadoop.fs.s3a.auth.ProfileAWSCredentialsProvider;
import org.apache.hadoop.fs.s3a.auth.delegation.CountInvocationsProvider;
import org.apache.hadoop.fs.s3a.impl.InstantiationIOException;
import org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils;
@@ -139,6 +143,35 @@ public void testInstantiationChain() throws Throwable {
assertCredentialProviders(expectedClasses, list);
}
+ @Test
+ public void testProfileAWSCredentialsProvider() throws Throwable {
+ Configuration conf = new Configuration(false);
+ conf.set(AWS_CREDENTIALS_PROVIDER, ProfileAWSCredentialsProvider.NAME);
+ File tempFile = File.createTempFile("testcred", ".conf", new File("target"));
+ tempFile.deleteOnExit();
+ try (FileWriter fileWriter = new FileWriter(tempFile);
+ BufferedWriter bufferedWriter = new BufferedWriter(fileWriter)) {
+ bufferedWriter.write("[default]\n"
+ + "aws_access_key_id = defaultaccesskeyid\n"
+ + "aws_secret_access_key = defaultsecretkeyid\n");
+ bufferedWriter.write("[nondefault]\n"
+ + "aws_access_key_id = nondefaultaccesskeyid\n"
+ + "aws_secret_access_key = nondefaultsecretkeyid\n");
+ }
+ conf.set(ProfileAWSCredentialsProvider.PROFILE_FILE, tempFile.getAbsolutePath());
+ URI testUri = new URI("s3a://bucket1");
+ AWSCredentialProviderList list = createAWSCredentialProviderList(testUri, conf);
+ assertCredentialProviders(Collections.singletonList(ProfileAWSCredentialsProvider.class), list);
+ AwsCredentials credentials = list.resolveCredentials();
+ Assertions.assertThat(credentials.accessKeyId()).isEqualTo("defaultaccesskeyid");
+ Assertions.assertThat(credentials.secretAccessKey()).isEqualTo("defaultsecretkeyid");
+ conf.set(ProfileAWSCredentialsProvider.PROFILE_NAME, "nondefault");
+ list = createAWSCredentialProviderList(testUri, conf);
+ credentials = list.resolveCredentials();
+ Assertions.assertThat(credentials.accessKeyId()).isEqualTo("nondefaultaccesskeyid");
+ Assertions.assertThat(credentials.secretAccessKey()).isEqualTo("nondefaultsecretkeyid");
+ }
+
@Test
public void testDefaultChain() throws Exception {
URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org