You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2022/07/25 09:14:56 UTC
[hadoop] branch branch-3.3 updated: HADOOP-18330. S3AFileSystem removes Path when calling createS3Client (#4572)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new fffb0bd6db7 HADOOP-18330. S3AFileSystem removes Path when calling createS3Client (#4572)
fffb0bd6db7 is described below
commit fffb0bd6db743f56fcdc777c80bdc8fef3fea141
Author: ashutoshpant <51...@users.noreply.github.com>
AuthorDate: Thu Jul 21 05:16:39 2022 -0400
HADOOP-18330. S3AFileSystem removes Path when calling createS3Client (#4572)
Adds a new parameter object in s3ClientCreationParameters that holds
the full s3a path URI
Contributed by Ashutosh Pant
---
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 1 +
.../org/apache/hadoop/fs/s3a/S3ClientFactory.java | 27 ++++++++++++++++++++++
.../hadoop/fs/s3a/ITestS3AEndpointRegion.java | 1 +
.../ITestSessionDelegationInFileystem.java | 1 +
4 files changed, 30 insertions(+)
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index ac2099bedce..f523c0f4318 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -888,6 +888,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
S3ClientFactory.S3ClientCreationParameters parameters = null;
parameters = new S3ClientFactory.S3ClientCreationParameters()
.withCredentialSet(credentials)
+ .withPathUri(name)
.withEndpoint(endpoint)
.withMetrics(statisticsContext.newStatisticsFromAwsSdk())
.withPathStyleAccess(conf.getBoolean(PATH_STYLE_ACCESS, false))
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
index 34674c78890..2216599679b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
@@ -115,6 +115,12 @@ public interface S3ClientFactory {
*/
private String userAgentSuffix = "";
+ /**
+ * S3A path.
+ * added in HADOOP-18330
+ */
+ private URI pathUri;
+
/**
* List of request handlers to include in the chain
* of request execution in the SDK.
@@ -264,5 +270,26 @@ public interface S3ClientFactory {
public Map<String, String> getHeaders() {
return headers;
}
+
+ /**
+ * Get the full s3 path.
+ * added in HADOOP-18330
+ * @return path URI
+ */
+ public URI getPathUri() {
+ return pathUri;
+ }
+
+ /**
+ * Set full s3a path.
+ * added in HADOOP-18330
+ * @param value new value
+ * @return the builder
+ */
+ public S3ClientCreationParameters withPathUri(
+ final URI value) {
+ pathUri = value;
+ return this;
+ }
}
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java
index 761dd558063..40217484f27 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java
@@ -176,6 +176,7 @@ public class ITestS3AEndpointRegion extends AbstractS3ATestBase {
S3ClientFactory.S3ClientCreationParameters parameters
= new S3ClientFactory.S3ClientCreationParameters()
.withCredentialSet(new AnonymousAWSCredentialsProvider())
+ .withPathUri(new URI("s3a://localhost/"))
.withEndpoint(endpoint)
.withMetrics(new EmptyS3AStatisticsContext()
.newStatisticsFromAwsSdk());
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java
index bc223bad457..03b7a0b07ae 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java
@@ -589,6 +589,7 @@ public class ITestSessionDelegationInFileystem extends AbstractDelegationIT {
S3ClientFactory.S3ClientCreationParameters parameters = null;
parameters = new S3ClientFactory.S3ClientCreationParameters()
.withCredentialSet(testingCreds)
+ .withPathUri(new URI("s3a://localhost/"))
.withEndpoint(DEFAULT_ENDPOINT)
.withMetrics(new EmptyS3AStatisticsContext()
.newStatisticsFromAwsSdk())
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org