You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2022/02/04 16:22:53 UTC
[hadoop] 03/03: HADOOP-18085. S3 SDK Upgrade causes AccessPoint ARN endpoint mistranslation (#3902)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 87ff57765a2dfdb06da28bf09bb90545e0f7d846
Author: Petre Bogdan Stolojan <st...@gmail.com>
AuthorDate: Fri Feb 4 15:37:08 2022 +0000
HADOOP-18085. S3 SDK Upgrade causes AccessPoint ARN endpoint mistranslation (#3902)
Part of HADOOP-17198. Support S3 Access Points.
HADOOP-18068. "upgrade AWS SDK to 1.12.132" broke the access point endpoint
translation.
Correct endpoints should start with "s3-accesspoint.", after SDK upgrade they start with
"s3.accesspoint-" which messes up tests + region detection by the SDK.
Contributed by Bogdan Stolojan
Change-Id: I0c0181628ab803afc39036003777eaec79aa378c
---
.../java/org/apache/hadoop/fs/s3a/ArnResource.java | 5 +-
.../org/apache/hadoop/fs/s3a/TestArnResource.java | 53 +++++++++++++++-------
2 files changed, 39 insertions(+), 19 deletions(-)
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java
index 7c866ac..0294f77 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java
@@ -21,12 +21,12 @@ package org.apache.hadoop.fs.s3a;
import javax.annotation.Nonnull;
import com.amazonaws.arn.Arn;
-import com.amazonaws.regions.RegionUtils;
/**
* Represents an Arn Resource, this can be an accesspoint or bucket.
*/
public final class ArnResource {
+ private final static String ACCESSPOINT_ENDPOINT_FORMAT = "s3-accesspoint.%s.amazonaws.com";
/**
* Resource name.
@@ -106,8 +106,7 @@ public final class ArnResource {
* @return resource endpoint.
*/
public String getEndpoint() {
- return RegionUtils.getRegion(accessPointRegionKey)
- .getServiceEndpoint("s3");
+ return String.format(ACCESSPOINT_ENDPOINT_FORMAT, region);
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java
index 5ac4775..36381bf 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java
@@ -39,40 +39,44 @@ import static org.apache.hadoop.test.LambdaTestUtils.intercept;
public class TestArnResource extends HadoopTestBase {
private final static Logger LOG = LoggerFactory.getLogger(TestArnResource.class);
+ private final static String MOCK_ACCOUNT = "123456789101";
+
@Test
public void parseAccessPointFromArn() throws IllegalArgumentException {
describe("Parse AccessPoint ArnResource from arn string");
String accessPoint = "testAp";
- String accountId = "123456789101";
String[][] regionPartitionEndpoints = new String[][] {
- {Regions.EU_WEST_1.getName(), "aws", "eu-west-1.amazonaws.com"},
- {Regions.US_GOV_EAST_1.getName(), "aws-us-gov",
- "us-gov-east-1.amazonaws.com"},
- {Regions.CN_NORTH_1.getName(), "aws-cn", "cn-north-1.amazonaws.com"},
+ {Regions.EU_WEST_1.getName(), "aws"},
+ {Regions.US_GOV_EAST_1.getName(), "aws-us-gov"},
+ {Regions.CN_NORTH_1.getName(), "aws-cn"},
};
for (String[] testPair : regionPartitionEndpoints) {
String region = testPair[0];
String partition = testPair[1];
- String endpoint = testPair[2];
-
- // arn:partition:service:region:account-id:resource-type/resource-id
- String arn = String.format("arn:%s:s3:%s:%s:accesspoint/%s", partition, region, accountId,
- accessPoint);
- ArnResource resource = ArnResource.accessPointFromArn(arn);
- assertEquals("Arn does not match", arn, resource.getFullArn());
+ ArnResource resource = getArnResourceFrom(partition, region, MOCK_ACCOUNT, accessPoint);
assertEquals("Access Point name does not match", accessPoint, resource.getName());
- assertEquals("Account Id does not match", accountId, resource.getOwnerAccountId());
+ assertEquals("Account Id does not match", MOCK_ACCOUNT, resource.getOwnerAccountId());
assertEquals("Region does not match", region, resource.getRegion());
- Assertions.assertThat(resource.getEndpoint())
- .describedAs("Endpoint does not match")
- .contains(endpoint);
}
}
@Test
+ public void makeSureEndpointHasTheCorrectFormat() {
+ // Access point (AP) endpoints are different from S3 bucket endpoints, thus when using APs the
+ // endpoints for the client are modified. This test makes sure endpoint is set up correctly.
+ ArnResource accessPoint = getArnResourceFrom("aws", "eu-west-1", MOCK_ACCOUNT,
+ "test");
+ String expected = "s3-accesspoint.eu-west-1.amazonaws.com";
+
+ Assertions.assertThat(accessPoint.getEndpoint())
+ .describedAs("Endpoint has invalid format. Access Point requests will not work")
+ .isEqualTo(expected);
+ }
+
+ @Test
public void invalidARNsMustThrow() throws Exception {
describe("Using an invalid ARN format must throw when initializing an ArnResource.");
@@ -80,6 +84,23 @@ public class TestArnResource extends HadoopTestBase {
ArnResource.accessPointFromArn("invalid:arn:resource"));
}
+ /**
+ * Create an {@link ArnResource} from string components
+ * @param partition - partition for ARN
+ * @param region - region for ARN
+ * @param accountId - accountId for ARN
+ * @param resourceName - ARN resource name
+ * @return ArnResource described by its properties
+ */
+ private ArnResource getArnResourceFrom(String partition, String region, String accountId,
+ String resourceName) {
+ // arn:partition:service:region:account-id:resource-type/resource-id
+ String arn = String.format("arn:%s:s3:%s:%s:accesspoint/%s", partition, region, accountId,
+ resourceName);
+
+ return ArnResource.accessPointFromArn(arn);
+ }
+
private void describe(String message) {
LOG.info(message);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org