You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2022/06/24 12:58:15 UTC
[hadoop] branch branch-3.3.4 updated (5d327b8fe22 -> 744f0bd4f7e)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a change to branch branch-3.3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
from 5d327b8fe22 HADOOP-18044. Hadoop - Upgrade to jQuery 3.6.0 (#3791)
new cf2675f7218 HADOOP-18068. upgrade AWS SDK to 1.12.132 (#3864)
new 744f0bd4f7e HADOOP-18085. S3 SDK Upgrade causes AccessPoint ARN endpoint mistranslation (#3902)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
NOTICE-binary | 2 +-
hadoop-project/pom.xml | 2 +-
.../java/org/apache/hadoop/fs/s3a/ArnResource.java | 5 +-
.../src/site/markdown/tools/hadoop-aws/testing.md | 1 +
.../org/apache/hadoop/fs/s3a/TestArnResource.java | 60 +++++++++++++++++-----
5 files changed, 51 insertions(+), 19 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/02: HADOOP-18068. upgrade AWS SDK to 1.12.132 (#3864)
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit cf2675f7218ef292c0fb89238f31d41cf47041a0
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Tue Jan 18 10:31:28 2022 +0000
HADOOP-18068. upgrade AWS SDK to 1.12.132 (#3864)
With this update, the versions of key shaded dependencies are
jackson 2.12.3
httpclient 4.5.13
Contributed by Steve Loughran
Change-Id: Id9ed677352d54e8ea71b9729b6a4bfedc6142825
---
NOTICE-binary | 2 +-
hadoop-project/pom.xml | 2 +-
.../src/site/markdown/tools/hadoop-aws/testing.md | 1 +
.../org/apache/hadoop/fs/s3a/TestArnResource.java | 19 +++++++++++++++----
4 files changed, 18 insertions(+), 6 deletions(-)
diff --git a/NOTICE-binary b/NOTICE-binary
index 60a71c6ce3d..2189de34e37 100644
--- a/NOTICE-binary
+++ b/NOTICE-binary
@@ -66,7 +66,7 @@ available from http://www.digip.org/jansson/.
AWS SDK for Java
-Copyright 2010-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Copyright 2010-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
This product includes software developed by
Amazon Technologies, Inc (http://www.amazon.com/).
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 0f42e7e231c..bbf242fa54a 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -187,7 +187,7 @@
<exec-maven-plugin.version>1.3.1</exec-maven-plugin.version>
<make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
<surefire.fork.timeout>900</surefire.fork.timeout>
- <aws-java-sdk.version>1.11.1026</aws-java-sdk.version>
+ <aws-java-sdk.version>1.12.132</aws-java-sdk.version>
<hsqldb.version>2.3.4</hsqldb.version>
<frontend-maven-plugin.version>1.11.2</frontend-maven-plugin.version>
<jasmine-maven-plugin.version>2.1</jasmine-maven-plugin.version>
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
index f5da283db36..8dec802dc27 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
@@ -1662,6 +1662,7 @@ to AWS services.
* Try and get other people, especially anyone with their own endpoints,
apps or different deployment environments, to run their own tests.
* Run the load tests, especially `ILoadTestS3ABulkDeleteThrottling`.
+* Checkout cloudstore, build it against your version of hadoop, then use its CLI to run some commands (`storediag` etc)
### Dealing with Deprecated APIs and New Features
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java
index 97069c7c8a1..5ac47752ec8 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs.s3a;
import com.amazonaws.regions.Regions;
+import org.assertj.core.api.Assertions;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -27,6 +28,14 @@ import org.apache.hadoop.test.HadoopTestBase;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+/**
+ * Verifies the mapping of ARN declaration of resource to the associated
+ * access point.
+ * The region mapping assertions have been brittle to changes across AWS SDK
+ * versions to only verify partial matches, rather than the FQDN of the
+ * endpoints.
+ *
+ */
public class TestArnResource extends HadoopTestBase {
private final static Logger LOG = LoggerFactory.getLogger(TestArnResource.class);
@@ -37,10 +46,10 @@ public class TestArnResource extends HadoopTestBase {
String accessPoint = "testAp";
String accountId = "123456789101";
String[][] regionPartitionEndpoints = new String[][] {
- {Regions.EU_WEST_1.getName(), "aws", "s3-accesspoint.eu-west-1.amazonaws.com"},
+ {Regions.EU_WEST_1.getName(), "aws", "eu-west-1.amazonaws.com"},
{Regions.US_GOV_EAST_1.getName(), "aws-us-gov",
- "s3-accesspoint.us-gov-east-1.amazonaws.com"},
- {Regions.CN_NORTH_1.getName(), "aws-cn", "s3-accesspoint.cn-north-1.amazonaws.com.cn"},
+ "us-gov-east-1.amazonaws.com"},
+ {Regions.CN_NORTH_1.getName(), "aws-cn", "cn-north-1.amazonaws.com"},
};
for (String[] testPair : regionPartitionEndpoints) {
@@ -57,7 +66,9 @@ public class TestArnResource extends HadoopTestBase {
assertEquals("Access Point name does not match", accessPoint, resource.getName());
assertEquals("Account Id does not match", accountId, resource.getOwnerAccountId());
assertEquals("Region does not match", region, resource.getRegion());
- assertEquals("Endpoint does not match", endpoint, resource.getEndpoint());
+ Assertions.assertThat(resource.getEndpoint())
+ .describedAs("Endpoint does not match")
+ .contains(endpoint);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/02: HADOOP-18085. S3 SDK Upgrade causes AccessPoint ARN endpoint mistranslation (#3902)
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 744f0bd4f7ea1f8b64bc3c8cc81dfd24cffa2505
Author: Petre Bogdan Stolojan <st...@gmail.com>
AuthorDate: Fri Feb 4 15:37:08 2022 +0000
HADOOP-18085. S3 SDK Upgrade causes AccessPoint ARN endpoint mistranslation (#3902)
Part of HADOOP-17198. Support S3 Access Points.
HADOOP-18068. "upgrade AWS SDK to 1.12.132" broke the access point endpoint
translation.
Correct endpoints should start with "s3-accesspoint.", after SDK upgrade they start with
"s3.accesspoint-" which messes up tests + region detection by the SDK.
Contributed by Bogdan Stolojan
Change-Id: I0c0181628ab803afc39036003777eaec79aa378c
---
.../java/org/apache/hadoop/fs/s3a/ArnResource.java | 5 +-
.../org/apache/hadoop/fs/s3a/TestArnResource.java | 53 +++++++++++++++-------
2 files changed, 39 insertions(+), 19 deletions(-)
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java
index 7c866ac9676..0294f772290 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java
@@ -21,12 +21,12 @@ package org.apache.hadoop.fs.s3a;
import javax.annotation.Nonnull;
import com.amazonaws.arn.Arn;
-import com.amazonaws.regions.RegionUtils;
/**
* Represents an Arn Resource, this can be an accesspoint or bucket.
*/
public final class ArnResource {
+ private final static String ACCESSPOINT_ENDPOINT_FORMAT = "s3-accesspoint.%s.amazonaws.com";
/**
* Resource name.
@@ -106,8 +106,7 @@ public final class ArnResource {
* @return resource endpoint.
*/
public String getEndpoint() {
- return RegionUtils.getRegion(accessPointRegionKey)
- .getServiceEndpoint("s3");
+ return String.format(ACCESSPOINT_ENDPOINT_FORMAT, region);
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java
index 5ac47752ec8..36381bf14b1 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java
@@ -39,39 +39,43 @@ import static org.apache.hadoop.test.LambdaTestUtils.intercept;
public class TestArnResource extends HadoopTestBase {
private final static Logger LOG = LoggerFactory.getLogger(TestArnResource.class);
+ private final static String MOCK_ACCOUNT = "123456789101";
+
@Test
public void parseAccessPointFromArn() throws IllegalArgumentException {
describe("Parse AccessPoint ArnResource from arn string");
String accessPoint = "testAp";
- String accountId = "123456789101";
String[][] regionPartitionEndpoints = new String[][] {
- {Regions.EU_WEST_1.getName(), "aws", "eu-west-1.amazonaws.com"},
- {Regions.US_GOV_EAST_1.getName(), "aws-us-gov",
- "us-gov-east-1.amazonaws.com"},
- {Regions.CN_NORTH_1.getName(), "aws-cn", "cn-north-1.amazonaws.com"},
+ {Regions.EU_WEST_1.getName(), "aws"},
+ {Regions.US_GOV_EAST_1.getName(), "aws-us-gov"},
+ {Regions.CN_NORTH_1.getName(), "aws-cn"},
};
for (String[] testPair : regionPartitionEndpoints) {
String region = testPair[0];
String partition = testPair[1];
- String endpoint = testPair[2];
-
- // arn:partition:service:region:account-id:resource-type/resource-id
- String arn = String.format("arn:%s:s3:%s:%s:accesspoint/%s", partition, region, accountId,
- accessPoint);
- ArnResource resource = ArnResource.accessPointFromArn(arn);
- assertEquals("Arn does not match", arn, resource.getFullArn());
+ ArnResource resource = getArnResourceFrom(partition, region, MOCK_ACCOUNT, accessPoint);
assertEquals("Access Point name does not match", accessPoint, resource.getName());
- assertEquals("Account Id does not match", accountId, resource.getOwnerAccountId());
+ assertEquals("Account Id does not match", MOCK_ACCOUNT, resource.getOwnerAccountId());
assertEquals("Region does not match", region, resource.getRegion());
- Assertions.assertThat(resource.getEndpoint())
- .describedAs("Endpoint does not match")
- .contains(endpoint);
}
}
+ @Test
+ public void makeSureEndpointHasTheCorrectFormat() {
+ // Access point (AP) endpoints are different from S3 bucket endpoints, thus when using APs the
+ // endpoints for the client are modified. This test makes sure endpoint is set up correctly.
+ ArnResource accessPoint = getArnResourceFrom("aws", "eu-west-1", MOCK_ACCOUNT,
+ "test");
+ String expected = "s3-accesspoint.eu-west-1.amazonaws.com";
+
+ Assertions.assertThat(accessPoint.getEndpoint())
+ .describedAs("Endpoint has invalid format. Access Point requests will not work")
+ .isEqualTo(expected);
+ }
+
@Test
public void invalidARNsMustThrow() throws Exception {
describe("Using an invalid ARN format must throw when initializing an ArnResource.");
@@ -80,6 +84,23 @@ public class TestArnResource extends HadoopTestBase {
ArnResource.accessPointFromArn("invalid:arn:resource"));
}
+ /**
+ * Create an {@link ArnResource} from string components
+ * @param partition - partition for ARN
+ * @param region - region for ARN
+ * @param accountId - accountId for ARN
+ * @param resourceName - ARN resource name
+ * @return ArnResource described by its properties
+ */
+ private ArnResource getArnResourceFrom(String partition, String region, String accountId,
+ String resourceName) {
+ // arn:partition:service:region:account-id:resource-type/resource-id
+ String arn = String.format("arn:%s:s3:%s:%s:accesspoint/%s", partition, region, accountId,
+ resourceName);
+
+ return ArnResource.accessPointFromArn(arn);
+ }
+
private void describe(String message) {
LOG.info(message);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org