You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2022/06/24 12:58:16 UTC
[hadoop] 01/02: HADOOP-18068. upgrade AWS SDK to 1.12.132 (#3864)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit cf2675f7218ef292c0fb89238f31d41cf47041a0
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Tue Jan 18 10:31:28 2022 +0000
HADOOP-18068. upgrade AWS SDK to 1.12.132 (#3864)
With this update, the versions of key shaded dependencies are
jackson 2.12.3
httpclient 4.5.13
Contributed by Steve Loughran
Change-Id: Id9ed677352d54e8ea71b9729b6a4bfedc6142825
---
NOTICE-binary | 2 +-
hadoop-project/pom.xml | 2 +-
.../src/site/markdown/tools/hadoop-aws/testing.md | 1 +
.../org/apache/hadoop/fs/s3a/TestArnResource.java | 19 +++++++++++++++----
4 files changed, 18 insertions(+), 6 deletions(-)
diff --git a/NOTICE-binary b/NOTICE-binary
index 60a71c6ce3d..2189de34e37 100644
--- a/NOTICE-binary
+++ b/NOTICE-binary
@@ -66,7 +66,7 @@ available from http://www.digip.org/jansson/.
AWS SDK for Java
-Copyright 2010-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Copyright 2010-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
This product includes software developed by
Amazon Technologies, Inc (http://www.amazon.com/).
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 0f42e7e231c..bbf242fa54a 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -187,7 +187,7 @@
<exec-maven-plugin.version>1.3.1</exec-maven-plugin.version>
<make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
<surefire.fork.timeout>900</surefire.fork.timeout>
- <aws-java-sdk.version>1.11.1026</aws-java-sdk.version>
+ <aws-java-sdk.version>1.12.132</aws-java-sdk.version>
<hsqldb.version>2.3.4</hsqldb.version>
<frontend-maven-plugin.version>1.11.2</frontend-maven-plugin.version>
<jasmine-maven-plugin.version>2.1</jasmine-maven-plugin.version>
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
index f5da283db36..8dec802dc27 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
@@ -1662,6 +1662,7 @@ to AWS services.
* Try and get other people, especially anyone with their own endpoints,
apps or different deployment environments, to run their own tests.
* Run the load tests, especially `ILoadTestS3ABulkDeleteThrottling`.
+* Checkout cloudstore, build it against your version of hadoop, then use its CLI to run some commands (`storediag` etc)
### Dealing with Deprecated APIs and New Features
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java
index 97069c7c8a1..5ac47752ec8 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs.s3a;
import com.amazonaws.regions.Regions;
+import org.assertj.core.api.Assertions;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -27,6 +28,14 @@ import org.apache.hadoop.test.HadoopTestBase;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+/**
+ * Verifies the mapping of ARN declaration of resource to the associated
+ * access point.
+ * The region mapping assertions have been brittle to changes across AWS SDK
+ * versions to only verify partial matches, rather than the FQDN of the
+ * endpoints.
+ *
+ */
public class TestArnResource extends HadoopTestBase {
private final static Logger LOG = LoggerFactory.getLogger(TestArnResource.class);
@@ -37,10 +46,10 @@ public class TestArnResource extends HadoopTestBase {
String accessPoint = "testAp";
String accountId = "123456789101";
String[][] regionPartitionEndpoints = new String[][] {
- {Regions.EU_WEST_1.getName(), "aws", "s3-accesspoint.eu-west-1.amazonaws.com"},
+ {Regions.EU_WEST_1.getName(), "aws", "eu-west-1.amazonaws.com"},
{Regions.US_GOV_EAST_1.getName(), "aws-us-gov",
- "s3-accesspoint.us-gov-east-1.amazonaws.com"},
- {Regions.CN_NORTH_1.getName(), "aws-cn", "s3-accesspoint.cn-north-1.amazonaws.com.cn"},
+ "us-gov-east-1.amazonaws.com"},
+ {Regions.CN_NORTH_1.getName(), "aws-cn", "cn-north-1.amazonaws.com"},
};
for (String[] testPair : regionPartitionEndpoints) {
@@ -57,7 +66,9 @@ public class TestArnResource extends HadoopTestBase {
assertEquals("Access Point name does not match", accessPoint, resource.getName());
assertEquals("Account Id does not match", accountId, resource.getOwnerAccountId());
assertEquals("Region does not match", region, resource.getRegion());
- assertEquals("Endpoint does not match", endpoint, resource.getEndpoint());
+ Assertions.assertThat(resource.getEndpoint())
+ .describedAs("Endpoint does not match")
+ .contains(endpoint);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org