You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ah...@apache.org on 2023/12/05 09:15:54 UTC
(hadoop) branch feature-HADOOP-18073-sdk-v2-upgrade-3.6-aws created (now d4808360467e)
This is an automated email from the ASF dual-hosted git repository.
ahmar pushed a change to branch feature-HADOOP-18073-sdk-v2-upgrade-3.6-aws
in repository https://gitbox.apache.org/repos/asf/hadoop.git
at d4808360467e preparing for 3.3.6-aws release
This branch includes the following new commits:
new 377cd2bd7a57 HADOOP-18778. Fixes failing tests when CSE is enabled. (#5763)
new 1fbe185f9ba8 HADOOP-18073. S3A: Upgrade AWS SDK to V2 (#5995)
new d5472f9d6db2 HADOOP-18888. S3A. createS3AsyncClient() always enables multipart uploads (#6056)
new c11b6ab8e43d HADOOP-18889. S3A v2 SDK third party support (#6141)
new 4be7d9e30342 HADOOP-18932. S3A. upgrade AWS v2 SDK to 2.20.160 and v1 to 1.12.565 (#6178)
new b7fee114e6f8 HADOOP-18939. NPE in AWS v2 SDK RetryOnErrorCodeCondition.shouldRetry() (#6193)
new 1808e92b0a35 HADOOP-18908. Improve S3A region handling. (#6187)
new f9d14259c52a HADOOP-18946. TestErrorTranslation failure (#6205)
new 8a1eb917ba67 HADOOP-18945. S3A. IAMInstanceCredentialsProvider failing. (#6202)
new e14b08b030d3 HADOOP-18889. Third party storage followup. (#6186)
new 726e2171a665 HADOOP-18948. S3A. Add option fs.s3a.directory.operations.purge.uploads to purge on rename/delete (#6218)
new af32f6c63ed4 HADOOP-18930. Make fs.s3a.create.performance a bucket-wide setting. (#6168)
new 927d3ce54dc8 HADOOP-18915. Tune/extend S3A http connection and thread pool settings (#6180)
new 141b0d35081c HADOOP-18925. S3A: option to enable/disable CopyFromLocalOperation (#6163)
new f7daa6f7d34f S3A: Upgrade AWS SDK version to 2.21.33 for Amazon S3 Express One Zone support (#6306)
new b330e9e34c07 HADOOP-18996. S3A to provide full support for S3 Express One Zone (#6308)
new d4808360467e preparing for 3.3.6-aws release
The 17 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 01/17: HADOOP-18778. Fixes failing tests when CSE is enabled. (#5763)
Posted by ah...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ahmar pushed a commit to branch feature-HADOOP-18073-sdk-v2-upgrade-3.6-aws
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 377cd2bd7a577b47bad182223c529ff87d2243a8
Author: ahmarsuhail <ah...@gmail.com>
AuthorDate: Wed Jul 26 17:26:49 2023 +0100
HADOOP-18778. Fixes failing tests when CSE is enabled. (#5763)
Contributed By: Ahmar Suhail <ah...@amazon.co.uk>
---
.../java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 4 ++--
.../org/apache/hadoop/fs/s3a/auth/RolePolicies.java | 2 +-
.../hadoop/fs/s3a/ITestS3APrefetchingCacheFiles.java | 1 +
.../hadoop/fs/s3a/ITestS3APrefetchingInputStream.java | 3 +++
.../apache/hadoop/fs/s3a/ITestS3ARequesterPays.java | 2 +-
.../apache/hadoop/fs/s3a/auth/ITestAssumeRole.java | 18 ++++++------------
.../fs/s3a/auth/ITestAssumedRoleCommitOperations.java | 3 +--
.../hadoop/fs/s3a/auth/ITestRestrictedReadAccess.java | 3 +--
.../fs/s3a/impl/ITestPartialRenamesDeletes.java | 19 ++++++++++++-------
.../hadoop/fs/s3a/s3guard/ITestS3GuardTool.java | 10 ++++++----
10 files changed, 34 insertions(+), 31 deletions(-)
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 6d9fd76d1346..07a14f63aa41 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -213,7 +213,7 @@ import static org.apache.hadoop.fs.s3a.Listing.toLocatedFileStatusIterator;
import static org.apache.hadoop.fs.s3a.S3AUtils.*;
import static org.apache.hadoop.fs.s3a.Statistic.*;
import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.INITIALIZE_SPAN;
-import static org.apache.hadoop.fs.s3a.auth.RolePolicies.STATEMENT_ALLOW_SSE_KMS_RW;
+import static org.apache.hadoop.fs.s3a.auth.RolePolicies.STATEMENT_ALLOW_KMS_RW;
import static org.apache.hadoop.fs.s3a.auth.RolePolicies.allowS3Operations;
import static org.apache.hadoop.fs.s3a.auth.delegation.S3ADelegationTokens.TokenIssuingPolicy.NoTokensAvailable;
import static org.apache.hadoop.fs.s3a.auth.delegation.S3ADelegationTokens.hasDelegationTokenBinding;
@@ -4218,7 +4218,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
// no attempt is made to qualify KMS access; there's no
// way to predict read keys, and not worried about granting
// too much encryption access.
- statements.add(STATEMENT_ALLOW_SSE_KMS_RW);
+ statements.add(STATEMENT_ALLOW_KMS_RW);
return statements;
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/RolePolicies.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/RolePolicies.java
index aa88a320ff2a..d46c9f6f9215 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/RolePolicies.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/RolePolicies.java
@@ -80,7 +80,7 @@ public final class RolePolicies {
* Statement to allow KMS R/W access access, so full use of
* SSE-KMS.
*/
- public static final Statement STATEMENT_ALLOW_SSE_KMS_RW =
+ public static final Statement STATEMENT_ALLOW_KMS_RW =
statement(true, KMS_ALL_KEYS, KMS_ALL_OPERATIONS);
/**
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingCacheFiles.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingCacheFiles.java
index 6ad8ef58a7f6..e678df700b87 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingCacheFiles.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingCacheFiles.java
@@ -105,6 +105,7 @@ public class ITestS3APrefetchingCacheFiles extends AbstractS3ACostTest {
@Test
public void testCacheFileExistence() throws Throwable {
describe("Verify that FS cache files exist on local FS");
+ skipIfClientSideEncryption();
try (FSDataInputStream in = fs.open(testFile)) {
byte[] buffer = new byte[prefetchBlockSize];
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingInputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingInputStream.java
index a7b59bb5d462..4998cbc946e1 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingInputStream.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingInputStream.java
@@ -106,6 +106,7 @@ public class ITestS3APrefetchingInputStream extends AbstractS3ACostTest {
@Test
public void testReadLargeFileFully() throws Throwable {
describe("read a large file fully, uses S3ACachingInputStream");
+ skipIfClientSideEncryption();
IOStatistics ioStats;
createLargeFile();
@@ -139,6 +140,7 @@ public class ITestS3APrefetchingInputStream extends AbstractS3ACostTest {
public void testReadLargeFileFullyLazySeek() throws Throwable {
describe("read a large file using readFully(position,buffer,offset,length),"
+ " uses S3ACachingInputStream");
+ skipIfClientSideEncryption();
IOStatistics ioStats;
createLargeFile();
@@ -170,6 +172,7 @@ public class ITestS3APrefetchingInputStream extends AbstractS3ACostTest {
@Test
public void testRandomReadLargeFile() throws Throwable {
describe("random read on a large file, uses S3ACachingInputStream");
+ skipIfClientSideEncryption();
IOStatistics ioStats;
createLargeFile();
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ARequesterPays.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ARequesterPays.java
index d3925d35a99d..c58f13efbf26 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ARequesterPays.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ARequesterPays.java
@@ -59,7 +59,7 @@ public class ITestS3ARequesterPays extends AbstractS3ATestBase {
@Test
public void testRequesterPaysOptionSuccess() throws Throwable {
describe("Test requester pays enabled case by reading last then first byte");
-
+ skipIfClientSideEncryption();
Configuration conf = this.createConfiguration();
conf.setBoolean(ALLOW_REQUESTER_PAYS, true);
// Enable bucket exists check, the first failure point people may encounter
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java
index 9fb09b4cede5..658c81cd8f2d 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java
@@ -426,8 +426,7 @@ public class ITestAssumeRole extends AbstractS3ATestBase {
bindRolePolicy(conf,
policy(
statement(false, S3_ALL_BUCKETS, S3_GET_OBJECT_TORRENT),
- ALLOW_S3_GET_BUCKET_LOCATION,
- STATEMENT_ALLOW_SSE_KMS_RW));
+ ALLOW_S3_GET_BUCKET_LOCATION, STATEMENT_ALLOW_KMS_RW));
Path path = path("testAssumeRoleStillIncludesRolePerms");
roleFS = (S3AFileSystem) path.getFileSystem(conf);
assertTouchForbidden(roleFS, path);
@@ -447,8 +446,7 @@ public class ITestAssumeRole extends AbstractS3ATestBase {
bindRolePolicy(conf,
policy(
statement(false, S3_ALL_BUCKETS, S3_PATH_WRITE_OPERATIONS),
- STATEMENT_ALL_S3,
- STATEMENT_ALLOW_SSE_KMS_READ));
+ STATEMENT_ALL_S3, STATEMENT_ALLOW_KMS_RW));
Path path = methodPath();
roleFS = (S3AFileSystem) path.getFileSystem(conf);
// list the root path, expect happy
@@ -495,8 +493,7 @@ public class ITestAssumeRole extends AbstractS3ATestBase {
Configuration conf = createAssumedRoleConfig();
bindRolePolicyStatements(conf,
- STATEMENT_ALL_BUCKET_READ_ACCESS,
- STATEMENT_ALLOW_SSE_KMS_RW,
+ STATEMENT_ALL_BUCKET_READ_ACCESS, STATEMENT_ALLOW_KMS_RW,
new Statement(Effects.Allow)
.addActions(S3_ALL_OPERATIONS)
.addResources(directory(restrictedDir)));
@@ -563,8 +560,7 @@ public class ITestAssumeRole extends AbstractS3ATestBase {
fs.delete(basePath, true);
fs.mkdirs(readOnlyDir);
- bindRolePolicyStatements(conf,
- STATEMENT_ALLOW_SSE_KMS_RW,
+ bindRolePolicyStatements(conf, STATEMENT_ALLOW_KMS_RW,
STATEMENT_ALL_BUCKET_READ_ACCESS,
new Statement(Effects.Allow)
.addActions(S3_PATH_RW_OPERATIONS)
@@ -714,8 +710,7 @@ public class ITestAssumeRole extends AbstractS3ATestBase {
S3AFileSystem fs = getFileSystem();
fs.delete(destDir, true);
- bindRolePolicyStatements(conf,
- STATEMENT_ALLOW_SSE_KMS_RW,
+ bindRolePolicyStatements(conf, STATEMENT_ALLOW_KMS_RW,
statement(true, S3_ALL_BUCKETS, S3_ALL_OPERATIONS),
new Statement(Effects.Deny)
.addActions(S3_PATH_WRITE_OPERATIONS)
@@ -746,8 +741,7 @@ public class ITestAssumeRole extends AbstractS3ATestBase {
describe("Restrict role to read only");
Configuration conf = createAssumedRoleConfig();
- bindRolePolicyStatements(conf,
- STATEMENT_ALLOW_SSE_KMS_RW,
+ bindRolePolicyStatements(conf, STATEMENT_ALLOW_KMS_RW,
statement(true, S3_ALL_BUCKETS, S3_ALL_OPERATIONS),
statement(false, S3_ALL_BUCKETS, S3_GET_BUCKET_LOCATION));
Path path = methodPath();
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumedRoleCommitOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumedRoleCommitOperations.java
index dabc0abc2af2..2dc8497d6180 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumedRoleCommitOperations.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumedRoleCommitOperations.java
@@ -61,8 +61,7 @@ public class ITestAssumedRoleCommitOperations extends ITestCommitOperations {
restrictedDir = super.path("restricted");
Configuration conf = newAssumedRoleConfig(getConfiguration(),
getAssumedRoleARN());
- bindRolePolicyStatements(conf,
- STATEMENT_ALLOW_SSE_KMS_RW,
+ bindRolePolicyStatements(conf, STATEMENT_ALLOW_KMS_RW,
statement(true, S3_ALL_BUCKETS, S3_BUCKET_READ_OPERATIONS),
new RoleModel.Statement(RoleModel.Effects.Allow)
.addActions(S3_PATH_RW_OPERATIONS)
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestRestrictedReadAccess.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestRestrictedReadAccess.java
index a16e1b5e492f..7151c38ad3e2 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestRestrictedReadAccess.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestRestrictedReadAccess.java
@@ -260,8 +260,7 @@ public class ITestRestrictedReadAccess extends AbstractS3ATestBase {
// it still has write access, which can be explored in the final
// step to delete files and directories.
roleConfig = createAssumedRoleConfig();
- bindRolePolicyStatements(roleConfig,
- STATEMENT_ALLOW_SSE_KMS_RW,
+ bindRolePolicyStatements(roleConfig, STATEMENT_ALLOW_KMS_RW,
statement(true, S3_ALL_BUCKETS, S3_ALL_OPERATIONS),
new Statement(Effects.Deny)
.addActions(S3_ALL_GET)
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestPartialRenamesDeletes.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestPartialRenamesDeletes.java
index 378f4a70433d..24f5ddf6d89b 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestPartialRenamesDeletes.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestPartialRenamesDeletes.java
@@ -56,6 +56,7 @@ import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_DELETE_REQUEST;
import static org.apache.hadoop.fs.s3a.auth.RoleModel.Effects;
import static org.apache.hadoop.fs.s3a.auth.RoleModel.Statement;
import static org.apache.hadoop.fs.s3a.auth.RoleModel.directory;
+import static org.apache.hadoop.fs.s3a.auth.RoleModel.resource;
import static org.apache.hadoop.fs.s3a.auth.RoleModel.statement;
import static org.apache.hadoop.fs.s3a.auth.RolePolicies.*;
import static org.apache.hadoop.fs.s3a.auth.RoleTestUtils.bindRolePolicyStatements;
@@ -144,6 +145,11 @@ public class ITestPartialRenamesDeletes extends AbstractS3ATestBase {
*/
private Path writableDir;
+ /**
+ * Instruction file created when using CSE, required to be added to policies.
+ */
+ private Path writableDirInstructionFile;
+
/**
* A directory to which restricted roles have only read access.
*/
@@ -216,6 +222,7 @@ public class ITestPartialRenamesDeletes extends AbstractS3ATestBase {
basePath = uniquePath();
readOnlyDir = new Path(basePath, "readonlyDir");
writableDir = new Path(basePath, "writableDir");
+ writableDirInstructionFile = new Path(basePath, "writableDir.instruction");
readOnlyChild = new Path(readOnlyDir, "child");
noReadDir = new Path(basePath, "noReadDir");
// the full FS
@@ -225,8 +232,7 @@ public class ITestPartialRenamesDeletes extends AbstractS3ATestBase {
// create the baseline assumed role
assumedRoleConfig = createAssumedRoleConfig();
- bindRolePolicyStatements(assumedRoleConfig,
- STATEMENT_ALLOW_SSE_KMS_RW,
+ bindRolePolicyStatements(assumedRoleConfig, STATEMENT_ALLOW_KMS_RW,
STATEMENT_ALL_BUCKET_READ_ACCESS, // root: r-x
new Statement(Effects.Allow) // dest: rwx
.addActions(S3_PATH_RW_OPERATIONS)
@@ -365,13 +371,13 @@ public class ITestPartialRenamesDeletes extends AbstractS3ATestBase {
public void testRenameParentPathNotWriteable() throws Throwable {
describe("rename with parent paths not writeable; multi=%s", multiDelete);
final Configuration conf = createAssumedRoleConfig();
- bindRolePolicyStatements(conf,
- STATEMENT_ALLOW_SSE_KMS_RW,
+ bindRolePolicyStatements(conf, STATEMENT_ALLOW_KMS_RW,
STATEMENT_ALL_BUCKET_READ_ACCESS,
new Statement(Effects.Allow)
.addActions(S3_PATH_RW_OPERATIONS)
.addResources(directory(readOnlyDir))
- .addResources(directory(writableDir)));
+ .addResources(directory(writableDir))
+ .addResources(resource(writableDirInstructionFile, false, false)));
roleFS = (S3AFileSystem) readOnlyDir.getFileSystem(conf);
S3AFileSystem fs = getFileSystem();
@@ -733,8 +739,7 @@ public class ITestPartialRenamesDeletes extends AbstractS3ATestBase {
// s3:DeleteObjectVersion permission, and attempt rename
// and then delete.
Configuration roleConfig = createAssumedRoleConfig();
- bindRolePolicyStatements(roleConfig,
- STATEMENT_ALLOW_SSE_KMS_RW,
+ bindRolePolicyStatements(roleConfig, STATEMENT_ALLOW_KMS_RW,
STATEMENT_ALL_BUCKET_READ_ACCESS, // root: r-x
new Statement(Effects.Allow) // dest: rwx
.addActions(S3_PATH_RW_OPERATIONS)
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java
index 23b14fd3792c..f7b9ad4f24b4 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java
@@ -70,6 +70,7 @@ public class ITestS3GuardTool extends AbstractS3GuardToolTestBase {
@Test
public void testLandsatBucketRequireUnencrypted() throws Throwable {
+ skipIfClientSideEncryption();
run(BucketInfo.NAME,
"-" + BucketInfo.ENCRYPTION_FLAG, "none",
getLandsatCSVFile(getConfiguration()));
@@ -178,8 +179,9 @@ public class ITestS3GuardTool extends AbstractS3GuardToolTestBase {
// least a second old
describe("Sleeping 1 second then confirming upload still there");
Thread.sleep(1000);
- LambdaTestUtils.eventually(5000, 1000,
- () -> { assertNumUploadsAge(path, 1, 1); });
+ LambdaTestUtils.eventually(5000, 1000, () -> {
+ assertNumUploadsAge(path, 1, 1);
+ });
// 7. Assert deletion works when age filter matches
describe("Doing aged deletion");
@@ -231,8 +233,8 @@ public class ITestS3GuardTool extends AbstractS3GuardToolTestBase {
* search all parts
* @throws Exception on failure
*/
- private void uploadCommandAssertCount(S3AFileSystem fs, String options[],
- Path path, int numUploads, int ageSeconds)
+ private void uploadCommandAssertCount(S3AFileSystem fs, String[] options, Path path,
+ int numUploads, int ageSeconds)
throws Exception {
List<String> allOptions = new ArrayList<>();
List<String> output = new ArrayList<>();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 05/17: HADOOP-18932. S3A. upgrade AWS v2 SDK to 2.20.160 and v1 to 1.12.565 (#6178)
Posted by ah...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ahmar pushed a commit to branch feature-HADOOP-18073-sdk-v2-upgrade-3.6-aws
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 4be7d9e3034232f7a612e409e136ef4a3baf63da
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Tue Oct 17 12:59:50 2023 +0100
HADOOP-18932. S3A. upgrade AWS v2 SDK to 2.20.160 and v1 to 1.12.565 (#6178)
v1 => 1.12.565
v2 => 2.20.160
Only the v2 one is distributed; v1 is needed in deployments only to support v1 credential providers
Contributed by Steve Loughran
---
LICENSE-binary | 2 +-
hadoop-project/pom.xml | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/LICENSE-binary b/LICENSE-binary
index f2e7d9b28b26..2df6cf730b7a 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -364,7 +364,7 @@ org.objenesis:objenesis:2.6
org.xerial.snappy:snappy-java:1.1.10.4
org.yaml:snakeyaml:2.0
org.wildfly.openssl:wildfly-openssl:1.1.3.Final
-software.amazon.awssdk:bundle:jar:2.20.128
+software.amazon.awssdk:bundle:jar:2.20.160
--------------------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 77e12835e8bf..7270a8953e51 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -191,10 +191,10 @@
<exec-maven-plugin.version>1.3.1</exec-maven-plugin.version>
<make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
<surefire.fork.timeout>900</surefire.fork.timeout>
- <aws-java-sdk.version>1.12.499</aws-java-sdk.version>
- <hsqldb.version>2.7.1</hsqldb.version>
- <aws-java-sdk-v2.version>2.20.128</aws-java-sdk-v2.version>
+ <aws-java-sdk.version>1.12.565</aws-java-sdk.version>
+ <aws-java-sdk-v2.version>2.20.160</aws-java-sdk-v2.version>
<aws.eventstream.version>1.0.1</aws.eventstream.version>
+ <hsqldb.version>2.7.1</hsqldb.version>
<frontend-maven-plugin.version>1.11.2</frontend-maven-plugin.version>
<jasmine-maven-plugin.version>2.1</jasmine-maven-plugin.version>
<phantomjs-maven-plugin.version>0.7</phantomjs-maven-plugin.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 12/17: HADOOP-18930. Make fs.s3a.create.performance a bucket-wide setting. (#6168)
Posted by ah...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ahmar pushed a commit to branch feature-HADOOP-18073-sdk-v2-upgrade-3.6-aws
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit af32f6c63ed4546024acf78ace55ce1041277004
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Fri Oct 27 12:23:55 2023 +0100
HADOOP-18930. Make fs.s3a.create.performance a bucket-wide setting. (#6168)
If fs.s3a.create.performance is set on a bucket
- All file overwrite checks are skipped, even if the caller says
otherwise.
- All directory existence checks are skipped.
- Marker deletion is *always* skipped.
This eliminates a HEAD and a LIST for every creation.
* New path capability "fs.s3a.create.performance.enabled" true
if the option is enabled.
* Parameterize ITestS3AContractCreate to expect the different
outcomes
* Parameterize ITestCreateFileCost similarly, with
changed cost assertions there.
* create(/) raises an IOE. existing bug only noticed here.
Contributed by Steve Loughran
---
.../filesystem/fsdataoutputstreambuilder.md | 6 ++
.../java/org/apache/hadoop/fs/s3a/Constants.java | 18 +++-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 54 +++++++---
.../hadoop/fs/s3a/impl/CreateFileBuilder.java | 7 ++
.../fs/contract/s3a/ITestS3AContractCreate.java | 93 ++++++++++++++++
.../hadoop/fs/s3a/ITestS3AFSMainOperations.java | 20 ++++
.../hadoop/fs/s3a/ITestS3AFileOperationCost.java | 18 +++-
.../hadoop/fs/s3a/ITestS3AFileSystemContract.java | 20 ++++
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 13 +++
.../apache/hadoop/fs/s3a/impl/ITestXAttrCost.java | 7 +-
.../fs/s3a/performance/ITestCreateFileCost.java | 118 +++++++++++++++++----
.../performance/ITestDirectoryMarkerListing.java | 8 +-
.../fs/s3a/performance/ITestS3ADeleteCost.java | 11 ++
.../fs/s3a/tools/AbstractMarkerToolTest.java | 5 +-
14 files changed, 356 insertions(+), 42 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md
index 59a93c5887a1..7bacf0e2436a 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md
@@ -224,6 +224,12 @@ be used as evidence at the inquest as proof that they made a
conscious decision to choose speed over safety and
that the outcome was their own fault.
+Note: the option can be set for an entire filesystem. Again, the safety checks
+are there to more closely match the semantics of a classic filesystem,
+and to reduce the likelihood that the object store ends up in a state which
+diverges so much from the classic directory + tree structur that applications
+get confused.
+
Accordingly: *Use if and only if you are confident that the conditions are met.*
### `fs.s3a.create.header` User-supplied header support
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 944ab5dcdcb0..16012871ff48 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -1192,12 +1192,26 @@ public final class Constants {
/**
* Flag for create performance.
- * This is *not* a configuration option; it is for use in the
- * {code createFile()} builder.
+ * This can be set in the {code createFile()} builder.
* Value {@value}.
*/
public static final String FS_S3A_CREATE_PERFORMANCE = "fs.s3a.create.performance";
+ /**
+ * Default value for create performance in an S3A FS.
+ * Value {@value}.
+ */
+ public static final boolean FS_S3A_CREATE_PERFORMANCE_DEFAULT = true;
+
+
+ /**
+ * Capability to indicate that the FS has been instantiated with
+ * {@link #FS_S3A_CREATE_PERFORMANCE} set to true.
+ * Value {@value}.
+ */
+ public static final String FS_S3A_CREATE_PERFORMANCE_ENABLED =
+ FS_S3A_CREATE_PERFORMANCE + ".enabled";
+
/**
* Prefix for adding a header to the object when created.
* The actual value must have a "." suffix and then the actual header.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index db2cedd37fc9..c7511adf8ab1 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -235,6 +235,7 @@ import static org.apache.hadoop.fs.s3a.commit.CommitConstants.FS_S3A_COMMITTER_S
import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.submit;
import static org.apache.hadoop.fs.s3a.impl.CreateFileBuilder.OPTIONS_CREATE_FILE_NO_OVERWRITE;
import static org.apache.hadoop.fs.s3a.impl.CreateFileBuilder.OPTIONS_CREATE_FILE_OVERWRITE;
+import static org.apache.hadoop.fs.s3a.impl.CreateFileBuilder.OPTIONS_CREATE_FILE_PERFORMANCE;
import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isObjectNotFound;
import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isUnknownBucket;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.AP_REQUIRED_EXCEPTION;
@@ -348,7 +349,8 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
private S3AStatisticsContext statisticsContext;
/** Storage Statistics Bonded to the instrumentation. */
private S3AStorageStatistics storageStatistics;
-
+ /** Should all create files be "performance" unless unset. */
+ private boolean performanceCreation;
/**
* Default input policy; may be overridden in
* {@code openFile()}.
@@ -662,6 +664,11 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
// verify there's no S3Guard in the store config.
checkNoS3Guard(this.getUri(), getConf());
+ // performance creation flag for code which wants performance
+ // at the risk of overwrites.
+ performanceCreation = conf.getBoolean(FS_S3A_CREATE_PERFORMANCE,
+ FS_S3A_CREATE_PERFORMANCE_DEFAULT);
+ LOG.debug("{} = {}", FS_S3A_CREATE_PERFORMANCE, performanceCreation);
allowAuthoritativePaths = S3Guard.getAuthoritativePaths(this);
// directory policy, which may look at authoritative paths
@@ -1878,14 +1885,22 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
Progressable progress) throws IOException {
final Path path = qualify(f);
+ // work out the options to pass down
+ CreateFileBuilder.CreateFileOptions options;
+ if (performanceCreation) {
+ options = OPTIONS_CREATE_FILE_PERFORMANCE;
+ }else {
+ options = overwrite
+ ? OPTIONS_CREATE_FILE_OVERWRITE
+ : OPTIONS_CREATE_FILE_NO_OVERWRITE;
+ }
+
// the span will be picked up inside the output stream
return trackDurationAndSpan(INVOCATION_CREATE, path, () ->
innerCreateFile(path,
progress,
getActiveAuditSpan(),
- overwrite
- ? OPTIONS_CREATE_FILE_OVERWRITE
- : OPTIONS_CREATE_FILE_NO_OVERWRITE));
+ options));
}
/**
@@ -1912,14 +1927,19 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
final CreateFileBuilder.CreateFileOptions options) throws IOException {
auditSpan.activate();
String key = pathToKey(path);
+ if (key.isEmpty()) {
+ // no matter the creation options, root cannot be written to.
+ throw new PathIOException("/", "Can't create root path");
+ }
EnumSet<CreateFlag> flags = options.getFlags();
- boolean overwrite = flags.contains(CreateFlag.OVERWRITE);
- boolean performance = options.isPerformance();
- boolean skipProbes = performance || isUnderMagicCommitPath(path);
+
+ boolean skipProbes = options.isPerformance() || isUnderMagicCommitPath(path);
if (skipProbes) {
LOG.debug("Skipping existence/overwrite checks");
} else {
try {
+ boolean overwrite = flags.contains(CreateFlag.OVERWRITE);
+
// get the status or throw an FNFE.
// when overwriting, there is no need to look for any existing file,
// just a directory (for safety)
@@ -1951,7 +1971,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
// put options are derived from the path and the
// option builder.
- boolean keep = performance || keepDirectoryMarkers(path);
+ boolean keep = options.isPerformance() || keepDirectoryMarkers(path);
final PutObjectOptions putOptions =
new PutObjectOptions(keep, null, options.getHeaders());
@@ -2031,11 +2051,14 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
final AuditSpan span = entryPoint(INVOCATION_CREATE_FILE,
pathToKey(qualified),
null);
- return new CreateFileBuilder(this,
+ final CreateFileBuilder builder = new CreateFileBuilder(this,
qualified,
- new CreateFileBuilderCallbacksImpl(INVOCATION_CREATE_FILE, span))
- .create()
- .overwrite(true);
+ new CreateFileBuilderCallbacksImpl(INVOCATION_CREATE_FILE, span));
+ builder
+ .create()
+ .overwrite(true)
+ .must(FS_S3A_CREATE_PERFORMANCE, performanceCreation);
+ return builder;
} catch (IOException e) {
// catch any IOEs raised in span creation and convert to
// an UncheckedIOException
@@ -2098,7 +2121,8 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
.create()
.withFlags(flags)
.blockSize(blockSize)
- .bufferSize(bufferSize);
+ .bufferSize(bufferSize)
+ .must(FS_S3A_CREATE_PERFORMANCE, performanceCreation);
if (progress != null) {
builder.progress(progress);
}
@@ -5365,6 +5389,10 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
case FS_S3A_CREATE_HEADER:
return true;
+ // is the FS configured for create file performance
+ case FS_S3A_CREATE_PERFORMANCE_ENABLED:
+ return performanceCreation;
+
default:
return super.hasPathCapability(p, cap);
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CreateFileBuilder.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CreateFileBuilder.java
index 0392afac59d9..ae2945989ddd 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CreateFileBuilder.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CreateFileBuilder.java
@@ -71,6 +71,12 @@ public class CreateFileBuilder extends
public static final CreateFileOptions OPTIONS_CREATE_FILE_NO_OVERWRITE =
new CreateFileOptions(CREATE_NO_OVERWRITE_FLAGS, true, false, null);
+ /**
+ * Performance create options.
+ */
+ public static final CreateFileOptions OPTIONS_CREATE_FILE_PERFORMANCE =
+ new CreateFileOptions(CREATE_OVERWRITE_FLAGS, true, true, null);
+
/**
* Callback interface.
*/
@@ -129,6 +135,7 @@ public class CreateFileBuilder extends
if (flags.contains(CreateFlag.APPEND)) {
throw new UnsupportedOperationException("Append is not supported");
}
+
if (!flags.contains(CreateFlag.CREATE) &&
!flags.contains(CreateFlag.OVERWRITE)) {
throw new PathIOException(path.toString(),
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java
index d2a858f615ef..7a2a10879dd8 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java
@@ -18,18 +18,111 @@
package org.apache.hadoop.fs.contract.s3a;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+
+import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
/**
* S3A contract tests creating files.
+ * Parameterized on the create performance flag as all overwrite
+ * tests are required to fail in create performance mode.
*/
+@RunWith(Parameterized.class)
public class ITestS3AContractCreate extends AbstractContractCreateTest {
+ /**
+ * This test suite is parameterized for the different create file
+ * options.
+ * @return a list of test parameters.
+ */
+ @Parameterized.Parameters
+ public static Collection<Object[]> params() {
+ return Arrays.asList(new Object[][]{
+ {false},
+ {true}
+ });
+ }
+
+ /**
+ * Is this test run in create performance mode?
+ */
+ private final boolean createPerformance;
+
+ public ITestS3AContractCreate(final boolean createPerformance) {
+ this.createPerformance = createPerformance;
+ }
+
@Override
protected AbstractFSContract createContract(Configuration conf) {
return new S3AContract(conf);
}
+ @Override
+ protected Configuration createConfiguration() {
+ final Configuration conf = super.createConfiguration();
+ removeBaseAndBucketOverrides(conf,
+ FS_S3A_CREATE_PERFORMANCE);
+ conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, createPerformance);
+ S3ATestUtils.disableFilesystemCaching(conf);
+ return conf;
+ }
+
+ @Override
+ public void testOverwriteNonEmptyDirectory() throws Throwable {
+ try {
+ super.testOverwriteNonEmptyDirectory();
+ failWithCreatePerformance();
+ } catch (AssertionError e) {
+ swallowWithCreatePerformance(e);
+ }
+ }
+
+ @Override
+ public void testOverwriteEmptyDirectory() throws Throwable {
+ try {
+ super.testOverwriteEmptyDirectory();
+ failWithCreatePerformance();
+ } catch (AssertionError e) {
+ swallowWithCreatePerformance(e);
+ }
+ }
+
+ @Override
+ public void testCreateFileOverExistingFileNoOverwrite() throws Throwable {
+ try {
+ super.testCreateFileOverExistingFileNoOverwrite();
+ failWithCreatePerformance();
+ } catch (AssertionError e) {
+ swallowWithCreatePerformance(e);
+ }
+ }
+
+ private void failWithCreatePerformance() {
+ if (createPerformance) {
+ fail("expected an assertion error in create performance mode");
+ }
+ }
+
+ /**
+ * Swallow an assertion error if the create performance flag is set.
+ * @param e assertion error
+ */
+ private void swallowWithCreatePerformance(final AssertionError e) {
+ // this is expected in create performance modea
+ if (!createPerformance) {
+ // but if the create performance flag is set, then it is supported
+ // and the assertion error is unexpected
+ throw e;
+ }
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFSMainOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFSMainOperations.java
index 6669e8426af0..013ec901d0a7 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFSMainOperations.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFSMainOperations.java
@@ -18,6 +18,9 @@
package org.apache.hadoop.fs.s3a;
+import java.io.IOException;
+
+import org.assertj.core.api.Assertions;
import org.junit.Ignore;
import org.apache.hadoop.conf.Configuration;
@@ -27,6 +30,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.s3a.S3AContract;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.createTestPath;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.isCreatePerformanceEnabled;
/**
* S3A Test suite for the FSMainOperationsBaseTest tests.
@@ -70,4 +74,20 @@ public class ITestS3AFSMainOperations extends FSMainOperationsBaseTest {
throws Exception {
}
+ @Override
+ public void testOverwrite() throws IOException {
+ boolean createPerformance = isCreatePerformanceEnabled(fSys);
+ try {
+ super.testOverwrite();
+ Assertions.assertThat(createPerformance)
+ .describedAs("create performance enabled")
+ .isFalse();
+ } catch (AssertionError e) {
+ // swallow the exception if create performance is enabled,
+ // else rethrow
+ if (!createPerformance) {
+ throw e;
+ }
+ }
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
index dae6312d4809..0e4a8eda5b29 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.fs.s3a;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum;
@@ -39,6 +40,8 @@ import java.util.EnumSet;
import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.Statistic.*;
import static org.apache.hadoop.fs.s3a.performance.OperationCost.*;
import static org.apache.hadoop.test.GenericTestUtils.getTestDir;
@@ -47,6 +50,9 @@ import static org.apache.hadoop.test.LambdaTestUtils.intercept;
/**
* Use metrics to assert about the cost of file API calls.
* Parameterized on directory marker keep vs delete.
+ * When the FS is instantiated with creation performance, things
+ * behave differently...its value is that of the marker keep flag,
+ * so deletion costs are the same.
*/
@RunWith(Parameterized.class)
public class ITestS3AFileOperationCost extends AbstractS3ACostTest {
@@ -71,6 +77,14 @@ public class ITestS3AFileOperationCost extends AbstractS3ACostTest {
super(keepMarkers);
}
+ @Override
+ public Configuration createConfiguration() {
+ final Configuration conf = super.createConfiguration();
+ removeBaseAndBucketOverrides(conf, FS_S3A_CREATE_PERFORMANCE);
+ conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, isKeepingMarkers());
+ return conf;
+ }
+
/**
* Test the cost of {@code listLocatedStatus(file)}.
*/
@@ -377,7 +391,7 @@ public class ITestS3AFileOperationCost extends AbstractS3ACostTest {
// create a bunch of files
int filesToCreate = 10;
for (int i = 0; i < filesToCreate; i++) {
- create(basePath.suffix("/" + i));
+ file(basePath.suffix("/" + i));
}
fs.globStatus(basePath.suffix("/*"));
@@ -396,7 +410,7 @@ public class ITestS3AFileOperationCost extends AbstractS3ACostTest {
// create a single file, globStatus returning a single file on a pattern
// triggers attempts at symlinks resolution if configured
String fileName = "/notASymlinkDOntResolveMeLikeOne";
- create(basePath.suffix(fileName));
+ file(basePath.suffix(fileName));
// unguarded: 2 head + 1 list from getFileStatus on path,
// plus 1 list to match the glob pattern
// no additional operations from symlink resolution
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
index 7ce7b8385cec..56827043c9b8 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
@@ -19,7 +19,9 @@
package org.apache.hadoop.fs.s3a;
import java.io.FileNotFoundException;
+import java.io.IOException;
+import org.assertj.core.api.Assertions;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -32,6 +34,7 @@ import org.apache.hadoop.fs.FileSystemContractBaseTest;
import org.apache.hadoop.fs.Path;
import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.isCreatePerformanceEnabled;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.junit.Assume.*;
import static org.junit.Assert.*;
@@ -137,4 +140,21 @@ public class ITestS3AFileSystemContract extends FileSystemContractBaseTest {
() -> super.testRenameNonExistentPath());
}
+
+ @Override
+ public void testOverwrite() throws IOException {
+ boolean createPerformance = isCreatePerformanceEnabled(fs);
+ try {
+ super.testOverwrite();
+ Assertions.assertThat(createPerformance)
+ .describedAs("create performance enabled")
+ .isFalse();
+ } catch (AssertionError e) {
+ // swallow the exception if create performance is enabled,
+ // else rethrow
+ if (!createPerformance) {
+ throw e;
+ }
+ }
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
index 3382c300b931..aa38186c6503 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
@@ -1554,4 +1554,17 @@ public final class S3ATestUtils {
return fs.getConf().getBoolean(Constants.ENABLE_MULTI_DELETE,
true);
}
+
+ /**
+ * Does this FS have create performance enabled?
+ * @param fs filesystem
+ * @return true if create performance is enabled
+ * @throws IOException IO problems
+ */
+ public static boolean isCreatePerformanceEnabled(FileSystem fs)
+ throws IOException {
+ return fs.hasPathCapability(new Path("/"), FS_S3A_CREATE_PERFORMANCE_ENABLED);
+ }
+
+
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestXAttrCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestXAttrCost.java
index f6fe68b8e6d3..71fdb7aeaa62 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestXAttrCost.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestXAttrCost.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.performance.AbstractS3ACostTest;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.isCreatePerformanceEnabled;
import static org.apache.hadoop.fs.s3a.Statistic.INVOCATION_OP_XATTR_LIST;
import static org.apache.hadoop.fs.s3a.Statistic.INVOCATION_XATTR_GET_MAP;
import static org.apache.hadoop.fs.s3a.Statistic.INVOCATION_XATTR_GET_NAMED;
@@ -43,6 +44,7 @@ import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.XA_CONTENT_TYPE;
import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.XA_STANDARD_HEADERS;
import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.decodeBytes;
import static org.apache.hadoop.fs.s3a.performance.OperationCost.CREATE_FILE_OVERWRITE;
+import static org.apache.hadoop.fs.s3a.performance.OperationCost.NO_HEAD_OR_LIST;
/**
* Invoke XAttr API calls against objects in S3 and validate header
@@ -95,8 +97,11 @@ public class ITestXAttrCost extends AbstractS3ACostTest {
public void testXAttrFile() throws Throwable {
describe("Test xattr on a file");
Path testFile = methodPath();
- create(testFile, true, CREATE_FILE_OVERWRITE);
S3AFileSystem fs = getFileSystem();
+ boolean createPerformance = isCreatePerformanceEnabled(fs);
+
+ create(testFile, true,
+ createPerformance ? NO_HEAD_OR_LIST : CREATE_FILE_OVERWRITE);
Map<String, byte[]> xAttrs = verifyMetrics(() ->
fs.getXAttrs(testFile),
with(INVOCATION_XATTR_GET_MAP, GET_METADATA_ON_OBJECT));
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java
index 39530d97bf79..2d128cffc5af 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java
@@ -19,16 +19,22 @@
package org.apache.hadoop.fs.s3a.performance;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
import org.assertj.core.api.Assertions;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FSDataOutputStreamBuilder;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
import static java.util.Objects.requireNonNull;
@@ -36,6 +42,7 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.toChar;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_HEADER;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
import static org.apache.hadoop.fs.s3a.Constants.XA_HEADER_PREFIX;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_BULK_DELETE_REQUEST;
import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_DELETE_REQUEST;
import static org.apache.hadoop.fs.s3a.performance.OperationCost.CREATE_FILE_NO_OVERWRITE;
@@ -45,6 +52,7 @@ import static org.apache.hadoop.fs.s3a.performance.OperationCost.FILE_STATUS_FIL
import static org.apache.hadoop.fs.s3a.performance.OperationCost.GET_FILE_STATUS_ON_DIR_MARKER;
import static org.apache.hadoop.fs.s3a.performance.OperationCost.GET_FILE_STATUS_ON_FILE;
import static org.apache.hadoop.fs.s3a.performance.OperationCost.HEAD_OPERATION;
+import static org.apache.hadoop.fs.s3a.performance.OperationCost.LIST_OPERATION;
import static org.apache.hadoop.fs.s3a.performance.OperationCost.NO_HEAD_OR_LIST;
/**
@@ -52,13 +60,55 @@ import static org.apache.hadoop.fs.s3a.performance.OperationCost.NO_HEAD_OR_LIST
* with the FS_S3A_CREATE_PERFORMANCE option.
*/
@SuppressWarnings("resource")
+@RunWith(Parameterized.class)
public class ITestCreateFileCost extends AbstractS3ACostTest {
+ /**
+ * This test suite is parameterized for the different create file
+ * options.
+ * @return a list of test parameters.
+ */
+ @Parameterized.Parameters
+ public static Collection<Object[]> params() {
+ return Arrays.asList(new Object[][]{
+ {false},
+ {true}
+ });
+ }
+
+ /**
+ * Flag for performance creation; all cost asserts need changing.
+ */
+ private final boolean createPerformance;
+
/**
* Create with markers kept, always.
*/
- public ITestCreateFileCost() {
+ public ITestCreateFileCost(final boolean createPerformance) {
+ // keep markers to permit assertions that create performance
+ // always skips marker deletion.
super(false);
+ this.createPerformance = createPerformance;
+ }
+
+ /**
+ * Determine the expected cost of a create operation;
+ * if {@link #createPerformance} is true, then the cost is always "no IO".
+ * @param source source cost
+ * @return cost to assert
+ */
+ private OperationCost expected(OperationCost source) {
+ return createPerformance ? NO_HEAD_OR_LIST : source;
+ }
+
+ @Override
+ public Configuration createConfiguration() {
+ final Configuration conf = super.createConfiguration();
+ removeBaseAndBucketOverrides(conf,
+ FS_S3A_CREATE_PERFORMANCE);
+ conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, createPerformance);
+ S3ATestUtils.disableFilesystemCaching(conf);
+ return conf;
}
@Test
@@ -67,7 +117,7 @@ public class ITestCreateFileCost extends AbstractS3ACostTest {
Path testFile = methodPath();
// when overwrite is false, the path is checked for existence.
create(testFile, false,
- CREATE_FILE_NO_OVERWRITE);
+ expected(CREATE_FILE_NO_OVERWRITE));
}
@Test
@@ -75,7 +125,7 @@ public class ITestCreateFileCost extends AbstractS3ACostTest {
describe("Test file creation with overwrite");
Path testFile = methodPath();
// when overwrite is true: only the directory checks take place.
- create(testFile, true, CREATE_FILE_OVERWRITE);
+ create(testFile, true, expected(CREATE_FILE_OVERWRITE));
}
@Test
@@ -85,21 +135,45 @@ public class ITestCreateFileCost extends AbstractS3ACostTest {
// now there is a file there, an attempt with overwrite == false will
// fail on the first HEAD.
- interceptOperation(FileAlreadyExistsException.class, "",
- FILE_STATUS_FILE_PROBE,
- () -> file(testFile, false));
+ if (!createPerformance) {
+ interceptOperation(FileAlreadyExistsException.class, "",
+ FILE_STATUS_FILE_PROBE,
+ () -> file(testFile, false));
+ } else {
+ create(testFile, false, NO_HEAD_OR_LIST);
+ }
}
@Test
- public void testCreateFileOverDir() throws Throwable {
- describe("Test cost of create file failing with existing dir");
+ public void testCreateFileOverDirNoOverwrite() throws Throwable {
+ describe("Test cost of create file overwrite=false failing with existing dir");
Path testFile = dir(methodPath());
- // now there is a file there, an attempt with overwrite == false will
+ // now there is a dir marker there, an attempt with overwrite == true will
// fail on the first HEAD.
- interceptOperation(FileAlreadyExistsException.class, "",
- GET_FILE_STATUS_ON_DIR_MARKER,
- () -> file(testFile, false));
+ if (!createPerformance) {
+ interceptOperation(FileAlreadyExistsException.class, "",
+ GET_FILE_STATUS_ON_DIR_MARKER,
+ () -> file(testFile, false));
+ } else {
+ create(testFile, false, NO_HEAD_OR_LIST);
+ }
+ }
+
+ @Test
+ public void testCreateFileOverDirWithOverwrite() throws Throwable {
+ describe("Test cost of create file overwrite=false failing with existing dir");
+ Path testFile = dir(methodPath());
+
+ // now there is a dir marker there, an attempt with overwrite == true will
+ // fail on the LIST; no HEAD is issued.
+ if (!createPerformance) {
+ interceptOperation(FileAlreadyExistsException.class, "",
+ LIST_OPERATION,
+ () -> file(testFile, true));
+ } else {
+ create(testFile, true, NO_HEAD_OR_LIST);
+ }
}
/**
@@ -117,14 +191,18 @@ public class ITestCreateFileCost extends AbstractS3ACostTest {
// files and so briefly the path not being present
// only make sure the dest path isn't a directory.
buildFile(testFile, true, false,
- FILE_STATUS_DIR_PROBE);
+ expected(FILE_STATUS_DIR_PROBE));
// now there is a file there, an attempt with overwrite == false will
// fail on the first HEAD.
- interceptOperation(FileAlreadyExistsException.class, "",
- GET_FILE_STATUS_ON_FILE,
- () -> buildFile(testFile, false, true,
- GET_FILE_STATUS_ON_FILE));
+ if (!createPerformance) {
+ interceptOperation(FileAlreadyExistsException.class, "",
+ GET_FILE_STATUS_ON_FILE,
+ () -> buildFile(testFile, false, true,
+ GET_FILE_STATUS_ON_FILE));
+ } else {
+ buildFile(testFile, false, true, NO_HEAD_OR_LIST);
+ }
}
@Test
@@ -162,7 +240,7 @@ public class ITestCreateFileCost extends AbstractS3ACostTest {
builder.must(FS_S3A_CREATE_HEADER + ".h1", custom);
verifyMetrics(() -> build(builder),
- always(CREATE_FILE_NO_OVERWRITE));
+ always(expected(CREATE_FILE_NO_OVERWRITE)));
// the header is there and the probe should be a single HEAD call.
String header = verifyMetrics(() ->
@@ -181,7 +259,7 @@ public class ITestCreateFileCost extends AbstractS3ACostTest {
verifyMetrics(() ->
build(fs.createFile(methodPath()).overwrite(true)),
- always(CREATE_FILE_OVERWRITE));
+ always(expected(CREATE_FILE_OVERWRITE)));
}
@@ -196,7 +274,7 @@ public class ITestCreateFileCost extends AbstractS3ACostTest {
.close();
return "";
},
- always(CREATE_FILE_OVERWRITE));
+ always(expected(CREATE_FILE_OVERWRITE)));
}
private FSDataOutputStream build(final FSDataOutputStreamBuilder builder)
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java
index cd4ee4440667..8dbf5baaa620 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java
@@ -54,6 +54,7 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY;
import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_DELETE;
import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_KEEP;
+import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@@ -80,8 +81,7 @@ import static org.apache.hadoop.util.functional.RemoteIterators.foreach;
* <p></p>
* Similarly: JUnit assertions over AssertJ.
* <p></p>
- * The tests work with unguarded buckets only -the bucket settings are changed
- * appropriately.
+ * s3a create performance is disabled for consistent assertions.
*/
@RunWith(Parameterized.class)
public class ITestDirectoryMarkerListing extends AbstractS3ATestBase {
@@ -199,11 +199,13 @@ public class ITestDirectoryMarkerListing extends AbstractS3ATestBase {
// directory marker options
removeBaseAndBucketOverrides(bucketName, conf,
- DIRECTORY_MARKER_POLICY);
+ DIRECTORY_MARKER_POLICY,
+ FS_S3A_CREATE_PERFORMANCE);
conf.set(DIRECTORY_MARKER_POLICY,
keepMarkers
? DIRECTORY_MARKER_POLICY_KEEP
: DIRECTORY_MARKER_POLICY_DELETE);
+ conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, false);
return conf;
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java
index fae2df973af7..97f51fe2c8dc 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java
@@ -30,6 +30,7 @@ import org.junit.runners.Parameterized;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.ContractTestUtils;
@@ -38,6 +39,8 @@ import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.Tristate;
import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum;
+import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.Statistic.*;
import static org.apache.hadoop.fs.s3a.performance.OperationCost.*;
import static org.apache.hadoop.fs.s3a.performance.OperationCostValidator.probe;
@@ -74,6 +77,14 @@ public class ITestS3ADeleteCost extends AbstractS3ACostTest {
super(keepMarkers);
}
+ @Override
+ public Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ removeBaseAndBucketOverrides(conf, FS_S3A_CREATE_PERFORMANCE);
+ conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, false);
+ return conf;
+ }
+
@Override
public void teardown() throws Exception {
if (isKeepingMarkers()) {
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/AbstractMarkerToolTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/AbstractMarkerToolTest.java
index 0c2473a5f61d..759a3bf129ee 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/AbstractMarkerToolTest.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/AbstractMarkerToolTest.java
@@ -73,12 +73,15 @@ public class AbstractMarkerToolTest extends AbstractS3ATestBase {
removeBaseAndBucketOverrides(bucketName, conf,
S3A_BUCKET_PROBE,
DIRECTORY_MARKER_POLICY,
- AUTHORITATIVE_PATH);
+ AUTHORITATIVE_PATH,
+ FS_S3A_CREATE_PERFORMANCE);
// base FS is legacy
conf.set(DIRECTORY_MARKER_POLICY, DIRECTORY_MARKER_POLICY_DELETE);
+ conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, false);
// turn off bucket probes for a bit of speedup in the connectors we create.
conf.setInt(S3A_BUCKET_PROBE, 0);
+
return conf;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 14/17: HADOOP-18925. S3A: option to enable/disable CopyFromLocalOperation (#6163)
Posted by ah...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ahmar pushed a commit to branch feature-HADOOP-18073-sdk-v2-upgrade-3.6-aws
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 141b0d35081c1219076d7d08c6c322d087a28d19
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Mon Nov 6 16:00:56 2023 +0000
HADOOP-18925. S3A: option to enable/disable CopyFromLocalOperation (#6163)
Add a new option:
fs.s3a.optimized.copy.from.local.enabled
This will enable (default) or disable the
optimized CopyFromLocalOperation upload operation
when copyFromLocalFile() is invoked.
When false the superclass implementation is used; duration
statistics are still collected, though audit span entries
in logs will be for the individual fs operations, not the
overall operation.
Contributed by Steve Loughran
---
.../java/org/apache/hadoop/fs/s3a/Constants.java | 13 +++
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 93 +++++++++++++++-------
.../tools/hadoop-aws/troubleshooting_s3a.md | 10 +++
.../hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java | 51 ++++++++++++
4 files changed, 138 insertions(+), 29 deletions(-)
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index e602c1b3dc36..140ea53ceb6f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -1496,4 +1496,17 @@ public final class Constants {
*/
public static final boolean DIRECTORY_OPERATIONS_PURGE_UPLOADS_DEFAULT = false;
+
+ /**
+ * Is the higher performance copy from local file to S3 enabled?
+ * This switch allows for it to be disabled if there are problems.
+ * Value: {@value}.
+ */
+ public static final String OPTIMIZED_COPY_FROM_LOCAL = "fs.s3a.optimized.copy.from.local.enabled";
+
+ /**
+ * Default value for {@link #OPTIMIZED_COPY_FROM_LOCAL}.
+ * Value: {@value}.
+ */
+ public static final boolean OPTIMIZED_COPY_FROM_LOCAL_DEFAULT = true;
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index cdcb0c9cb148..3e995fb1ddb4 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -464,6 +464,12 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
*/
private String scheme = FS_S3A;
+ /**
+ * Flag to indicate that the higher performance copyFromLocalFile implementation
+ * should be used.
+ */
+ private boolean optimizedCopyFromLocal;
+
/** Add any deprecated keys. */
@SuppressWarnings("deprecation")
private static void addDeprecatedKeys() {
@@ -700,6 +706,9 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
AWS_S3_VECTOR_ACTIVE_RANGE_READS, DEFAULT_AWS_S3_VECTOR_ACTIVE_RANGE_READS, 1);
vectoredIOContext = populateVectoredIOContext(conf);
scheme = (this.uri != null && this.uri.getScheme() != null) ? this.uri.getScheme() : FS_S3A;
+ optimizedCopyFromLocal = conf.getBoolean(OPTIMIZED_COPY_FROM_LOCAL,
+ OPTIMIZED_COPY_FROM_LOCAL_DEFAULT);
+ LOG.debug("Using optimized copyFromLocal implementation: {}", optimizedCopyFromLocal);
} catch (SdkException e) {
// amazon client exception: stop all services then throw the translation
cleanupWithLogger(LOG, span);
@@ -4028,9 +4037,9 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* the given dst name.
*
* This version doesn't need to create a temporary file to calculate the md5.
- * Sadly this doesn't seem to be used by the shell cp :(
+ * If {@link Constants#OPTIMIZED_COPY_FROM_LOCAL} is set to false,
+ * the superclass implementation is used.
*
- * delSrc indicates if the source should be removed
* @param delSrc whether to delete the src
* @param overwrite whether to overwrite an existing file
* @param src path
@@ -4038,35 +4047,59 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @throws IOException IO problem
* @throws FileAlreadyExistsException the destination file exists and
* overwrite==false
- * @throws SdkException failure in the AWS SDK
*/
@Override
@AuditEntryPoint
public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src,
Path dst) throws IOException {
checkNotClosed();
- LOG.debug("Copying local file from {} to {}", src, dst);
- trackDurationAndSpan(INVOCATION_COPY_FROM_LOCAL_FILE, dst,
- () -> new CopyFromLocalOperation(
- createStoreContext(),
- src,
- dst,
- delSrc,
- overwrite,
- createCopyFromLocalCallbacks()).execute());
+ LOG.debug("Copying local file from {} to {} (delSrc={} overwrite={}",
+ src, dst, delSrc, overwrite);
+ if (optimizedCopyFromLocal) {
+ trackDurationAndSpan(INVOCATION_COPY_FROM_LOCAL_FILE, dst, () ->
+ new CopyFromLocalOperation(
+ createStoreContext(),
+ src,
+ dst,
+ delSrc,
+ overwrite,
+ createCopyFromLocalCallbacks(getActiveAuditSpan()))
+ .execute());
+ } else {
+ // call the superclass, but still count statistics.
+ // there is no overall span here, as each FS API call will
+ // be in its own span.
+ LOG.debug("Using base copyFromLocalFile implementation");
+ trackDurationAndSpan(INVOCATION_COPY_FROM_LOCAL_FILE, dst, () -> {
+ super.copyFromLocalFile(delSrc, overwrite, src, dst);
+ return null;
+ });
+ }
}
+ /**
+ * Create the CopyFromLocalCallbacks;
+ * protected to assist in mocking.
+ * @param span audit span.
+ * @return the callbacks
+ * @throws IOException failure to get the local fs.
+ */
protected CopyFromLocalOperation.CopyFromLocalOperationCallbacks
- createCopyFromLocalCallbacks() throws IOException {
+ createCopyFromLocalCallbacks(final AuditSpanS3A span) throws IOException {
LocalFileSystem local = getLocal(getConf());
- return new CopyFromLocalCallbacksImpl(local);
+ return new CopyFromLocalCallbacksImpl(span, local);
}
protected final class CopyFromLocalCallbacksImpl implements
CopyFromLocalOperation.CopyFromLocalOperationCallbacks {
+
+ /** Span to use for all operations. */
+ private final AuditSpanS3A span;
private final LocalFileSystem local;
- private CopyFromLocalCallbacksImpl(LocalFileSystem local) {
+ private CopyFromLocalCallbacksImpl(final AuditSpanS3A span,
+ LocalFileSystem local) {
+ this.span = span;
this.local = local;
}
@@ -4088,20 +4121,18 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
@Override
public void copyLocalFileFromTo(File file, Path from, Path to) throws IOException {
- trackDurationAndSpan(
- OBJECT_PUT_REQUESTS,
- to,
- () -> {
- final String key = pathToKey(to);
- Progressable progress = null;
- PutObjectRequest.Builder putObjectRequestBuilder =
- newPutObjectRequestBuilder(key, file.length(), false);
- S3AFileSystem.this.invoker.retry("putObject(" + "" + ")", to.toString(), true,
- () -> executePut(putObjectRequestBuilder.build(), progress, putOptionsForPath(to),
- file));
-
- return null;
- });
+ // the duration of the put is measured, but the active span is the
+ // constructor-supplied one -this ensures all audit log events are grouped correctly
+ span.activate();
+ trackDuration(getDurationTrackerFactory(), OBJECT_PUT_REQUESTS.getSymbol(), () -> {
+ final String key = pathToKey(to);
+ PutObjectRequest.Builder putObjectRequestBuilder =
+ newPutObjectRequestBuilder(key, file.length(), false);
+ final String dest = to.toString();
+ S3AFileSystem.this.invoker.retry("putObject(" + dest + ")", dest, true, () ->
+ executePut(putObjectRequestBuilder.build(), null, putOptionsForPath(to), file));
+ return null;
+ });
}
@Override
@@ -5403,6 +5434,10 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
case FS_S3A_CREATE_PERFORMANCE_ENABLED:
return performanceCreation;
+ // is the optimized copy from local enabled.
+ case OPTIMIZED_COPY_FROM_LOCAL:
+ return optimizedCopyFromLocal;
+
default:
return super.hasPathCapability(p, cap);
}
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
index 8c57db1faf4c..be359336463a 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
@@ -1544,3 +1544,13 @@ software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP
When this happens, try to set `fs.s3a.connection.request.timeout` to a larger value or disable it
completely by setting it to `0`.
+
+### <a name="debug-switches"></a> Debugging Switches
+
+There are some switches which can be set to enable/disable features and assist
+in isolating problems and at least make them "go away".
+
+
+| Key | Default | Action |
+|------|---------|----------|
+| `fs.s3a.optimized.copy.from.local.enabled` | `true` | [HADOOP-18925](https://issues.apache.org/jira/browse/HADOOP-18925) enable/disable CopyFromLocalOperation. Also a path capability. |
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
index dfac771dd78d..f9600de6d20c 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.fs.s3a;
import java.io.File;
+import java.util.Arrays;
+import java.util.Collection;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractContractCopyFromLocalTest;
@@ -26,18 +28,67 @@ import org.apache.hadoop.fs.contract.AbstractFSContract;
import org.apache.hadoop.fs.contract.s3a.S3AContract;
import org.apache.hadoop.fs.Path;
+
+import org.assertj.core.api.Assertions;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import static org.apache.hadoop.fs.s3a.Constants.OPTIMIZED_COPY_FROM_LOCAL;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+/**
+ * Test copying files from the local filesystem to S3A.
+ * Parameterized on whether or not the optimized
+ * copyFromLocalFile is enabled.
+ */
+@RunWith(Parameterized.class)
public class ITestS3ACopyFromLocalFile extends
AbstractContractCopyFromLocalTest {
+ /**
+ * Parameterization.
+ */
+ @Parameterized.Parameters(name = "enabled={0}")
+ public static Collection<Object[]> params() {
+ return Arrays.asList(new Object[][]{
+ {true},
+ {false},
+ });
+ }
+ private final boolean enabled;
+
+ public ITestS3ACopyFromLocalFile(final boolean enabled) {
+ this.enabled = enabled;
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ final Configuration conf = super.createConfiguration();
+
+ removeBaseAndBucketOverrides(getTestBucketName(conf), conf,
+ OPTIMIZED_COPY_FROM_LOCAL);
+ conf.setBoolean(OPTIMIZED_COPY_FROM_LOCAL, enabled);
+ disableFilesystemCaching(conf);
+ return conf;
+ }
@Override
protected AbstractFSContract createContract(Configuration conf) {
return new S3AContract(conf);
}
+ @Test
+ public void testOptionPropagation() throws Throwable {
+ Assertions.assertThat(getFileSystem().hasPathCapability(new Path("/"),
+ OPTIMIZED_COPY_FROM_LOCAL))
+ .describedAs("path capability of %s", OPTIMIZED_COPY_FROM_LOCAL)
+ .isEqualTo(enabled);
+
+ }
+
@Test
public void testLocalFilesOnly() throws Throwable {
describe("Copying into other file systems must fail");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 03/17: HADOOP-18888. S3A. createS3AsyncClient() always enables multipart uploads (#6056)
Posted by ah...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ahmar pushed a commit to branch feature-HADOOP-18073-sdk-v2-upgrade-3.6-aws
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit d5472f9d6db27e4c44cd336c614aa6211e945fa2
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Fri Sep 15 15:45:09 2023 +0100
HADOOP-18888. S3A. createS3AsyncClient() always enables multipart uploads (#6056)
* The multipart flag fs.s3a.multipart.uploads.enabled is passed to the async client created
* s3A connector bypasses the transfer manager entirely if disabled or for small files.
Contributed by Steve Loughran
---
.../hadoop/fs/s3a/DefaultS3ClientFactory.java | 2 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 93 +++++++++++++++-------
.../org/apache/hadoop/fs/s3a/S3AInternals.java | 6 ++
.../org/apache/hadoop/fs/s3a/S3ClientFactory.java | 24 ++++++
.../fs/s3a/scale/AbstractSTestS3AHugeFiles.java | 4 +-
.../fs/s3a/scale/ITestS3AHugeFilesNoMultipart.java | 53 +++++-------
6 files changed, 118 insertions(+), 64 deletions(-)
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
index 98c72d276628..c85263f1903a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
@@ -112,7 +112,7 @@ public class DefaultS3ClientFactory extends Configured
return configureClientBuilder(S3AsyncClient.builder(), parameters, conf, bucket)
.httpClientBuilder(httpClientBuilder)
.multipartConfiguration(multipartConfiguration)
- .multipartEnabled(true)
+ .multipartEnabled(parameters.isMultipartCopy())
.build();
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index d0a32148c9d5..da828123ec77 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -440,6 +440,12 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
*/
private boolean isMultipartUploadEnabled = DEFAULT_MULTIPART_UPLOAD_ENABLED;
+ /**
+ * Should file copy operations use the S3 transfer manager?
+ * True unless multipart upload is disabled.
+ */
+ private boolean isMultipartCopyEnabled;
+
/**
* A cache of files that should be deleted when the FileSystem is closed
* or the JVM is exited.
@@ -576,6 +582,10 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
this.prefetchBlockSize = (int) prefetchBlockSizeLong;
this.prefetchBlockCount =
intOption(conf, PREFETCH_BLOCK_COUNT_KEY, PREFETCH_BLOCK_DEFAULT_COUNT, 1);
+ this.isMultipartUploadEnabled = conf.getBoolean(MULTIPART_UPLOADS_ENABLED,
+ DEFAULT_MULTIPART_UPLOAD_ENABLED);
+ // multipart copy and upload are the same; this just makes it explicit
+ this.isMultipartCopyEnabled = isMultipartUploadEnabled;
initThreadPools(conf);
@@ -983,6 +993,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
.withRequesterPays(conf.getBoolean(ALLOW_REQUESTER_PAYS, DEFAULT_ALLOW_REQUESTER_PAYS))
.withExecutionInterceptors(auditManager.createExecutionInterceptors())
.withMinimumPartSize(partSize)
+ .withMultipartCopyEnabled(isMultipartCopyEnabled)
.withMultipartThreshold(multiPartThreshold)
.withTransferManagerExecutor(unboundedThreadPool)
.withRegion(region);
@@ -1468,6 +1479,11 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
LOG.debug("Sharing credentials for: {}", purpose);
return credentials.share();
}
+
+ @Override
+ public boolean isMultipartCopyEnabled() {
+ return S3AFileSystem.this.isMultipartUploadEnabled;
+ }
}
/**
@@ -4432,37 +4448,56 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
e);
}
- return readInvoker.retry(
- action, srcKey,
- true,
- () -> {
- CopyObjectRequest.Builder copyObjectRequestBuilder =
- getRequestFactory().newCopyObjectRequestBuilder(srcKey, dstKey, srcom);
- changeTracker.maybeApplyConstraint(copyObjectRequestBuilder);
- incrementStatistic(OBJECT_COPY_REQUESTS);
-
- Copy copy = transferManager.copy(
- CopyRequest.builder()
- .copyObjectRequest(copyObjectRequestBuilder.build())
- .build());
+ CopyObjectRequest.Builder copyObjectRequestBuilder =
+ getRequestFactory().newCopyObjectRequestBuilder(srcKey, dstKey, srcom);
+ changeTracker.maybeApplyConstraint(copyObjectRequestBuilder);
+ CopyObjectResponse response;
- try {
- CompletedCopy completedCopy = copy.completionFuture().join();
- CopyObjectResponse result = completedCopy.response();
- changeTracker.processResponse(result);
- incrementWriteOperations();
- instrumentation.filesCopied(1, size);
- return result;
- } catch (CompletionException e) {
- Throwable cause = e.getCause();
- if (cause instanceof SdkException) {
- SdkException awsException = (SdkException)cause;
- changeTracker.processException(awsException, "copy");
- throw awsException;
+ // transfer manager is skipped if disabled or the file is too small to worry about
+ final boolean useTransferManager = isMultipartCopyEnabled && size >= multiPartThreshold;
+ if (useTransferManager) {
+ // use transfer manager
+ response = readInvoker.retry(
+ action, srcKey,
+ true,
+ () -> {
+ incrementStatistic(OBJECT_COPY_REQUESTS);
+
+ Copy copy = transferManager.copy(
+ CopyRequest.builder()
+ .copyObjectRequest(copyObjectRequestBuilder.build())
+ .build());
+
+ try {
+ CompletedCopy completedCopy = copy.completionFuture().join();
+ return completedCopy.response();
+ } catch (CompletionException e) {
+ Throwable cause = e.getCause();
+ if (cause instanceof SdkException) {
+ SdkException awsException = (SdkException)cause;
+ changeTracker.processException(awsException, "copy");
+ throw awsException;
+ }
+ throw extractException(action, srcKey, e);
}
- throw extractException(action, srcKey, e);
- }
- });
+ });
+ } else {
+ // single part copy bypasses transfer manager
+ // note, this helps with some mock testing, e.g. HBoss. as there is less to mock.
+ response = readInvoker.retry(
+ action, srcKey,
+ true,
+ () -> {
+ LOG.debug("copyFile: single part copy {} -> {} of size {}", srcKey, dstKey, size);
+ incrementStatistic(OBJECT_COPY_REQUESTS);
+ return s3Client.copyObject(copyObjectRequestBuilder.build());
+ });
+ }
+
+ changeTracker.processResponse(response);
+ incrementWriteOperations();
+ instrumentation.filesCopied(1, size);
+ return response;
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java
index 23c4d3501206..18d6c1af586f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java
@@ -115,4 +115,10 @@ public interface S3AInternals {
@AuditEntryPoint
@Retries.RetryTranslated
HeadBucketResponse getBucketMetadata() throws IOException;
+
+ /**
+ * Is multipart copy enabled?
+ * @return true if the transfer manager is used to copy files.
+ */
+ boolean isMultipartCopyEnabled();
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
index d4504cd08d74..e2e792ebfb66 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
@@ -156,6 +156,11 @@ public interface S3ClientFactory {
*/
private long multiPartThreshold;
+ /**
+ * Multipart upload enabled.
+ */
+ private boolean multipartCopy = true;
+
/**
* Executor that the transfer manager will use to execute background tasks.
*/
@@ -399,5 +404,24 @@ public interface S3ClientFactory {
public Region getRegion() {
return region;
}
+
+ /**
+ * Set the multipart flag..
+ *
+ * @param value new value
+ * @return the builder
+ */
+ public S3ClientCreationParameters withMultipartCopyEnabled(final boolean value) {
+ this.multipartCopy = value;
+ return this;
+ }
+
+ /**
+ * Get the multipart flag.
+ * @return multipart flag
+ */
+ public boolean isMultipartCopy() {
+ return multipartCopy;
+ }
}
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java
index f30eb0f11ae0..628b56704182 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java
@@ -355,10 +355,10 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
/**
* Is this expected to be a multipart upload?
* Assertions will change if not.
- * @return true by default.
+ * @return what the filesystem expects.
*/
protected boolean expectMultipartUpload() {
- return true;
+ return getFileSystem().getS3AInternals().isMultipartCopyEnabled();
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesNoMultipart.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesNoMultipart.java
index ed300dba01ea..e154ab5676f8 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesNoMultipart.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesNoMultipart.java
@@ -18,13 +18,10 @@
package org.apache.hadoop.fs.s3a.scale;
+import org.assertj.core.api.Assertions;
+
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.s3a.Constants;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.apache.hadoop.fs.s3a.api.UnsupportedRequestException;
import static org.apache.hadoop.fs.contract.ContractTestUtils.IO_CHUNK_BUFFER_SIZE;
import static org.apache.hadoop.fs.s3a.Constants.MIN_MULTIPART_THRESHOLD;
@@ -33,7 +30,6 @@ import static org.apache.hadoop.fs.s3a.Constants.MULTIPART_SIZE;
import static org.apache.hadoop.fs.s3a.Constants.MULTIPART_UPLOADS_ENABLED;
import static org.apache.hadoop.fs.s3a.Constants.REQUEST_TIMEOUT;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
-import static org.apache.hadoop.test.LambdaTestUtils.intercept;
/**
* Use a single PUT for the whole upload/rename/delete workflow; include verification
@@ -41,11 +37,6 @@ import static org.apache.hadoop.test.LambdaTestUtils.intercept;
*/
public class ITestS3AHugeFilesNoMultipart extends AbstractSTestS3AHugeFiles {
- /**
- * Size to ensure MPUs don't happen in transfer manager.
- */
- public static final String S_1T = "1T";
-
public static final String SINGLE_PUT_REQUEST_TIMEOUT = "1h";
/**
@@ -56,11 +47,23 @@ public class ITestS3AHugeFilesNoMultipart extends AbstractSTestS3AHugeFiles {
return Constants.FAST_UPLOAD_BUFFER_DISK;
}
+ /**
+ * Multipart upload is always disabled.
+ * @return false
+ */
@Override
protected boolean expectMultipartUpload() {
return false;
}
+ /**
+ * Is multipart copy enabled?
+ * @return true if the transfer manager is used to copy files.
+ */
+ private boolean isMultipartCopyEnabled() {
+ return getFileSystem().getS3AInternals().isMultipartCopyEnabled();
+ }
+
/**
* Create a configuration without multipart upload,
* and a long request timeout to allow for a very slow
@@ -77,35 +80,21 @@ public class ITestS3AHugeFilesNoMultipart extends AbstractSTestS3AHugeFiles {
MULTIPART_SIZE,
REQUEST_TIMEOUT);
conf.setInt(IO_CHUNK_BUFFER_SIZE, 655360);
- conf.set(MIN_MULTIPART_THRESHOLD, S_1T);
- conf.set(MULTIPART_SIZE, S_1T);
+ conf.setInt(MIN_MULTIPART_THRESHOLD, MULTIPART_MIN_SIZE);
+ conf.setInt(MULTIPART_SIZE, MULTIPART_MIN_SIZE);
conf.setBoolean(MULTIPART_UPLOADS_ENABLED, false);
conf.set(REQUEST_TIMEOUT, SINGLE_PUT_REQUEST_TIMEOUT);
return conf;
}
/**
- * After the file is created, attempt a rename with an FS
- * instance with a small multipart threshold;
- * this MUST be rejected.
+ * Verify multipart copy is disabled.
*/
@Override
public void test_030_postCreationAssertions() throws Throwable {
- assumeHugeFileExists();
- final Path hugefile = getHugefile();
- final Path hugefileRenamed = getHugefileRenamed();
- describe("renaming %s to %s", hugefile, hugefileRenamed);
- S3AFileSystem fs = getFileSystem();
- fs.delete(hugefileRenamed, false);
- // create a new fs with a small multipart threshold; expect rename failure.
- final Configuration conf = new Configuration(fs.getConf());
- conf.setInt(MIN_MULTIPART_THRESHOLD, MULTIPART_MIN_SIZE);
- conf.setInt(MULTIPART_SIZE, MULTIPART_MIN_SIZE);
- S3ATestUtils.disableFilesystemCaching(conf);
-
- try (FileSystem fs2 = FileSystem.get(fs.getUri(), conf)) {
- intercept(UnsupportedRequestException.class, () ->
- fs2.rename(hugefile, hugefileRenamed));
- }
+ super.test_030_postCreationAssertions();
+ Assertions.assertThat(isMultipartCopyEnabled())
+ .describedAs("Multipart copy should be disabled in %s", getFileSystem())
+ .isFalse();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 06/17: HADOOP-18939. NPE in AWS v2 SDK RetryOnErrorCodeCondition.shouldRetry() (#6193)
Posted by ah...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ahmar pushed a commit to branch feature-HADOOP-18073-sdk-v2-upgrade-3.6-aws
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit b7fee114e6f87bc5b820a056cb378fddc12dd545
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Tue Oct 17 15:17:16 2023 +0100
HADOOP-18939. NPE in AWS v2 SDK RetryOnErrorCodeCondition.shouldRetry() (#6193)
MultiObjectDeleteException to fill in the error details
See also: https://github.com/aws/aws-sdk-java-v2/issues/4600
Contributed by Steve Loughran
---
.../fs/s3a/impl/MultiObjectDeleteException.java | 33 +++++++++++++++++++++-
.../hadoop/fs/s3a/impl/TestErrorTranslation.java | 22 +++++++++++++++
2 files changed, 54 insertions(+), 1 deletion(-)
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MultiObjectDeleteException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MultiObjectDeleteException.java
index 6082c2f08daf..72ead1fb151f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MultiObjectDeleteException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MultiObjectDeleteException.java
@@ -22,6 +22,8 @@ import java.io.IOException;
import java.nio.file.AccessDeniedException;
import java.util.List;
+import software.amazon.awssdk.awscore.exception.AwsErrorDetails;
+import software.amazon.awssdk.http.SdkHttpResponse;
import software.amazon.awssdk.services.s3.model.S3Error;
import software.amazon.awssdk.services.s3.model.S3Exception;
import org.slf4j.Logger;
@@ -55,10 +57,39 @@ public class MultiObjectDeleteException extends S3Exception {
*/
public static final String ACCESS_DENIED = "AccessDenied";
+ /**
+ * Field value for the superclass builder: {@value}.
+ */
+ private static final int STATUS_CODE = SC_200_OK;
+
+ /**
+ * Field value for the superclass builder: {@value}.
+ */
+ private static final String ERROR_CODE = "MultiObjectDeleteException";
+
+ /**
+ * Field value for the superclass builder: {@value}.
+ */
+ private static final String SERVICE_NAME = "Amazon S3";
+
+ /**
+ * Extracted error list.
+ */
private final List<S3Error> errors;
public MultiObjectDeleteException(List<S3Error> errors) {
- super(builder().message(errors.toString()).statusCode(SC_200_OK));
+ super(builder()
+ .message(errors.toString())
+ .awsErrorDetails(
+ AwsErrorDetails.builder()
+ .errorCode(ERROR_CODE)
+ .errorMessage(ERROR_CODE)
+ .serviceName(SERVICE_NAME)
+ .sdkHttpResponse(SdkHttpResponse.builder()
+ .statusCode(STATUS_CODE)
+ .build())
+ .build())
+ .statusCode(STATUS_CODE));
this.errors = errors;
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java
index 71536880dd5c..dd4abe64c65c 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java
@@ -22,15 +22,20 @@ import java.io.IOException;
import java.net.ConnectException;
import java.net.NoRouteToHostException;
import java.net.UnknownHostException;
+import java.util.Collections;
+import org.assertj.core.api.Assertions;
import org.junit.Test;
+import software.amazon.awssdk.awscore.retry.conditions.RetryOnErrorCodeCondition;
import software.amazon.awssdk.core.exception.SdkClientException;
+import software.amazon.awssdk.core.retry.RetryPolicyContext;
import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.test.AbstractHadoopTestBase;
import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractNetworkException;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.junit.Assert.assertTrue;
/**
* Unit tests related to the {@link ErrorTranslation} class.
@@ -112,4 +117,21 @@ public class TestErrorTranslation extends AbstractHadoopTestBase {
}
}
+
+ @Test
+ public void testMultiObjectExceptionFilledIn() throws Throwable {
+
+ MultiObjectDeleteException ase =
+ new MultiObjectDeleteException(Collections.emptyList());
+ RetryPolicyContext context = RetryPolicyContext.builder()
+ .exception(ase)
+ .build();
+ RetryOnErrorCodeCondition retry = RetryOnErrorCodeCondition.create("");
+ assertTrue("retry policy of MultiObjectException",
+ retry.shouldRetry(context));
+
+ Assertions.assertThat(retry.shouldRetry(context))
+ .describedAs("retry policy of MultiObjectException")
+ .isFalse();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 10/17: HADOOP-18889. Third party storage followup. (#6186)
Posted by ah...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ahmar pushed a commit to branch feature-HADOOP-18073-sdk-v2-upgrade-3.6-aws
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit e14b08b030d3b6a4742390ea4fd2917932279320
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Tue Oct 24 18:17:52 2023 +0100
HADOOP-18889. Third party storage followup. (#6186)
Followup to HADOOP-18889 third party store support;
Fix some minor review comments which came in after the merge.
---
.../src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 2 +-
.../main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java | 10 ----------
2 files changed, 1 insertion(+), 11 deletions(-)
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index ff2a35a6315a..4a0c8594341d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -1358,7 +1358,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
public String getBucketLocation(String bucketName) throws IOException {
final String region = trackDurationAndSpan(
STORE_EXISTS_PROBE, bucketName, null, () ->
- once("getBucketLocation()", bucketName, () ->
+ invoker.retry("getBucketLocation()", bucketName, true, () ->
// If accessPoint then region is known from Arn
accessPoint != null
? accessPoint.getRegion()
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
index 5d34688cebe1..21c390c07940 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
@@ -81,16 +81,6 @@ public final class SignerFactory {
SIGNERS.put(signerType, signerClass);
}
- /**
- * Check if the signer has already been registered.
- * @param signerType signer to get
- * @throws IllegalArgumentException if the signer type is unknown.
- */
- public static void verifySignerRegistered(String signerType) {
- checkArgument(isSignerRegistered(signerType),
- "unknown signer type: %s", signerType);
- }
-
/**
* Check if the signer has already been registered.
* @param signerType signer to get
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 15/17: S3A: Upgrade AWS SDK version to 2.21.33 for Amazon S3 Express One Zone support (#6306)
Posted by ah...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ahmar pushed a commit to branch feature-HADOOP-18073-sdk-v2-upgrade-3.6-aws
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit f7daa6f7d34f0e493fa2459193361c2cf2381571
Author: ahmarsuhail <ah...@amazon.co.uk>
AuthorDate: Wed Nov 29 13:16:19 2023 +0000
S3A: Upgrade AWS SDK version to 2.21.33 for Amazon S3 Express One Zone support (#6306)
With this upgrade, it is possible to connect to an Amazon S3 Express One Zone bucket.
Some tests from the S3A test suite will currently fail against a one zone bucket, as one zone buckets
do not support some S3 standard features (eg: SSE-KMS), and certain operations behave slightly
differently (eg: listMPU will return a directory that has incomplete MPUs).
Contributed by Ahmar Suhail
---
LICENSE-binary | 2 +-
hadoop-project/pom.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/LICENSE-binary b/LICENSE-binary
index 2df6cf730b7a..3279137509ef 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -364,7 +364,7 @@ org.objenesis:objenesis:2.6
org.xerial.snappy:snappy-java:1.1.10.4
org.yaml:snakeyaml:2.0
org.wildfly.openssl:wildfly-openssl:1.1.3.Final
-software.amazon.awssdk:bundle:jar:2.20.160
+software.amazon.awssdk:bundle:jar:2.21.33
--------------------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 7270a8953e51..dd37935ad064 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -192,7 +192,7 @@
<make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
<surefire.fork.timeout>900</surefire.fork.timeout>
<aws-java-sdk.version>1.12.565</aws-java-sdk.version>
- <aws-java-sdk-v2.version>2.20.160</aws-java-sdk-v2.version>
+ <aws-java-sdk-v2.version>2.21.33</aws-java-sdk-v2.version>
<aws.eventstream.version>1.0.1</aws.eventstream.version>
<hsqldb.version>2.7.1</hsqldb.version>
<frontend-maven-plugin.version>1.11.2</frontend-maven-plugin.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 11/17: HADOOP-18948. S3A. Add option fs.s3a.directory.operations.purge.uploads to purge on rename/delete (#6218)
Posted by ah...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ahmar pushed a commit to branch feature-HADOOP-18073-sdk-v2-upgrade-3.6-aws
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 726e2171a665a0f9dd1449abb80391ce0b483981
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Wed Oct 25 17:39:16 2023 +0100
HADOOP-18948. S3A. Add option fs.s3a.directory.operations.purge.uploads to purge on rename/delete (#6218)
S3A directory delete and rename will optionally abort all pending multipart uploads
in their under their to-be-deleted paths when.
fs.s3a.directory.operations.purge.upload is true
It is off by default.
The filesystems hasPathCapability("fs.s3a.directory.operations.purge.upload")
probe will return true when this feature is enabled.
Multipart uploads may accrue from interrupted data writes, uncommitted
staging/magic committer jobs and other operations/applications. On AWS S3
lifecycle rules are the recommended way to clean these; this change improves
support for stores which lack these rules.
Contributed by Steve Loughran
---
.../hadoop/fs/statistics/StoreStatisticNames.java | 7 +
.../java/org/apache/hadoop/fs/s3a/Constants.java | 15 ++
.../org/apache/hadoop/fs/s3a/MultipartUtils.java | 6 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 152 ++++++++++++++-----
.../java/org/apache/hadoop/fs/s3a/Statistic.java | 5 +-
.../hadoop/fs/s3a/impl/CallableSupplier.java | 33 +++--
.../apache/hadoop/fs/s3a/impl/DeleteOperation.java | 46 +++++-
.../hadoop/fs/s3a/impl/OperationCallbacks.java | 12 ++
.../apache/hadoop/fs/s3a/impl/RenameOperation.java | 39 ++++-
.../apache/hadoop/fs/s3a/s3guard/S3GuardTool.java | 4 +-
.../tools/hadoop-aws/third_party_stores.md | 57 ++++++-
.../hadoop/fs/s3a/ITestS3AMultipartUtils.java | 5 +-
.../apache/hadoop/fs/s3a/MultipartTestUtils.java | 7 +-
.../apache/hadoop/fs/s3a/auth/ITestAssumeRole.java | 5 +-
.../ITestUploadPurgeOnDirectoryOperations.java | 163 +++++++++++++++++++++
.../fs/s3a/performance/AbstractS3ACostTest.java | 7 +
.../hadoop/fs/s3a/s3guard/ITestS3GuardTool.java | 19 ++-
17 files changed, 499 insertions(+), 83 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java
index c04c1bb47fce..19ee9d1414ec 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java
@@ -244,6 +244,13 @@ public final class StoreStatisticNames {
public static final String OBJECT_MULTIPART_UPLOAD_ABORTED =
"object_multipart_aborted";
+ /**
+ * Object multipart list request.
+ * Value :{@value}.
+ */
+ public static final String OBJECT_MULTIPART_UPLOAD_LIST =
+ "object_multipart_list";
+
/**
* Object put/multipart upload count.
* Value :{@value}.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 67d21ddfaa71..944ab5dcdcb0 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -1318,4 +1318,19 @@ public final class Constants {
* The bucket region header.
*/
public static final String BUCKET_REGION_HEADER = "x-amz-bucket-region";
+
+ /**
+ * Should directory operations purge uploads?
+ * This adds at least one parallelized list operation to the call,
+ * plus the overhead of deletions.
+ * Value: {@value}.
+ */
+ public static final String DIRECTORY_OPERATIONS_PURGE_UPLOADS =
+ "fs.s3a.directory.operations.purge.uploads";
+
+ /**
+ * Default value of {@link #DIRECTORY_OPERATIONS_PURGE_UPLOADS}: {@value}.
+ */
+ public static final boolean DIRECTORY_OPERATIONS_PURGE_UPLOADS_DEFAULT = false;
+
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java
index efca093204c2..b2057c211da7 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.fs.s3a.api.RequestFactory;
import org.apache.hadoop.fs.s3a.impl.StoreContext;
import org.apache.hadoop.fs.store.audit.AuditSpan;
-import static org.apache.hadoop.fs.s3a.Statistic.MULTIPART_UPLOAD_LIST;
+import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_MULTIPART_UPLOAD_LIST;
import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfOperation;
@@ -66,7 +66,7 @@ public final class MultipartUtils {
* @param maxKeys maximum batch size to request at a time from S3.
* @return an iterator of matching uploads
*/
- static MultipartUtils.UploadIterator listMultipartUploads(
+ static RemoteIterator<MultipartUpload> listMultipartUploads(
final StoreContext storeContext,
S3Client s3,
@Nullable String prefix,
@@ -196,7 +196,7 @@ public final class MultipartUtils {
listing = invoker.retry("listMultipartUploads", prefix, true,
trackDurationOfOperation(storeContext.getInstrumentation(),
- MULTIPART_UPLOAD_LIST.getSymbol(),
+ OBJECT_MULTIPART_UPLOAD_LIST.getSymbol(),
() -> s3.listMultipartUploads(requestBuilder.build())));
LOG.debug("Listing found {} upload(s)",
listing.uploads().size());
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 4a0c8594341d..db2cedd37fc9 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -258,6 +258,7 @@ import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDura
import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfSupplier;
import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
import static org.apache.hadoop.util.Preconditions.checkArgument;
+import static org.apache.hadoop.util.functional.RemoteIterators.foreach;
import static org.apache.hadoop.util.functional.RemoteIterators.typeCastingRemoteIterator;
/**
@@ -384,6 +385,11 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
private SignerManager signerManager;
private S3AInternals s3aInternals;
+ /**
+ * Do directory operations purge pending uploads?
+ */
+ private boolean dirOperationsPurgeUploads;
+
/**
* Page size for deletions.
*/
@@ -565,6 +571,9 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
//check but do not store the block size
longBytesOption(conf, FS_S3A_BLOCK_SIZE, DEFAULT_BLOCKSIZE, 1);
enableMultiObjectsDelete = conf.getBoolean(ENABLE_MULTI_DELETE, true);
+ // should the delete also purge uploads.
+ dirOperationsPurgeUploads = conf.getBoolean(DIRECTORY_OPERATIONS_PURGE_UPLOADS,
+ DIRECTORY_OPERATIONS_PURGE_UPLOADS_DEFAULT);
this.isMultipartUploadEnabled = conf.getBoolean(MULTIPART_UPLOADS_ENABLED,
DEFAULT_MULTIPART_UPLOAD_ENABLED);
@@ -1231,7 +1240,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
purgeBefore);
invoker.retry("Purging multipart uploads", bucket, true,
() -> {
- MultipartUtils.UploadIterator uploadIterator =
+ RemoteIterator<MultipartUpload> uploadIterator =
MultipartUtils.listMultipartUploads(createStoreContext(), s3Client, null, maxKeys);
while (uploadIterator.hasNext()) {
@@ -2280,12 +2289,14 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
// Initiate the rename.
// this will call back into this class via the rename callbacks
+ final StoreContext storeContext = createStoreContext();
RenameOperation renameOperation = new RenameOperation(
- createStoreContext(),
+ storeContext,
src, srcKey, p.getLeft(),
dst, dstKey, p.getRight(),
- new OperationCallbacksImpl(),
- pageSize);
+ new OperationCallbacksImpl(storeContext),
+ pageSize,
+ dirOperationsPurgeUploads);
return renameOperation.execute();
}
@@ -2306,8 +2317,19 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
/** Audit Span at time of creation. */
private final AuditSpan auditSpan;
- private OperationCallbacksImpl() {
- auditSpan = getActiveAuditSpan();
+ private final StoreContext storeContext;
+
+ private OperationCallbacksImpl(final StoreContext storeContext) {
+ this.storeContext = requireNonNull(storeContext);
+ this.auditSpan = storeContext.getActiveAuditSpan();
+ }
+
+ /**
+ * Get the audit span.
+ * @return the span
+ */
+ private AuditSpan getAuditSpan() {
+ return auditSpan;
}
@Override
@@ -2407,7 +2429,29 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
Listing.ACCEPT_ALL_BUT_S3N,
auditSpan));
}
- }
+
+ /**
+ * Abort multipart uploads under a path.
+ * @param prefix prefix for uploads to abort
+ * @return a count of aborts
+ * @throws IOException trouble; FileNotFoundExceptions are swallowed.
+ */
+ @Override
+ @Retries.RetryTranslated
+ public long abortMultipartUploadsUnderPrefix(String prefix)
+ throws IOException {
+ getAuditSpan().activate();
+ // this deactivates the audit span somehow
+ final RemoteIterator<MultipartUpload> uploads =
+ S3AFileSystem.this.listUploadsUnderPrefix(storeContext, prefix);
+ // so reactivate it.
+ getAuditSpan().activate();
+ return foreach(uploads, upload ->
+ invoker.retry("Aborting multipart commit", upload.key(), true, () ->
+ S3AFileSystem.this.abortMultipartUpload(upload)));
+ }
+
+ } // end OperationCallbacksImpl
/**
* Callbacks from {@link Listing}.
@@ -3368,14 +3412,17 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
// span covers delete, getFileStatus, fake directory operations.
try (AuditSpan span = createSpan(INVOCATION_DELETE.getSymbol(),
path.toString(), null)) {
+ // SC will include active span
+ final StoreContext storeContext = createStoreContext();
boolean outcome = trackDuration(getDurationTrackerFactory(),
INVOCATION_DELETE.getSymbol(),
new DeleteOperation(
- createStoreContext(),
+ storeContext,
innerGetFileStatus(path, true, StatusProbeEnum.ALL),
recursive,
- new OperationCallbacksImpl(),
- pageSize));
+ new OperationCallbacksImpl(storeContext),
+ pageSize,
+ dirOperationsPurgeUploads));
if (outcome) {
try {
maybeCreateFakeParentDirectory(path);
@@ -5148,13 +5195,39 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
@InterfaceAudience.Private
@Retries.RetryTranslated
@AuditEntryPoint
- public MultipartUtils.UploadIterator listUploads(@Nullable String prefix)
+ public RemoteIterator<MultipartUpload> listUploads(@Nullable String prefix)
+ throws IOException {
+ // span is picked up retained in the listing.
+ checkNotClosed();
+ try (AuditSpan span = createSpan(MULTIPART_UPLOAD_LIST.getSymbol(),
+ prefix, null)) {
+ return listUploadsUnderPrefix(createStoreContext(), prefix);
+ }
+ }
+
+ /**
+ * List any pending multipart uploads whose keys begin with prefix, using
+ * an iterator that can handle an unlimited number of entries.
+ * See {@link #listMultipartUploads(String)} for a non-iterator version of
+ * this.
+ * @param storeContext store conext.
+ * @param prefix optional key prefix to search
+ * @return Iterator over multipart uploads.
+ * @throws IOException on failure
+ */
+ @InterfaceAudience.Private
+ @Retries.RetryTranslated
+ public RemoteIterator<MultipartUpload> listUploadsUnderPrefix(
+ final StoreContext storeContext,
+ final @Nullable String prefix)
throws IOException {
// span is picked up retained in the listing.
- return trackDurationAndSpan(MULTIPART_UPLOAD_LIST, prefix, null, () ->
- MultipartUtils.listMultipartUploads(
- createStoreContext(), s3Client, prefix, maxKeys
- ));
+ String p = prefix;
+ if (prefix != null && !prefix.isEmpty() && !prefix.endsWith("/")) {
+ p = prefix + "/";
+ }
+ // duration tracking is done in iterator.
+ return MultipartUtils.listMultipartUploads(storeContext, s3Client, p, maxKeys);
}
/**
@@ -5176,9 +5249,10 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
}
String p = prefix;
return invoker.retry("listMultipartUploads", p, true, () -> {
- ListMultipartUploadsRequest.Builder requestBuilder = getRequestFactory()
- .newListMultipartUploadsRequestBuilder(p);
- return s3Client.listMultipartUploads(requestBuilder.build()).uploads();
+ final ListMultipartUploadsRequest request = getRequestFactory()
+ .newListMultipartUploadsRequestBuilder(p).build();
+ return trackDuration(getInstrumentation(), MULTIPART_UPLOAD_LIST.getSymbol(), () ->
+ s3Client.listMultipartUploads(request).uploads());
});
}
@@ -5187,37 +5261,35 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* Retry policy: none.
* @param destKey destination key
* @param uploadId Upload ID
+ * @throws IOException IO failure, including any uprated SdkException
*/
- @Retries.OnceRaw
- void abortMultipartUpload(String destKey, String uploadId) {
- LOG.info("Aborting multipart upload {} to {}", uploadId, destKey);
- s3Client.abortMultipartUpload(
- getRequestFactory().newAbortMultipartUploadRequestBuilder(
- destKey,
- uploadId).build());
+ @Retries.OnceTranslated
+ public void abortMultipartUpload(String destKey, String uploadId) throws IOException {
+ LOG.debug("Aborting multipart upload {} to {}", uploadId, destKey);
+ trackDuration(getInstrumentation(), OBJECT_MULTIPART_UPLOAD_ABORTED.getSymbol(), () ->
+ s3Client.abortMultipartUpload(
+ getRequestFactory().newAbortMultipartUploadRequestBuilder(
+ destKey,
+ uploadId).build()));
}
/**
* Abort a multipart upload.
* Retry policy: none.
* @param upload the listed upload to abort.
+ * @throws IOException IO failure, including any uprated SdkException
*/
- @Retries.OnceRaw
- void abortMultipartUpload(MultipartUpload upload) {
- String destKey;
- String uploadId;
- destKey = upload.key();
- uploadId = upload.uploadId();
- if (LOG.isInfoEnabled()) {
+ @Retries.OnceTranslated
+ public void abortMultipartUpload(MultipartUpload upload) throws IOException {
+ String destKey = upload.key();
+ String uploadId = upload.uploadId();
+ if (LOG.isDebugEnabled()) {
DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
LOG.debug("Aborting multipart upload {} to {} initiated by {} on {}",
uploadId, destKey, upload.initiator(),
df.format(Date.from(upload.initiated())));
}
- s3Client.abortMultipartUpload(
- getRequestFactory().newAbortMultipartUploadRequestBuilder(
- destKey,
- uploadId).build());
+ abortMultipartUpload(destKey, uploadId);
}
/**
@@ -5263,13 +5335,17 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
case STORE_CAPABILITY_DIRECTORY_MARKER_AWARE:
return true;
+ // Do directory operations purge uploads.
+ case DIRECTORY_OPERATIONS_PURGE_UPLOADS:
+ return dirOperationsPurgeUploads;
+
// etags are avaialable in listings, but they
// are not consistent across renames.
// therefore, only availability is declared
case CommonPathCapabilities.ETAGS_AVAILABLE:
return true;
- /*
+ /*
* Marker policy capabilities are handed off.
*/
case STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_KEEP:
@@ -5539,7 +5615,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
throws IOException {
createSpan("marker-tool-scan", target,
null);
- return new MarkerToolOperationsImpl(new OperationCallbacksImpl());
+ return new MarkerToolOperationsImpl(new OperationCallbacksImpl(createStoreContext()));
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
index f4e28aa62783..72fc75b64241 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
@@ -242,7 +242,10 @@ public enum Statistic {
StoreStatisticNames.OBJECT_MULTIPART_UPLOAD_ABORTED,
"Object multipart upload aborted",
TYPE_DURATION),
- OBJECT_PUT_REQUESTS(
+ OBJECT_MULTIPART_UPLOAD_LIST(
+ StoreStatisticNames.OBJECT_MULTIPART_UPLOAD_LIST,
+ "Object multipart list request issued",
+ TYPE_DURATION), OBJECT_PUT_REQUESTS(
StoreStatisticNames.OBJECT_PUT_REQUEST,
"Object put/multipart upload count",
TYPE_DURATION),
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CallableSupplier.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CallableSupplier.java
index 015620741921..e0580df08a76 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CallableSupplier.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CallableSupplier.java
@@ -22,6 +22,7 @@ import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.List;
+import java.util.Optional;
import java.util.concurrent.Callable;
import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletableFuture;
@@ -155,19 +156,21 @@ public final class CallableSupplier<T> implements Supplier<T> {
* Wait for a single of future to complete, extracting IOEs afterwards.
* @param future future to wait for.
* @param <T> type
+ * @return the result
* @throws IOException if one of the called futures raised an IOE.
* @throws RuntimeException if one of the futures raised one.
*/
- public static <T> void waitForCompletion(
+ public static <T> T waitForCompletion(
final CompletableFuture<T> future)
throws IOException {
try (DurationInfo ignore =
new DurationInfo(LOG, false, "Waiting for task completion")) {
- future.join();
+ return future.join();
} catch (CancellationException e) {
throw new IOException(e);
} catch (CompletionException e) {
raiseInnerCause(e);
+ return null;
}
}
@@ -175,31 +178,35 @@ public final class CallableSupplier<T> implements Supplier<T> {
* Wait for a single of future to complete, ignoring exceptions raised.
* @param future future to wait for.
* @param <T> type
+ * @return the outcome if successfully retrieved.
*/
- public static <T> void waitForCompletionIgnoringExceptions(
+ public static <T> Optional<T> waitForCompletionIgnoringExceptions(
@Nullable final CompletableFuture<T> future) {
- if (future != null) {
- try (DurationInfo ignore =
- new DurationInfo(LOG, false, "Waiting for task completion")) {
- future.join();
- } catch (Exception e) {
- LOG.debug("Ignoring exception raised in task completion: ");
- }
+
+ try {
+ return maybeAwaitCompletion(future);
+ } catch (Exception e) {
+ LOG.debug("Ignoring exception raised in task completion: ", e);
+ return Optional.empty();
}
}
/**
* Block awaiting completion for any non-null future passed in;
* No-op if a null arg was supplied.
+ * @param <T> return type
* @param future future
+ * @return the outcome; is empty if the future was null/had no return value
* @throws IOException if one of the called futures raised an IOE.
* @throws RuntimeException if one of the futures raised one.
*/
- public static void maybeAwaitCompletion(
- @Nullable final CompletableFuture<Void> future)
+ public static <T> Optional<T> maybeAwaitCompletion(
+ @Nullable final CompletableFuture<T> future)
throws IOException {
if (future != null) {
- waitForCompletion(future);
+ return Optional.ofNullable(waitForCompletion(future));
+ } else {
+ return Optional.empty();
}
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java
index 444693589060..7cc94cf89203 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.fs.s3a.impl;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
@@ -41,6 +42,7 @@ import org.apache.hadoop.fs.s3a.Tristate;
import org.apache.hadoop.util.DurationInfo;
+import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.waitForCompletionIgnoringExceptions;
import static org.apache.hadoop.fs.store.audit.AuditingFunctions.callableWithinAuditSpan;
import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument;
import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.maybeAwaitCompletion;
@@ -110,6 +112,16 @@ public class DeleteOperation extends ExecutingStoreOperation<Boolean> {
*/
private long filesDeleted;
+ /**
+ * Do directory operations purge pending uploads?
+ */
+ private final boolean dirOperationsPurgeUploads;
+
+ /**
+ * Count of uploads aborted.
+ */
+ private Optional<Long> uploadsAborted = Optional.empty();
+
/**
* Constructor.
* @param context store context
@@ -117,12 +129,14 @@ public class DeleteOperation extends ExecutingStoreOperation<Boolean> {
* @param recursive recursive delete?
* @param callbacks callback provider
* @param pageSize size of delete pages
+ * @param dirOperationsPurgeUploads Do directory operations purge pending uploads?
*/
public DeleteOperation(final StoreContext context,
final S3AFileStatus status,
final boolean recursive,
final OperationCallbacks callbacks,
- final int pageSize) {
+ final int pageSize,
+ final boolean dirOperationsPurgeUploads) {
super(context);
this.status = status;
@@ -134,12 +148,22 @@ public class DeleteOperation extends ExecutingStoreOperation<Boolean> {
this.pageSize = pageSize;
executor = MoreExecutors.listeningDecorator(
context.createThrottledExecutor(1));
+ this.dirOperationsPurgeUploads = dirOperationsPurgeUploads;
}
public long getFilesDeleted() {
return filesDeleted;
}
+ /**
+ * Get the count of uploads aborted.
+ * Non-empty iff enabled, and the operations completed without errors.
+ * @return count of aborted uploads.
+ */
+ public Optional<Long> getUploadsAborted() {
+ return uploadsAborted;
+ }
+
/**
* Delete a file or directory tree.
* <p>
@@ -236,6 +260,17 @@ public class DeleteOperation extends ExecutingStoreOperation<Boolean> {
try (DurationInfo ignored =
new DurationInfo(LOG, false, "deleting %s", dirKey)) {
+ final CompletableFuture<Long> abortUploads;
+ if (dirOperationsPurgeUploads) {
+ final StoreContext sc = getStoreContext();
+ final String key = sc.pathToKey(path) + "/";
+ LOG.debug("All uploads under {} will be deleted", key);
+ abortUploads = submit(sc.getExecutor(), sc.getActiveAuditSpan(), () ->
+ callbacks.abortMultipartUploadsUnderPrefix(key));
+ } else {
+ abortUploads = null;
+ }
+
// init the lists of keys and paths to delete
resetDeleteList();
deleteFuture = null;
@@ -257,10 +292,10 @@ public class DeleteOperation extends ExecutingStoreOperation<Boolean> {
LOG.debug("Deleting final batch of listed files");
submitNextBatch();
maybeAwaitCompletion(deleteFuture);
-
+ uploadsAborted = waitForCompletionIgnoringExceptions(abortUploads);
}
- LOG.debug("Delete \"{}\" completed; deleted {} objects", path,
- filesDeleted);
+ LOG.debug("Delete \"{}\" completed; deleted {} objects and aborted {} uploads", path,
+ filesDeleted, uploadsAborted.orElse(0L));
}
/**
@@ -313,7 +348,8 @@ public class DeleteOperation extends ExecutingStoreOperation<Boolean> {
throws IOException {
// delete a single page of keys and the metadata.
// block for any previous batch.
- maybeAwaitCompletion(deleteFuture);
+ maybeAwaitCompletion(deleteFuture).ifPresent(count ->
+ LOG.debug("Deleted {} uploads", count));
// delete the current page of keys and paths
deleteFuture = submitDelete(keys);
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/OperationCallbacks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/OperationCallbacks.java
index e0d9c7c6aada..9c88870633a3 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/OperationCallbacks.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/OperationCallbacks.java
@@ -164,4 +164,16 @@ public interface OperationCallbacks {
Path path,
String key)
throws IOException;
+
+ /**
+ * Abort multipart uploads under a path; paged.
+ * @param prefix prefix for uploads to abort
+ * @return a count of aborts
+ * @throws IOException trouble; FileNotFoundExceptions are swallowed.
+ */
+ @Retries.RetryTranslated
+ default long abortMultipartUploadsUnderPrefix(String prefix)
+ throws IOException {
+ return 0;
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RenameOperation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RenameOperation.java
index 4bb15f74965a..288b3c0aae58 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RenameOperation.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RenameOperation.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.AtomicLong;
@@ -44,6 +45,7 @@ import org.apache.hadoop.util.DurationInfo;
import org.apache.hadoop.util.OperationDuration;
import static org.apache.hadoop.fs.s3a.S3AUtils.translateException;
+import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.waitForCompletionIgnoringExceptions;
import static org.apache.hadoop.fs.store.audit.AuditingFunctions.callableWithinAuditSpan;
import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.submit;
import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.waitForCompletion;
@@ -124,9 +126,18 @@ public class RenameOperation extends ExecutingStoreOperation<Long> {
private final List<ObjectIdentifier> keysToDelete =
new ArrayList<>();
+ /**
+ * Do directory operations purge pending uploads?
+ */
+ private final boolean dirOperationsPurgeUploads;
+
+ /**
+ * Count of uploads aborted.
+ */
+ private Optional<Long> uploadsAborted = Optional.empty();
+
/**
* Initiate the rename.
- *
* @param storeContext store context
* @param sourcePath source path
* @param sourceKey key of source
@@ -136,6 +147,7 @@ public class RenameOperation extends ExecutingStoreOperation<Long> {
* @param destStatus destination status.
* @param callbacks callback provider
* @param pageSize size of delete requests
+ * @param dirOperationsPurgeUploads Do directory operations purge pending uploads?
*/
public RenameOperation(
final StoreContext storeContext,
@@ -146,7 +158,8 @@ public class RenameOperation extends ExecutingStoreOperation<Long> {
final String destKey,
final S3AFileStatus destStatus,
final OperationCallbacks callbacks,
- final int pageSize) {
+ final int pageSize,
+ final boolean dirOperationsPurgeUploads) {
super(storeContext);
this.sourcePath = sourcePath;
this.sourceKey = sourceKey;
@@ -159,6 +172,16 @@ public class RenameOperation extends ExecutingStoreOperation<Long> {
&& pageSize <= InternalConstants.MAX_ENTRIES_TO_DELETE,
"page size out of range: %s", pageSize);
this.pageSize = pageSize;
+ this.dirOperationsPurgeUploads = dirOperationsPurgeUploads;
+ }
+
+ /**
+ * Get the count of uploads aborted.
+ * Non-empty iff enabled, and the operations completed without errors.
+ * @return count of aborted uploads.
+ */
+ public Optional<Long> getUploadsAborted() {
+ return uploadsAborted;
}
/**
@@ -341,6 +364,16 @@ public class RenameOperation extends ExecutingStoreOperation<Long> {
throw new RenameFailedException(srcKey, dstKey,
"cannot rename a directory to a subdirectory of itself ");
}
+ // start the async dir cleanup
+ final CompletableFuture<Long> abortUploads;
+ if (dirOperationsPurgeUploads) {
+ final String key = srcKey;
+ LOG.debug("All uploads under {} will be deleted", key);
+ abortUploads = submit(getStoreContext().getExecutor(), () ->
+ callbacks.abortMultipartUploadsUnderPrefix(key));
+ } else {
+ abortUploads = null;
+ }
if (destStatus != null
&& destStatus.isEmptyDirectory() == Tristate.TRUE) {
@@ -422,6 +455,8 @@ public class RenameOperation extends ExecutingStoreOperation<Long> {
// have been deleted.
completeActiveCopiesAndDeleteSources("final copy and delete");
+ // and if uploads were being aborted, wait for that to finish
+ uploadsAborted = waitForCompletionIgnoringExceptions(abortUploads);
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
index 22fc630dad1f..ea1ea908486e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
@@ -47,8 +47,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FilterFileSystem;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.s3a.Constants;
-import org.apache.hadoop.fs.s3a.MultipartUtils;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.WriteOperationHelper;
import org.apache.hadoop.fs.s3a.auth.RolePolicies;
@@ -683,7 +683,7 @@ public abstract class S3GuardTool extends Configured implements Tool,
private void processUploads(PrintStream out) throws IOException {
final S3AFileSystem fs = getFilesystem();
- MultipartUtils.UploadIterator uploads = fs.listUploads(prefix);
+ RemoteIterator<MultipartUpload> uploads = fs.listUploads(prefix);
// create a span so that the write operation helper
// is within one
AuditSpan span =
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/third_party_stores.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/third_party_stores.md
index a7ea7b2e5902..0216e46014c7 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/third_party_stores.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/third_party_stores.md
@@ -39,11 +39,12 @@ The features which may be unavailable include:
* Optional Bucket Probes at startup (`fs.s3a.bucket.probe = 0`).
This is now the default -do not change it.
* List API to use (`fs.s3a.list.version = 1`)
+* Bucket lifecycle rules to clean up pending uploads.
## Configuring s3a to connect to a third party store
-### Connecting to a third party object store over HTTPS
+## Connecting to a third party object store over HTTPS
The core setting for a third party store is to change the endpoint in `fs.s3a.endpoint`.
@@ -89,6 +90,57 @@ then these must be set, either in XML or (preferred) in a JCEKS file.
If per-bucket settings are used here, then third-party stores and credentials may be used alongside an AWS store.
+
+
+## Other issues
+
+### Coping without bucket lifecycle rules
+
+Not all third-party stores support bucket lifecycle rules to clean up buckets
+of incomplete uploads.
+
+This can be addressed in two ways
+* Command line: `hadoop s3guard uploads -abort -force \<path>`.
+* With `fs.s3a.multipart.purge` and a purge age set in `fs.s3a.multipart.purge.age`
+* In rename/delete `fs.s3a.directory.operations.purge.uploads = true`.
+
+#### S3Guard uploads command
+
+This can be executed on a schedule, or manually
+
+```
+hadoop s3guard uploads -abort -force s3a://bucket/
+```
+
+Consult the [S3Guard documentation](s3guard.html) for the full set of parameters.
+
+#### In startup: `fs.s3a.multipart.purge`
+
+This lists all uploads in a bucket when a filesystem is created and deletes
+all of those above a certain age.
+
+This can hurt performance on a large bucket, as the purge scans the entire tree,
+and is executed whenever a filesystem is created -which can happen many times during
+hive, spark, distcp jobs.
+
+For this reason, this option may be deleted in future, however it has long been
+available in the S3A client and so guaranteed to work across versions.
+
+#### During rename and delete: `fs.s3a.directory.operations.purge.uploads`
+
+When `fs.s3a.directory.operations.purge.uploads` is set, when a directory is renamed
+or deleted, then in parallel with the delete an attempt is made to list
+all pending uploads.
+If there are any, they are aborted (sequentially).
+
+* This is disabled by default: it adds overhead and extra cost.
+* Because it only applies to the directories being processed, directories which
+ are not renamed or deleted will retain all incomplete uploads.
+* There is no age checking: all uploads will be aborted.
+* If any other process is writing to the same directory tree, their operations
+will be cancelled.
+
+
# Troubleshooting
The most common problem when talking to third-party stores are
@@ -412,4 +464,5 @@ It is also a way to regression test foundational S3A third-party store compatibi
```
_Note_ If anyone is set up to test this reguarly, please let the hadoop developer team know if regressions do surface,
-as it is not a common test configuration.
\ No newline at end of file
+as it is not a common test configuration.
+[]
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMultipartUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMultipartUtils.java
index 263a857e0330..e0559b7c49ed 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMultipartUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMultipartUtils.java
@@ -24,6 +24,7 @@ import software.amazon.awssdk.services.s3.model.MultipartUpload;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.store.audit.AuditSpan;
import java.io.IOException;
@@ -76,7 +77,7 @@ public class ITestS3AMultipartUtils extends AbstractS3ATestBase {
// 2. Verify all uploads are found listing by prefix
describe("Verifying upload list by prefix");
- MultipartUtils.UploadIterator uploads = fs.listUploads(getPartPrefix(fs));
+ RemoteIterator<MultipartUpload> uploads = fs.listUploads(getPartPrefix(fs));
assertUploadsPresent(uploads, keySet);
// 3. Verify all uploads are found listing without prefix
@@ -97,7 +98,7 @@ public class ITestS3AMultipartUtils extends AbstractS3ATestBase {
* @param ourUploads set up uploads that should be present
* @throws IOException on I/O error
*/
- private void assertUploadsPresent(MultipartUtils.UploadIterator list,
+ private void assertUploadsPresent(RemoteIterator<MultipartUpload> list,
Set<MultipartTestUtils.IdKey> ourUploads) throws IOException {
// Don't modify passed-in set, use copy.
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java
index 3e343a9ea85f..3f6870be46b2 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java
@@ -23,6 +23,7 @@ import software.amazon.awssdk.services.s3.model.MultipartUpload;
import software.amazon.awssdk.services.s3.model.UploadPartRequest;
import software.amazon.awssdk.services.s3.model.UploadPartResponse;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
import org.apache.hadoop.fs.store.audit.AuditSpan;
import org.apache.hadoop.io.IOUtils;
@@ -96,7 +97,7 @@ public final class MultipartTestUtils {
String key = fs.pathToKey(path);
AuditSpan span = null;
try {
- MultipartUtils.UploadIterator uploads = fs.listUploads(key);
+ RemoteIterator<MultipartUpload> uploads = fs.listUploads(key);
span = fs.createSpan("multipart", path.toString(), null);
final WriteOperationHelper helper
= fs.getWriteOperationHelper();
@@ -118,7 +119,7 @@ public final class MultipartTestUtils {
public static void assertNoUploadsAt(S3AFileSystem fs, Path path) throws
Exception {
String key = fs.pathToKey(path);
- MultipartUtils.UploadIterator uploads = fs.listUploads(key);
+ RemoteIterator<MultipartUpload> uploads = fs.listUploads(key);
while (uploads.hasNext()) {
MultipartUpload upload = uploads.next();
Assert.fail("Found unexpected upload " + upload.key() + " " +
@@ -130,7 +131,7 @@ public final class MultipartTestUtils {
public static int countUploadsAt(S3AFileSystem fs, Path path) throws
IOException {
String key = fs.pathToKey(path);
- MultipartUtils.UploadIterator uploads = fs.listUploads(key);
+ RemoteIterator<MultipartUpload> uploads = fs.listUploads(key);
int count = 0;
while (uploads.hasNext()) {
MultipartUpload upload = uploads.next();
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java
index 5534bb77c0dd..12234301b50d 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.stream.IntStream;
import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.services.s3.model.MultipartUpload;
import software.amazon.awssdk.services.sts.model.StsException;
import com.fasterxml.jackson.core.JsonProcessingException;
import org.assertj.core.api.Assertions;
@@ -40,10 +41,10 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.s3a.AWSBadRequestException;
import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
-import org.apache.hadoop.fs.s3a.MultipartUtils;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.S3ATestConstants;
import org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider;
@@ -463,7 +464,7 @@ public class ITestAssumeRole extends AbstractS3ATestBase {
// list multipart uploads.
// This is part of the read policy.
int counter = 0;
- MultipartUtils.UploadIterator iterator = roleFS.listUploads("/");
+ RemoteIterator<MultipartUpload> iterator = roleFS.listUploads("/");
while (iterator.hasNext()) {
counter++;
iterator.next();
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestUploadPurgeOnDirectoryOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestUploadPurgeOnDirectoryOperations.java
new file mode 100644
index 000000000000..9e0702737598
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestUploadPurgeOnDirectoryOperations.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+import software.amazon.awssdk.services.s3.model.MultipartUpload;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.performance.AbstractS3ACostTest;
+import org.apache.hadoop.fs.store.audit.AuditSpan;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertFileHasLength;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
+import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_OPERATIONS_PURGE_UPLOADS;
+import static org.apache.hadoop.fs.s3a.MultipartTestUtils.clearAnyUploads;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static org.apache.hadoop.fs.s3a.Statistic.MULTIPART_UPLOAD_LIST;
+import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_MULTIPART_UPLOAD_ABORTED;
+import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_MULTIPART_UPLOAD_LIST;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_COMMITTER_ENABLED;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_PATH_PREFIX;
+import static org.apache.hadoop.util.functional.RemoteIterators.toList;
+
+/**
+ * Test behavior of purging uploads in rename and delete.
+ */
+public class ITestUploadPurgeOnDirectoryOperations extends AbstractS3ACostTest {
+
+ @Override
+ public Configuration createConfiguration() {
+ final Configuration conf = super.createConfiguration();
+ removeBaseAndBucketOverrides(conf,
+ DIRECTORY_OPERATIONS_PURGE_UPLOADS,
+ MAGIC_COMMITTER_ENABLED);
+ conf.setBoolean(DIRECTORY_OPERATIONS_PURGE_UPLOADS, true);
+ conf.setBoolean(MAGIC_COMMITTER_ENABLED, true);
+ return conf;
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ final S3AFileSystem fs = getFileSystem();
+ assertHasPathCapabilities(fs, new Path("/"),
+ DIRECTORY_OPERATIONS_PURGE_UPLOADS);
+ clearAnyUploads(fs, methodPath());
+ }
+
+ @Test
+ public void testDeleteWithPendingUpload() throws Throwable {
+
+ final S3AFileSystem fs = getFileSystem();
+ final Path dir = methodPath();
+
+ // create a magic file.
+ createMagicFile(fs, dir);
+
+ // and there's a pending upload
+ assertUploadCount(dir, 1);
+
+ // delete the dir, with a cost of 1 abort, 1 list.
+ verifyMetrics(() -> fs.delete(dir, true),
+ with(OBJECT_MULTIPART_UPLOAD_ABORTED, 1), // abort
+ with(OBJECT_MULTIPART_UPLOAD_LIST, 1), // HTTP request inside iterator
+ with(MULTIPART_UPLOAD_LIST, 0)); // api list call
+
+
+ // and the pending upload is gone
+ assertUploadCount(dir, 0);
+ }
+
+ @Test
+ public void testRenameWithPendingUpload() throws Throwable {
+
+ final S3AFileSystem fs = getFileSystem();
+ final Path base = methodPath();
+ final Path dir = new Path(base, "src");
+ final Path dest = new Path(base, "dest");
+
+ // create a magic file.
+ createMagicFile(fs, dir);
+
+ // and there's a pending upload
+ assertUploadCount(dir, 1);
+
+ // rename the dir, with a cost of 1 abort, 1 list.
+ verifyMetrics(() -> fs.rename(dir, dest),
+ with(OBJECT_MULTIPART_UPLOAD_ABORTED, 1), // abort
+ with(OBJECT_MULTIPART_UPLOAD_LIST, 1), // HTTP request inside iterator
+ with(MULTIPART_UPLOAD_LIST, 0)); // api list call
+
+ // and there isn't
+ assertUploadCount(dir, 0);
+ }
+
+ /**
+ * Create a magic file of "real" length more than 0 bytes long.
+ * @param fs filesystem
+ * @param dir directory
+ * @return the path
+ * @throws IOException creation failure.p
+ */
+ private static Path createMagicFile(final S3AFileSystem fs, final Path dir) throws IOException {
+ Path magicFile = new Path(dir, MAGIC_PATH_PREFIX + "001/file.txt");
+ createFile(fs, magicFile, true, "123".getBytes(StandardCharsets.UTF_8));
+
+ // the file exists but is a 0 byte marker file.
+ assertFileHasLength(fs, magicFile, 0);
+ return magicFile;
+ }
+
+ /**
+ * Assert the upload count under a dir is the expected value.
+ * Failure message will include the list of entries.
+ * @param dir dir
+ * @param expected expected count
+ * @throws IOException listing problem
+ */
+ private void assertUploadCount(final Path dir, final int expected) throws IOException {
+ Assertions.assertThat(toList(listUploads(dir)))
+ .describedAs("uploads under %s", dir)
+ .hasSize(expected);
+ }
+
+ /**
+ * List uploads; use the same APIs that the directory operations use,
+ * so implicitly validating them.
+ * @param dir directory to list
+ * @return full list of entries
+ * @throws IOException listing problem
+ */
+ private RemoteIterator<MultipartUpload> listUploads(Path dir) throws IOException {
+ final S3AFileSystem fs = getFileSystem();
+ try (AuditSpan ignored = span()) {
+ final StoreContext sc = fs.createStoreContext();
+ return fs.listUploadsUnderPrefix(sc, sc.pathToKey(dir));
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java
index 48378ce75dc9..e37717bfa1e3 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java
@@ -91,6 +91,13 @@ public class AbstractS3ACostTest extends AbstractS3ATestBase {
this.keepMarkers = keepMarkers;
}
+ /**
+ * Constructor with markers kept.
+ */
+ public AbstractS3ACostTest() {
+ this(true);
+ }
+
@Override
public Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java
index f7b9ad4f24b4..daa4c2c8483a 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java
@@ -95,22 +95,22 @@ public class ITestS3GuardTool extends AbstractS3GuardToolTestBase {
LOG.info("Exec output=\n{}", output);
}
- private final static String UPLOAD_PREFIX = "test-upload-prefix";
private final static String UPLOAD_NAME = "test-upload";
@Test
public void testUploads() throws Throwable {
S3AFileSystem fs = getFileSystem();
- Path path = path(UPLOAD_PREFIX + "/" + UPLOAD_NAME);
+ Path path = methodPath();
+ Path file = new Path(path, UPLOAD_NAME);
describe("Cleaning up any leftover uploads from previous runs.");
- final String key = fs.pathToKey(path);
+ final String key = fs.pathToKey(file);
try {
// 1. Make sure key doesn't already exist
clearAnyUploads(fs, path);
// 2. Confirm no uploads are listed via API
- assertNoUploadsAt(fs, path.getParent());
+ assertNoUploadsAt(fs, path);
// 3. Confirm no uploads are listed via CLI
describe("Confirming CLI lists nothing.");
@@ -125,8 +125,6 @@ public class ITestS3GuardTool extends AbstractS3GuardToolTestBase {
// 6. Confirm part exists via CLI, direct path and parent path
describe("Confirming CLI lists one part");
assertNumUploads(path, 1);
- assertNumUploads(path.getParent(), 1);
- // 7. Use CLI to delete part, assert it worked
describe("Deleting part via CLI");
assertNumDeleted(fs, path, 1);
@@ -148,22 +146,23 @@ public class ITestS3GuardTool extends AbstractS3GuardToolTestBase {
@Test
public void testUploadListByAge() throws Throwable {
S3AFileSystem fs = getFileSystem();
- Path path = path(UPLOAD_PREFIX + "/" + UPLOAD_NAME);
+ Path path = methodPath();
+ Path file = new Path(path, UPLOAD_NAME);
describe("Cleaning up any leftover uploads from previous runs.");
+
// 1. Make sure key doesn't already exist
clearAnyUploads(fs, path);
// 2. Create a upload part
describe("Uploading single part.");
- final String key = fs.pathToKey(path);
+ final String key = fs.pathToKey(file);
createPartUpload(fs, key, 128, 1);
//try (AuditSpan span = fs.startOperation("multipart", key, null)) {
try {
- // 3. Confirm it exists via API.. may want to wrap with
- // LambdaTestUtils.eventually() ?
+ // 3. Confirm it exists via API
assertEquals("Should be one upload", 1, countUploadsAt(fs, path));
// 4. Confirm part does appear in listing with long age filter
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 13/17: HADOOP-18915. Tune/extend S3A http connection and thread pool settings (#6180)
Posted by ah...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ahmar pushed a commit to branch feature-HADOOP-18073-sdk-v2-upgrade-3.6-aws
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 927d3ce54dc8d659c93c30fc223f74dcfb428db1
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Wed Nov 29 15:12:44 2023 +0000
HADOOP-18915. Tune/extend S3A http connection and thread pool settings (#6180)
Increases existing pool sizes, as with server scale and vector
IO, larger pools are needed
fs.s3a.connection.maximum 200
fs.s3a.threads.max 96
Adds new configuration options for v2 sdk internal timeouts,
both with default of 60s:
fs.s3a.connection.acquisition.timeout
fs.s3a.connection.idle.time
All the pool/timoeut options are covered in performance.md
Moves all timeout/duration options in the s3a FS to taking
temporal units (h, m, s, ms,...); retaining the previous default
unit (normally millisecond)
Adds a minimum duration for most of these, in order to recover from
deployments where a timeout has been set on the assumption the unit
was seconds, not millis.
Uses java.time.Duration throughout the codebase;
retaining the older numeric constants in
org.apache.hadoop.fs.s3a.Constants for backwards compatibility;
these are now deprecated.
Adds new class AWSApiCallTimeoutException to be raised on
sdk-related methods and also gateway timeouts. This is a subclass
of org.apache.hadoop.net.ConnectTimeoutException to support
existing retry logic.
+ reverted default value of fs.s3a.create.performance to false;
inadvertently set to true during testing.
Contributed by Steve Loughran.
---
.../src/main/resources/core-default.xml | 28 +-
.../hadoop/fs/s3a/AWSApiCallTimeoutException.java | 41 +++
.../java/org/apache/hadoop/fs/s3a/Constants.java | 191 +++++++++--
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 20 +-
.../org/apache/hadoop/fs/s3a/S3ARetryPolicy.java | 3 +
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 23 +-
.../apache/hadoop/fs/s3a/impl/AWSClientConfig.java | 351 +++++++++++++++++----
.../hadoop/fs/s3a/impl/ConfigurationHelper.java | 132 ++++++++
.../hadoop/fs/s3a/impl/InternalConstants.java | 3 +
.../src/site/markdown/tools/hadoop-aws/index.md | 103 ++----
.../site/markdown/tools/hadoop-aws/performance.md | 111 +++++--
.../hadoop/fs/s3a/TestS3AExceptionTranslation.java | 53 +++-
.../fs/s3a/impl/ITestConnectionTimeouts.java | 149 +++++++++
.../hadoop/fs/s3a/impl/TestAwsClientConfig.java | 188 +++++++++++
.../fs/s3a/performance/ITestUnbufferDraining.java | 30 +-
15 files changed, 1197 insertions(+), 229 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 1a391c9cf22f..b9b6ec3d8f20 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1332,7 +1332,7 @@
<property>
<name>fs.s3a.connection.maximum</name>
- <value>96</value>
+ <value>200</value>
<description>Controls the maximum number of simultaneous connections to S3.
This must be bigger than the value of fs.s3a.threads.max so as to stop
threads being blocked waiting for new HTTPS connections.
@@ -1410,14 +1410,21 @@
<property>
<name>fs.s3a.connection.establish.timeout</name>
- <value>5000</value>
- <description>Socket connection setup timeout in milliseconds.</description>
+ <value>5s</value>
+ <description>Socket connection setup timeout in milliseconds; this will be retried
+ more than once.</description>
</property>
<property>
<name>fs.s3a.connection.timeout</name>
- <value>200000</value>
- <description>Socket connection timeout in milliseconds.</description>
+ <value>200s</value>
+ <description>Socket connection timeout.</description>
+</property>
+
+<property>
+ <name>fs.s3a.connection.ttl</name>
+ <value>5m</value>
+ <description>Expiry time for any active connection.</description>
</property>
<property>
@@ -1441,14 +1448,14 @@
<property>
<name>fs.s3a.threads.max</name>
- <value>64</value>
+ <value>96</value>
<description>The total number of threads available in the filesystem for data
uploads *or any other queued filesystem operation*.</description>
</property>
<property>
<name>fs.s3a.threads.keepalivetime</name>
- <value>60</value>
+ <value>60s</value>
<description>Number of seconds a thread can be idle before being
terminated.</description>
</property>
@@ -1528,7 +1535,7 @@
<property>
<name>fs.s3a.multipart.purge.age</name>
- <value>86400</value>
+ <value>24h</value>
<description>Minimum age in seconds of multipart uploads to purge
on startup if "fs.s3a.multipart.purge" is true
</description>
@@ -1893,18 +1900,15 @@
<property>
<name>fs.s3a.connection.request.timeout</name>
- <value>0</value>
+ <value>0s</value>
<description>
Time out on HTTP requests to the AWS service; 0 means no timeout.
- Measured in seconds; the usual time suffixes are all supported
Important: this is the maximum duration of any AWS service call,
including upload and copy operations. If non-zero, it must be larger
than the time to upload multi-megabyte blocks to S3 from the client,
and to rename many-GB files. Use with care.
- Values that are larger than Integer.MAX_VALUE milliseconds are
- converged to Integer.MAX_VALUE milliseconds
</description>
</property>
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSApiCallTimeoutException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSApiCallTimeoutException.java
new file mode 100644
index 000000000000..c5a72f018521
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSApiCallTimeoutException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.net.ConnectTimeoutException;
+
+/**
+ * IOException equivalent of an {@code ApiCallTimeoutException}.
+ * Declared as a subclass of {@link ConnectTimeoutException} to allow
+ * for existing code to catch it.
+ */
+public class AWSApiCallTimeoutException extends ConnectTimeoutException {
+
+ /**
+ * Constructor.
+ * @param operation operation in progress
+ * @param cause cause.
+ */
+ public AWSApiCallTimeoutException(
+ final String operation,
+ final Exception cause) {
+ super(operation);
+ initCause(cause);
+ }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 16012871ff48..e602c1b3dc36 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
+import java.time.Duration;
import java.util.concurrent.TimeUnit;
/**
@@ -33,7 +34,14 @@ import java.util.concurrent.TimeUnit;
* as deprecated and simply ignored.
*
* All S3Guard related constants are marked as Deprecated and either ignored (ddb config)
- * or rejected (setting the metastore to anything other than the null store)
+ * or rejected (setting the metastore to anything other than the null store).
+ * <p>
+ * Timeout default values are declared as integers or long values in milliseconds and
+ * occasionally seconds.
+ * There are now {@code Duration} constants for these default values; the original
+ * fields are retained for compatibility, and derive their value from the Duration equivalent.
+ * <p>
+ * New timeout/duration constants do not get the equivalent integer/long fields.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
@@ -144,25 +152,47 @@ public final class Constants {
SimpleAWSCredentialsProvider.NAME;
- // the maximum number of tasks cached if all threads are already uploading
+ /**
+ * The maximum number of tasks queued (other than prefetcher tasks) if all threads are
+ * busy: {@value}.
+ */
public static final String MAX_TOTAL_TASKS = "fs.s3a.max.total.tasks";
+ /**
+ * Default value for {@link #MAX_TOTAL_TASKS}: {@value}.
+ */
public static final int DEFAULT_MAX_TOTAL_TASKS = 32;
- // number of simultaneous connections to s3
+ /**
+ * Number of simultaneous connections to S3: {@value}.
+ */
public static final String MAXIMUM_CONNECTIONS = "fs.s3a.connection.maximum";
- public static final int DEFAULT_MAXIMUM_CONNECTIONS = 96;
+
+ /**
+ * Default value for {@link #MAXIMUM_CONNECTIONS}: {@value}.
+ * Future releases are likely to increase this value.
+ * Keep in sync with the value in {@code core-default.xml}
+ */
+ public static final int DEFAULT_MAXIMUM_CONNECTIONS = 200;
/**
* Configuration option to configure expiration time of
- * s3 http connection from the connection pool in milliseconds: {@value}.
+ * S3 http connection from the connection pool: {@value}.
*/
public static final String CONNECTION_TTL = "fs.s3a.connection.ttl";
/**
- * Default value for {@code CONNECTION_TTL}: {@value}.
+ * Default duration for {@link #CONNECTION_TTL}: 5 minutes.
+ */
+ public static final Duration DEFAULT_CONNECTION_TTL_DURATION =
+ Duration.ofMinutes(5);
+
+ /**
+ * Default value in millis for {@link #CONNECTION_TTL}: 5 minutes.
+ * @deprecated use {@link #DEFAULT_CONNECTION_TTL_DURATION}
*/
- public static final long DEFAULT_CONNECTION_TTL = 5 * 60_000;
+ public static final long DEFAULT_CONNECTION_TTL =
+ DEFAULT_CONNECTION_TTL_DURATION.toMillis();
// connect to s3 over ssl?
public static final String SECURE_CONNECTIONS =
@@ -264,19 +294,111 @@ public final class Constants {
public static final boolean EXPERIMENTAL_AWS_INTERNAL_THROTTLING_DEFAULT =
true;
- // milliseconds until we give up trying to establish a connection to s3
+ /**
+ * This is the minimum operation duration unless programmatically set.
+ * It ensures that even if a configuration has mistaken a millisecond
+ * option for seconds, a viable duration will actually be used.
+ * Value: 15s.
+ */
+ public static final Duration MINIMUM_NETWORK_OPERATION_DURATION = Duration.ofSeconds(15);
+
+ /**
+ * Milliseconds until a connection is established: {@value}.
+ */
public static final String ESTABLISH_TIMEOUT =
"fs.s3a.connection.establish.timeout";
- public static final int DEFAULT_ESTABLISH_TIMEOUT = 5000;
- // milliseconds until we give up on a connection to s3
+ /**
+ * Default TCP/(and TLS?) establish timeout: 30 seconds.
+ */
+ public static final Duration DEFAULT_ESTABLISH_TIMEOUT_DURATION = Duration.ofSeconds(30);
+
+ /**
+ * Default establish timeout in millis: 30 seconds.
+ * @deprecated use {@link #DEFAULT_ESTABLISH_TIMEOUT_DURATION}
+ */
+ public static final int DEFAULT_ESTABLISH_TIMEOUT =
+ (int)DEFAULT_ESTABLISH_TIMEOUT_DURATION.toMillis();
+
+ /**
+ * Milliseconds until we give up on a connection to s3: {@value}.
+ */
public static final String SOCKET_TIMEOUT = "fs.s3a.connection.timeout";
- public static final int DEFAULT_SOCKET_TIMEOUT = 200000;
- // milliseconds until a request is timed-out
+ /**
+ * Default socket timeout: 200 seconds.
+ */
+ public static final Duration DEFAULT_SOCKET_TIMEOUT_DURATION = Duration.ofSeconds(200);
+
+ /**
+ * Default socket timeout: {@link #DEFAULT_SOCKET_TIMEOUT_DURATION}.
+ * @deprecated use {@link #DEFAULT_SOCKET_TIMEOUT_DURATION}
+ */
+ public static final int DEFAULT_SOCKET_TIMEOUT = (int)DEFAULT_SOCKET_TIMEOUT_DURATION.toMillis();
+
+ /**
+ * Time until a request is timed-out: {@value}.
+ * If zero, there is no timeout.
+ */
public static final String REQUEST_TIMEOUT =
"fs.s3a.connection.request.timeout";
- public static final int DEFAULT_REQUEST_TIMEOUT = 0;
+
+ /**
+ * Default duration of a request before it is timed out: Zero.
+ */
+ public static final Duration DEFAULT_REQUEST_TIMEOUT_DURATION = Duration.ZERO;
+
+ /**
+ * Default duration of a request before it is timed out: Zero.
+ * @deprecated use {@link #DEFAULT_REQUEST_TIMEOUT_DURATION}
+ */
+ public static final int DEFAULT_REQUEST_TIMEOUT =
+ (int)DEFAULT_REQUEST_TIMEOUT_DURATION.toMillis();
+
+ /**
+ * Acquisition timeout for connections from the pool:
+ * {@value}.
+ * Default unit is milliseconds for consistency with other options.
+ */
+ public static final String CONNECTION_ACQUISITION_TIMEOUT =
+ "fs.s3a.connection.acquisition.timeout";
+
+ /**
+ * Default acquisition timeout: 60 seconds.
+ */
+ public static final Duration DEFAULT_CONNECTION_ACQUISITION_TIMEOUT_DURATION =
+ Duration.ofSeconds(60);
+
+ /**
+ * Should TCP Keepalive be enabled on the socket?
+ * This adds some network IO, but finds failures faster.
+ * {@value}.
+ */
+ public static final String CONNECTION_KEEPALIVE =
+ "fs.s3a.connection.keepalive";
+
+ /**
+ * Default value of {@link #CONNECTION_KEEPALIVE}: {@value}.
+ */
+ public static final boolean DEFAULT_CONNECTION_KEEPALIVE = false;
+
+ /**
+ * Maximum idle time for connections in the pool: {@value}.
+ * <p>
+ * Too low: overhead of creating connections.
+ * Too high, risk of stale connections and inability to use the
+ * adaptive load balancing of the S3 front end.
+ * <p>
+ * Default unit is milliseconds for consistency with other options.
+ */
+ public static final String CONNECTION_IDLE_TIME =
+ "fs.s3a.connection.idle.time";
+
+ /**
+ * Default idle time: 60 seconds.
+ */
+ public static final Duration DEFAULT_CONNECTION_IDLE_TIME_DURATION =
+ Duration.ofSeconds(60);
// socket send buffer to be used in Amazon client
public static final String SOCKET_SEND_BUFFER = "fs.s3a.socket.send.buffer";
@@ -290,13 +412,34 @@ public final class Constants {
public static final String MAX_PAGING_KEYS = "fs.s3a.paging.maximum";
public static final int DEFAULT_MAX_PAGING_KEYS = 5000;
- // the maximum number of threads to allow in the pool used by TransferManager
+ /**
+ * The maximum number of threads to allow in the pool used by S3A.
+ * Value: {@value}.
+ */
public static final String MAX_THREADS = "fs.s3a.threads.max";
- public static final int DEFAULT_MAX_THREADS = 10;
- // the time an idle thread waits before terminating
+ /**
+ * Default value of {@link #MAX_THREADS}: {@value}.
+ */
+ public static final int DEFAULT_MAX_THREADS = 96;
+
+ /**
+ * The time an idle thread waits before terminating: {@value}.
+ * This is in SECONDS unless the optional unit is given.
+ */
public static final String KEEPALIVE_TIME = "fs.s3a.threads.keepalivetime";
- public static final int DEFAULT_KEEPALIVE_TIME = 60;
+
+ /**
+ * Default value of {@link #KEEPALIVE_TIME}: 60s.
+ */
+ public static final Duration DEFAULT_KEEPALIVE_TIME_DURATION = Duration.ofSeconds(60);
+
+ /**
+ * Default value of {@link #KEEPALIVE_TIME}: 60s.
+ * @deprecated use {@link #DEFAULT_KEEPALIVE_TIME_DURATION}
+ */
+ public static final int DEFAULT_KEEPALIVE_TIME =
+ (int)DEFAULT_KEEPALIVE_TIME_DURATION.getSeconds();
// size of each of or multipart pieces in bytes
public static final String MULTIPART_SIZE = "fs.s3a.multipart.size";
@@ -501,10 +644,17 @@ public final class Constants {
"fs.s3a.multipart.purge";
public static final boolean DEFAULT_PURGE_EXISTING_MULTIPART = false;
- // purge any multipart uploads older than this number of seconds
+ /**
+ * purge any multipart uploads older than this number of seconds.
+ */
public static final String PURGE_EXISTING_MULTIPART_AGE =
"fs.s3a.multipart.purge.age";
- public static final long DEFAULT_PURGE_EXISTING_MULTIPART_AGE = 86400;
+
+ /**
+ * Default Age.
+ */
+ public static final long DEFAULT_PURGE_EXISTING_MULTIPART_AGE =
+ Duration.ofDays(1).getSeconds();
/**
* s3 server-side encryption, see
@@ -1201,8 +1351,7 @@ public final class Constants {
* Default value for create performance in an S3A FS.
* Value {@value}.
*/
- public static final boolean FS_S3A_CREATE_PERFORMANCE_DEFAULT = true;
-
+ public static final boolean FS_S3A_CREATE_PERFORMANCE_DEFAULT = false;
/**
* Capability to indicate that the FS has been instantiated with
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index c7511adf8ab1..cdcb0c9cb148 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -28,6 +28,7 @@ import java.net.URI;
import java.nio.file.AccessDeniedException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
+import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
@@ -122,6 +123,7 @@ import org.apache.hadoop.fs.s3a.impl.AWSCannedACL;
import org.apache.hadoop.fs.s3a.impl.AWSHeaders;
import org.apache.hadoop.fs.s3a.impl.BulkDeleteRetryHandler;
import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy;
+import org.apache.hadoop.fs.s3a.impl.ConfigurationHelper;
import org.apache.hadoop.fs.s3a.impl.ContextAccessors;
import org.apache.hadoop.fs.s3a.impl.CopyFromLocalOperation;
import org.apache.hadoop.fs.s3a.impl.CreateFileBuilder;
@@ -821,8 +823,13 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
}
int totalTasks = intOption(conf,
MAX_TOTAL_TASKS, DEFAULT_MAX_TOTAL_TASKS, 1);
- long keepAliveTime = longOption(conf, KEEPALIVE_TIME,
- DEFAULT_KEEPALIVE_TIME, 0);
+ // keepalive time takes a time suffix; default unit is seconds
+ long keepAliveTime = ConfigurationHelper.getDuration(conf,
+ KEEPALIVE_TIME,
+ Duration.ofSeconds(DEFAULT_KEEPALIVE_TIME),
+ TimeUnit.SECONDS,
+ Duration.ZERO).getSeconds();
+
int numPrefetchThreads = this.prefetchEnabled ? this.prefetchBlockCount : 0;
int activeTasksForBoundedThreadPool = maxThreads;
@@ -1218,12 +1225,15 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
private void initMultipartUploads(Configuration conf) throws IOException {
boolean purgeExistingMultipart = conf.getBoolean(PURGE_EXISTING_MULTIPART,
DEFAULT_PURGE_EXISTING_MULTIPART);
- long purgeExistingMultipartAge = longOption(conf,
- PURGE_EXISTING_MULTIPART_AGE, DEFAULT_PURGE_EXISTING_MULTIPART_AGE, 0);
if (purgeExistingMultipart) {
try {
- abortOutstandingMultipartUploads(purgeExistingMultipartAge);
+ Duration purgeDuration = ConfigurationHelper.getDuration(conf,
+ PURGE_EXISTING_MULTIPART_AGE,
+ Duration.ofSeconds(DEFAULT_PURGE_EXISTING_MULTIPART_AGE),
+ TimeUnit.SECONDS,
+ Duration.ZERO);
+ abortOutstandingMultipartUploads(purgeDuration.getSeconds());
} catch (AccessDeniedException e) {
instrumentation.errorIgnored();
LOG.debug("Failed to purge multipart uploads against {}," +
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
index cc30da4fbe04..9438ac22bdb1 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
@@ -201,6 +201,9 @@ public class S3ARetryPolicy implements RetryPolicy {
// Treated as an immediate failure
policyMap.put(AWSBadRequestException.class, fail);
+ // API call timeout.
+ policyMap.put(AWSApiCallTimeoutException.class, retryAwsClientExceptions);
+
// use specific retry policy for aws client exceptions
// nested IOExceptions will already have been extracted and used
// in this map.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index 155bd8a6105a..94fa2d35fc6f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.fs.s3a;
import software.amazon.awssdk.awscore.exception.AwsServiceException;
import software.amazon.awssdk.core.exception.AbortedException;
+import software.amazon.awssdk.core.exception.ApiCallAttemptTimeoutException;
+import software.amazon.awssdk.core.exception.ApiCallTimeoutException;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.core.retry.RetryUtils;
import software.amazon.awssdk.services.s3.model.S3Exception;
@@ -169,9 +171,23 @@ public final class S3AUtils {
operation,
StringUtils.isNotEmpty(path)? (" on " + path) : "",
exception);
+
+ // timeout issues
+ // ApiCallAttemptTimeoutException: a single HTTP request attempt failed.
+ // ApiCallTimeoutException: a request with any configured retries failed.
+ // The ApiCallTimeoutException exception should be the only one seen in
+ // the S3A code, but for due diligence both are handled and mapped to
+ // our own AWSApiCallTimeoutException.
+ if (exception instanceof ApiCallTimeoutException
+ || exception instanceof ApiCallAttemptTimeoutException) {
+ // An API call to an AWS service timed out.
+ // This is a subclass of ConnectTimeoutException so
+ // all retry logic for that exception is handled without
+ // having to look down the stack for a
+ return new AWSApiCallTimeoutException(message, exception);
+ }
if (!(exception instanceof AwsServiceException)) {
// exceptions raised client-side: connectivity, auth, network problems...
-
Exception innerCause = containsInterruptedException(exception);
if (innerCause != null) {
// interrupted IO, or a socket exception underneath that class
@@ -291,6 +307,11 @@ public final class S3AUtils {
ioe = new AWSServiceThrottledException(message, ase);
break;
+ // gateway timeout
+ case SC_504_GATEWAY_TIMEOUT:
+ ioe = new AWSApiCallTimeoutException(message, ase);
+ break;
+
// internal error
case SC_500_INTERNAL_SERVER_ERROR:
ioe = new AWSStatus500Exception(message, ase);
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java
index 3400daff509b..263562fe8a70 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java
@@ -28,29 +28,37 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption;
+import software.amazon.awssdk.core.retry.RetryMode;
import software.amazon.awssdk.core.retry.RetryPolicy;
import software.amazon.awssdk.http.apache.ApacheHttpClient;
import software.amazon.awssdk.http.apache.ProxyConfiguration;
import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
+import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.s3a.S3AUtils;
import org.apache.hadoop.fs.s3a.auth.SignerFactory;
import org.apache.hadoop.util.VersionInfo;
-import org.apache.http.client.utils.URIBuilder;
+import static org.apache.hadoop.fs.s3a.Constants.CONNECTION_ACQUISITION_TIMEOUT;
import static org.apache.hadoop.fs.s3a.Constants.AWS_SERVICE_IDENTIFIER_S3;
import static org.apache.hadoop.fs.s3a.Constants.AWS_SERVICE_IDENTIFIER_STS;
+import static org.apache.hadoop.fs.s3a.Constants.CONNECTION_IDLE_TIME;
+import static org.apache.hadoop.fs.s3a.Constants.CONNECTION_KEEPALIVE;
import static org.apache.hadoop.fs.s3a.Constants.CONNECTION_TTL;
-import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_CONNECTION_TTL;
-import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_ESTABLISH_TIMEOUT;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_CONNECTION_ACQUISITION_TIMEOUT_DURATION;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_CONNECTION_IDLE_TIME_DURATION;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_CONNECTION_KEEPALIVE;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_CONNECTION_TTL_DURATION;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_ESTABLISH_TIMEOUT_DURATION;
import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_MAXIMUM_CONNECTIONS;
import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_MAX_ERROR_RETRIES;
-import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_REQUEST_TIMEOUT;
-import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_SOCKET_TIMEOUT;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_REQUEST_TIMEOUT_DURATION;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_SOCKET_TIMEOUT_DURATION;
import static org.apache.hadoop.fs.s3a.Constants.ESTABLISH_TIMEOUT;
import static org.apache.hadoop.fs.s3a.Constants.MAXIMUM_CONNECTIONS;
import static org.apache.hadoop.fs.s3a.Constants.MAX_ERROR_RETRIES;
+import static org.apache.hadoop.fs.s3a.Constants.MINIMUM_NETWORK_OPERATION_DURATION;
import static org.apache.hadoop.fs.s3a.Constants.PROXY_DOMAIN;
import static org.apache.hadoop.fs.s3a.Constants.PROXY_HOST;
import static org.apache.hadoop.fs.s3a.Constants.PROXY_PASSWORD;
@@ -64,19 +72,40 @@ import static org.apache.hadoop.fs.s3a.Constants.SIGNING_ALGORITHM_S3;
import static org.apache.hadoop.fs.s3a.Constants.SIGNING_ALGORITHM_STS;
import static org.apache.hadoop.fs.s3a.Constants.SOCKET_TIMEOUT;
import static org.apache.hadoop.fs.s3a.Constants.USER_AGENT_PREFIX;
-import static org.apache.hadoop.fs.s3a.S3AUtils.longOption;
+import static org.apache.hadoop.fs.s3a.impl.ConfigurationHelper.enforceMinimumDuration;
+import static org.apache.hadoop.fs.s3a.impl.ConfigurationHelper.getDuration;
+import static org.apache.hadoop.util.Preconditions.checkArgument;
+
/**
* Methods for configuring the S3 client.
* These methods are used when creating and configuring
- * {@link software.amazon.awssdk.services.s3.S3Client} which communicates with the S3 service.
+ * the HTTP clients which communicate with the S3 service.
+ * <p>
+ * See {@code software.amazon.awssdk.http.SdkHttpConfigurationOption}
+ * for the default values.
*/
public final class AWSClientConfig {
+
private static final Logger LOG = LoggerFactory.getLogger(AWSClientConfig.class);
+ /**
+ * The minimum operation duration.
+ */
+ private static Duration minimumOperationDuration = MINIMUM_NETWORK_OPERATION_DURATION;
+
+
private AWSClientConfig() {
}
+ /**
+ * Create the config for a given service...the service identifier is used
+ * to determine signature implementation.
+ * @param conf configuration
+ * @param awsServiceIdentifier service
+ * @return the builder inited with signer, timeouts and UA.
+ * @throws IOException failure.
+ */
public static ClientOverrideConfiguration.Builder createClientConfigBuilder(Configuration conf,
String awsServiceIdentifier) throws IOException {
ClientOverrideConfiguration.Builder overrideConfigBuilder =
@@ -99,7 +128,10 @@ public final class AWSClientConfig {
}
/**
- * Configures the http client.
+ * Create and configure the http client-based connector with timeouts for:
+ * connection acquisition, max idle, timeout, TTL, socket and keepalive.
+ * SSL channel mode is set up via
+ * {@link NetworkBinding#bindSSLChannelMode(Configuration, ApacheHttpClient.Builder)}.
*
* @param conf The Hadoop configuration
* @return Http client builder
@@ -107,24 +139,17 @@ public final class AWSClientConfig {
*/
public static ApacheHttpClient.Builder createHttpClientBuilder(Configuration conf)
throws IOException {
+ final ConnectionSettings conn = createConnectionSettings(conf);
ApacheHttpClient.Builder httpClientBuilder =
- ApacheHttpClient.builder();
-
- httpClientBuilder.maxConnections(S3AUtils.intOption(conf, MAXIMUM_CONNECTIONS,
- DEFAULT_MAXIMUM_CONNECTIONS, 1));
-
- int connectionEstablishTimeout =
- S3AUtils.intOption(conf, ESTABLISH_TIMEOUT, DEFAULT_ESTABLISH_TIMEOUT, 0);
- int socketTimeout = S3AUtils.intOption(conf, SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT, 0);
-
- httpClientBuilder.connectionTimeout(Duration.ofMillis(connectionEstablishTimeout));
- httpClientBuilder.socketTimeout(Duration.ofMillis(socketTimeout));
-
- // set the connection TTL irrespective of whether the connection is in use or not.
- // this can balance requests over different S3 servers, and avoid failed
- // connections. See HADOOP-18845.
- long ttl = longOption(conf, CONNECTION_TTL, DEFAULT_CONNECTION_TTL, -1);
- httpClientBuilder.connectionTimeToLive(Duration.ofMillis(ttl));
+ ApacheHttpClient.builder()
+ .connectionAcquisitionTimeout(conn.getAcquisitionTimeout())
+ .connectionMaxIdleTime(conn.getMaxIdleTime())
+ .connectionTimeout(conn.getEstablishTimeout())
+ .connectionTimeToLive(conn.getConnectionTTL())
+ .maxConnections(conn.getMaxConnections())
+ .socketTimeout(conn.getSocketTimeout())
+ .tcpKeepAlive(conn.isKeepAlive())
+ .useIdleConnectionReaper(true); // true by default in the SDK
NetworkBinding.bindSSLChannelMode(conf, httpClientBuilder);
@@ -132,31 +157,26 @@ public final class AWSClientConfig {
}
/**
- * Configures the async http client.
- *
+ * Create and configure the async http client with timeouts for:
+ * connection acquisition, max idle, timeout, TTL, socket and keepalive.
+ * This is netty based and does not allow for the SSL channel mode to be set.
* @param conf The Hadoop configuration
- * @return Http client builder
+ * @return Async Http client builder
*/
public static NettyNioAsyncHttpClient.Builder createAsyncHttpClientBuilder(Configuration conf) {
- NettyNioAsyncHttpClient.Builder httpClientBuilder =
- NettyNioAsyncHttpClient.builder();
+ final ConnectionSettings conn = createConnectionSettings(conf);
- httpClientBuilder.maxConcurrency(S3AUtils.intOption(conf, MAXIMUM_CONNECTIONS,
- DEFAULT_MAXIMUM_CONNECTIONS, 1));
-
- int connectionEstablishTimeout =
- S3AUtils.intOption(conf, ESTABLISH_TIMEOUT, DEFAULT_ESTABLISH_TIMEOUT, 0);
- int socketTimeout = S3AUtils.intOption(conf, SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT, 0);
-
- httpClientBuilder.connectionTimeout(Duration.ofMillis(connectionEstablishTimeout));
- httpClientBuilder.readTimeout(Duration.ofMillis(socketTimeout));
- httpClientBuilder.writeTimeout(Duration.ofMillis(socketTimeout));
-
- // set the connection TTL irrespective of whether the connection is in use or not.
- // this can balance requests over different S3 servers, and avoid failed
- // connections. See HADOOP-18845.
- long ttl = longOption(conf, CONNECTION_TTL, DEFAULT_CONNECTION_TTL, -1);
- httpClientBuilder.connectionTimeToLive(Duration.ofMillis(ttl));
+ NettyNioAsyncHttpClient.Builder httpClientBuilder =
+ NettyNioAsyncHttpClient.builder()
+ .connectionAcquisitionTimeout(conn.getAcquisitionTimeout())
+ .connectionMaxIdleTime(conn.getMaxIdleTime())
+ .connectionTimeout(conn.getEstablishTimeout())
+ .connectionTimeToLive(conn.getConnectionTTL())
+ .maxConcurrency(conn.getMaxConnections())
+ .readTimeout(conn.getSocketTimeout())
+ .tcpKeepAlive(conn.isKeepAlive())
+ .useIdleConnectionReaper(true) // true by default in the SDK
+ .writeTimeout(conn.getSocketTimeout());
// TODO: Don't think you can set a socket factory for the netty client.
// NetworkBinding.bindSSLChannelMode(conf, awsConf);
@@ -166,13 +186,19 @@ public final class AWSClientConfig {
/**
* Configures the retry policy.
+ * Retry policy is {@code RetryMode.ADAPTIVE}, which
+ * "dynamically limits the rate of AWS requests to maximize success rate",
+ * possibly at the expense of latency.
+ * Based on the ABFS experience, it is better to limit the rate requests are
+ * made rather than have to resort to exponential backoff after failures come
+ * in -especially as that backoff is per http connection.
*
* @param conf The Hadoop configuration
* @return Retry policy builder
*/
public static RetryPolicy.Builder createRetryPolicyBuilder(Configuration conf) {
- RetryPolicy.Builder retryPolicyBuilder = RetryPolicy.builder();
+ RetryPolicy.Builder retryPolicyBuilder = RetryPolicy.builder(RetryMode.ADAPTIVE);
retryPolicyBuilder.numRetries(S3AUtils.intOption(conf, MAX_ERROR_RETRIES,
DEFAULT_MAX_ERROR_RETRIES, 0));
@@ -236,11 +262,14 @@ public final class AWSClientConfig {
}
return proxyConfigBuilder.build();
+
}
/**
* Configures the proxy for the async http client.
- *
+ * <p>
+ * Although this is netty specific, it is part of the AWS SDK public API
+ * and not any shaded netty internal class.
* @param conf The Hadoop configuration
* @param bucket Optional bucket to use to look up per-bucket proxy secrets
* @return Proxy configuration
@@ -248,7 +277,7 @@ public final class AWSClientConfig {
*/
public static software.amazon.awssdk.http.nio.netty.ProxyConfiguration
createAsyncProxyConfiguration(Configuration conf,
- String bucket) throws IOException {
+ String bucket) throws IOException {
software.amazon.awssdk.http.nio.netty.ProxyConfiguration.Builder proxyConfigBuilder =
software.amazon.awssdk.http.nio.netty.ProxyConfiguration.builder();
@@ -307,7 +336,7 @@ public final class AWSClientConfig {
return proxyConfigBuilder.build();
}
- /***
+ /**
* Builds a URI, throws an IllegalArgumentException in case of errors.
*
* @param host proxy host
@@ -316,7 +345,7 @@ public final class AWSClientConfig {
*/
private static URI buildURI(String scheme, String host, int port) {
try {
- return new URIBuilder().setScheme(scheme).setHost(host).setPort(port).build();
+ return new URI(scheme, null, host, port, null, null, null);
} catch (URISyntaxException e) {
String msg =
"Proxy error: incorrect " + PROXY_HOST + " or " + PROXY_PORT;
@@ -346,6 +375,13 @@ public final class AWSClientConfig {
clientConfig.putAdvancedOption(SdkAdvancedClientOption.USER_AGENT_PREFIX, userAgent);
}
+ /**
+ * Initializes the signer override for the given service.
+ * @param conf hadoop configuration
+ * @param clientConfig client configuration to update
+ * @param awsServiceIdentifier service name
+ * @throws IOException failure.
+ */
private static void initSigner(Configuration conf,
ClientOverrideConfiguration.Builder clientConfig, String awsServiceIdentifier)
throws IOException {
@@ -371,24 +407,219 @@ public final class AWSClientConfig {
}
/**
- * Configures request timeout.
+ * Configures request timeout in the client configuration.
+ * This is independent of the timeouts set in the sync and async HTTP clients;
+ * the same method
*
* @param conf Hadoop configuration
* @param clientConfig AWS SDK configuration to update
*/
private static void initRequestTimeout(Configuration conf,
ClientOverrideConfiguration.Builder clientConfig) {
- long requestTimeoutMillis = conf.getTimeDuration(REQUEST_TIMEOUT,
- DEFAULT_REQUEST_TIMEOUT, TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
+ // Get the connection settings
+ final Duration callTimeout = createApiConnectionSettings(conf).getApiCallTimeout();
- if (requestTimeoutMillis > Integer.MAX_VALUE) {
- LOG.debug("Request timeout is too high({} ms). Setting to {} ms instead",
- requestTimeoutMillis, Integer.MAX_VALUE);
- requestTimeoutMillis = Integer.MAX_VALUE;
+ if (callTimeout.toMillis() > 0) {
+ clientConfig.apiCallAttemptTimeout(callTimeout);
+ clientConfig.apiCallTimeout(callTimeout);
}
+ }
- if(requestTimeoutMillis > 0) {
- clientConfig.apiCallAttemptTimeout(Duration.ofMillis(requestTimeoutMillis));
+ /**
+ * Reset the minimum operation duration to the default.
+ * For test use only; Logs at INFO.
+ * <p>
+ * This MUST be called in test teardown in any test suite which
+ * called {@link #setMinimumOperationDuration(Duration)}.
+ */
+ @VisibleForTesting
+ public static void resetMinimumOperationDuration() {
+ setMinimumOperationDuration(MINIMUM_NETWORK_OPERATION_DURATION);
+ }
+
+ /**
+ * Set the minimum operation duration.
+ * This is for testing and will log at info; does require a non-negative duration.
+ * <p>
+ * Test suites must call {@link #resetMinimumOperationDuration()} in their teardown
+ * to avoid breaking subsequent tests in the same process.
+ * @param duration non-negative duration
+ * @throws IllegalArgumentException if the duration is negative.
+ */
+ @VisibleForTesting
+ public static void setMinimumOperationDuration(Duration duration) {
+ LOG.info("Setting minimum operation duration to {}ms", duration.toMillis());
+ checkArgument(duration.compareTo(Duration.ZERO) >= 0,
+ "Duration must be positive: %sms", duration.toMillis());
+ minimumOperationDuration = duration;
+ }
+
+ /**
+ * Get the current minimum operation duration.
+ * @return current duration.
+ */
+ public static Duration getMinimumOperationDuration() {
+ return minimumOperationDuration;
+ }
+
+ /**
+ * Settings for the AWS client, rather than the http client.
+ */
+ static class ClientSettings {
+ private final Duration apiCallTimeout;
+
+ private ClientSettings(final Duration apiCallTimeout) {
+ this.apiCallTimeout = apiCallTimeout;
+ }
+
+ Duration getApiCallTimeout() {
+ return apiCallTimeout;
+ }
+
+ @Override
+ public String toString() {
+ return "ClientSettings{" +
+ "apiCallTimeout=" + apiCallTimeout +
+ '}';
}
}
+
+ /**
+ * All the connection settings, wrapped as a class for use by
+ * both the sync and async client.
+ */
+ static class ConnectionSettings {
+ private final int maxConnections;
+ private final boolean keepAlive;
+ private final Duration acquisitionTimeout;
+ private final Duration connectionTTL;
+ private final Duration establishTimeout;
+ private final Duration maxIdleTime;
+ private final Duration socketTimeout;
+
+ private ConnectionSettings(
+ final int maxConnections,
+ final boolean keepAlive,
+ final Duration acquisitionTimeout,
+ final Duration connectionTTL,
+ final Duration establishTimeout,
+ final Duration maxIdleTime,
+ final Duration socketTimeout) {
+ this.maxConnections = maxConnections;
+ this.keepAlive = keepAlive;
+ this.acquisitionTimeout = acquisitionTimeout;
+ this.connectionTTL = connectionTTL;
+ this.establishTimeout = establishTimeout;
+ this.maxIdleTime = maxIdleTime;
+ this.socketTimeout = socketTimeout;
+ }
+
+ int getMaxConnections() {
+ return maxConnections;
+ }
+
+ boolean isKeepAlive() {
+ return keepAlive;
+ }
+
+ Duration getAcquisitionTimeout() {
+ return acquisitionTimeout;
+ }
+
+ Duration getConnectionTTL() {
+ return connectionTTL;
+ }
+
+ Duration getEstablishTimeout() {
+ return establishTimeout;
+ }
+
+ Duration getMaxIdleTime() {
+ return maxIdleTime;
+ }
+
+ Duration getSocketTimeout() {
+ return socketTimeout;
+ }
+
+ @Override
+ public String toString() {
+ return "ConnectionSettings{" +
+ "maxConnections=" + maxConnections +
+ ", keepAlive=" + keepAlive +
+ ", acquisitionTimeout=" + acquisitionTimeout +
+ ", connectionTTL=" + connectionTTL +
+ ", establishTimeout=" + establishTimeout +
+ ", maxIdleTime=" + maxIdleTime +
+ ", socketTimeout=" + socketTimeout +
+ '}';
+ }
+ }
+
+ /**
+ * Build a client settings object.
+ * @param conf configuration to evaluate
+ * @return connection settings.
+ */
+ static ClientSettings createApiConnectionSettings(Configuration conf) {
+
+ Duration apiCallTimeout = getDuration(conf, REQUEST_TIMEOUT,
+ DEFAULT_REQUEST_TIMEOUT_DURATION, TimeUnit.MILLISECONDS, Duration.ZERO);
+
+ // if the API call timeout is set, it must be at least the minimum duration
+ if (apiCallTimeout.compareTo(Duration.ZERO) > 0) {
+ apiCallTimeout = enforceMinimumDuration(REQUEST_TIMEOUT,
+ apiCallTimeout, minimumOperationDuration);
+ }
+ return new ClientSettings(apiCallTimeout);
+ }
+
+ /**
+ * Build the HTTP connection settings object from the configuration.
+ * All settings are calculated, including the api call timeout.
+ * @param conf configuration to evaluate
+ * @return connection settings.
+ */
+ static ConnectionSettings createConnectionSettings(Configuration conf) {
+
+ int maxConnections = S3AUtils.intOption(conf, MAXIMUM_CONNECTIONS,
+ DEFAULT_MAXIMUM_CONNECTIONS, 1);
+
+ final boolean keepAlive = conf.getBoolean(CONNECTION_KEEPALIVE,
+ DEFAULT_CONNECTION_KEEPALIVE);
+
+ // time to acquire a connection from the pool
+ Duration acquisitionTimeout = getDuration(conf, CONNECTION_ACQUISITION_TIMEOUT,
+ DEFAULT_CONNECTION_ACQUISITION_TIMEOUT_DURATION, TimeUnit.MILLISECONDS,
+ minimumOperationDuration);
+
+ // set the connection TTL irrespective of whether the connection is in use or not.
+ // this can balance requests over different S3 servers, and avoid failed
+ // connections. See HADOOP-18845.
+ Duration connectionTTL = getDuration(conf, CONNECTION_TTL,
+ DEFAULT_CONNECTION_TTL_DURATION, TimeUnit.MILLISECONDS,
+ null);
+
+ Duration establishTimeout = getDuration(conf, ESTABLISH_TIMEOUT,
+ DEFAULT_ESTABLISH_TIMEOUT_DURATION, TimeUnit.MILLISECONDS,
+ minimumOperationDuration);
+
+ // limit on the time a connection can be idle in the pool
+ Duration maxIdleTime = getDuration(conf, CONNECTION_IDLE_TIME,
+ DEFAULT_CONNECTION_IDLE_TIME_DURATION, TimeUnit.MILLISECONDS, Duration.ZERO);
+
+ Duration socketTimeout = getDuration(conf, SOCKET_TIMEOUT,
+ DEFAULT_SOCKET_TIMEOUT_DURATION, TimeUnit.MILLISECONDS,
+ minimumOperationDuration);
+
+ return new ConnectionSettings(
+ maxConnections,
+ keepAlive,
+ acquisitionTimeout,
+ connectionTTL,
+ establishTimeout,
+ maxIdleTime,
+ socketTimeout);
+ }
+
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ConfigurationHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ConfigurationHelper.java
new file mode 100644
index 000000000000..8acd20a82155
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ConfigurationHelper.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+import java.time.Duration;
+import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.store.LogExactlyOnce;
+
+/**
+ * Helper class for configuration; where methods related to extracting
+ * configuration should go instead of {@code S3AUtils}.
+ */
+@InterfaceAudience.Private
+public final class ConfigurationHelper {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ConfigurationHelper.class);
+
+ /** Log to warn of range issues on a timeout. */
+ private static final LogExactlyOnce DURATION_WARN_LOG = new LogExactlyOnce(LOG);
+
+ private ConfigurationHelper() {
+ }
+
+ /**
+ * Get a duration. This may be negative; callers must check or set the minimum to zero.
+ * If the config option is greater than {@code Integer.MAX_VALUE} milliseconds,
+ * it is set to that max.
+ * If {@code minimumDuration} is set, and the value is less than that, then
+ * the minimum is used.
+ * Logs the value for diagnostics.
+ * @param conf config
+ * @param name option name
+ * @param defaultDuration default duration
+ * @param defaultUnit default unit on the config option if not declared.
+ * @param minimumDuration optional minimum duration;
+ * @return duration. may be negative.
+ */
+ public static Duration getDuration(
+ final Configuration conf,
+ final String name,
+ final Duration defaultDuration,
+ final TimeUnit defaultUnit,
+ @Nullable final Duration minimumDuration) {
+ long timeMillis = conf.getTimeDuration(name,
+ defaultDuration.toMillis(), defaultUnit, TimeUnit.MILLISECONDS);
+
+ if (timeMillis > Integer.MAX_VALUE) {
+ DURATION_WARN_LOG.warn("Option {} is too high({} ms). Setting to {} ms instead",
+ name, timeMillis, Integer.MAX_VALUE);
+ LOG.debug("Option {} is too high({} ms). Setting to {} ms instead",
+ name, timeMillis, Integer.MAX_VALUE);
+ timeMillis = Integer.MAX_VALUE;
+ }
+
+ Duration duration = enforceMinimumDuration(name, Duration.ofMillis(timeMillis),
+ minimumDuration);
+ LOG.debug("Duration of {} = {}", name, duration);
+ return duration;
+ }
+
+ /**
+ * Set a duration as a time in seconds, with the suffix "s" added.
+ * @param conf configuration to update.
+ * @param name option name
+ * @param duration duration
+ */
+ public static void setDurationAsSeconds(
+ final Configuration conf,
+ final String name,
+ final Duration duration) {
+ conf.set(name, duration.getSeconds() + "s");
+ }
+
+ /**
+ * Set a duration as a time in milliseconds, with the suffix "ms" added.
+ * @param conf configuration to update.
+ * @param name option name
+ * @param duration duration
+ */
+ public static void setDurationAsMillis(
+ final Configuration conf,
+ final String name,
+ final Duration duration) {
+ conf.set(name, duration.toMillis()+ "ms");
+ }
+
+ /**
+ * Enforce a minimum duration of a configuration option, if the supplied
+ * value is non-null.
+ * @param name option name
+ * @param duration duration to check
+ * @param minimumDuration minimum duration; may be null
+ * @return a duration which, if the minimum duration is set, is at least that value.
+ */
+ public static Duration enforceMinimumDuration(
+ final String name,
+ final Duration duration,
+ @Nullable final Duration minimumDuration) {
+
+ if (minimumDuration != null && duration.compareTo(minimumDuration) < 0) {
+ String message = String.format("Option %s is too low (%,d ms). Setting to %,d ms instead",
+ name, duration.toMillis(), minimumDuration.toMillis());
+ DURATION_WARN_LOG.warn(message);
+ LOG.debug(message);
+ return minimumDuration;
+ }
+ return duration;
+ }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java
index 3409bd3cc79e..8b29313faf2d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java
@@ -172,6 +172,9 @@ public final class InternalConstants {
/** 503 status code: Service Unavailable. on AWS S3: throttle response. */
public static final int SC_503_SERVICE_UNAVAILABLE = 503;
+ /** 504 Gateway Timeout. AWS SDK considers retryable. */
+ public static final int SC_504_GATEWAY_TIMEOUT = 504;
+
/** Name of the log for throttling events. Value: {@value}. */
public static final String THROTTLE_LOG_NAME =
"org.apache.hadoop.fs.s3a.throttled";
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 076ea0fd5b8a..ed9cfddf0fe2 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -703,8 +703,12 @@ to allow different buckets to override the shared settings. This is commonly
used to change the endpoint, encryption and authentication mechanisms of buckets.
and various minor options.
-Here are the S3A properties for use in production; some testing-related
-options are covered in [Testing](./testing.md).
+Here are some the S3A properties for use in production.
+
+* See [Performance](./performance.html) for performance related settings including
+ thread and network pool options.
+* Testing-related options are covered in [Testing](./testing.md).
+
```xml
@@ -830,16 +834,6 @@ options are covered in [Testing](./testing.md).
</description>
</property>
-<property>
- <name>fs.s3a.connection.maximum</name>
- <value>96</value>
- <description>Controls the maximum number of simultaneous connections to S3.
- This must be bigger than the value of fs.s3a.threads.max so as to stop
- threads being blocked waiting for new HTTPS connections.
- Why not equal? The AWS SDK transfer manager also uses these connections.
- </description>
-</property>
-
<property>
<name>fs.s3a.connection.ssl.enabled</name>
<value>true</value>
@@ -908,18 +902,6 @@ options are covered in [Testing](./testing.md).
</description>
</property>
-<property>
- <name>fs.s3a.connection.establish.timeout</name>
- <value>5000</value>
- <description>Socket connection setup timeout in milliseconds.</description>
-</property>
-
-<property>
- <name>fs.s3a.connection.timeout</name>
- <value>200000</value>
- <description>Socket connection timeout in milliseconds.</description>
-</property>
-
<property>
<name>fs.s3a.socket.send.buffer</name>
<value>8192</value>
@@ -939,43 +921,6 @@ options are covered in [Testing](./testing.md).
directory listings at a time.</description>
</property>
-<property>
- <name>fs.s3a.threads.max</name>
- <value>64</value>
- <description>The total number of threads available in the filesystem for data
- uploads *or any other queued filesystem operation*.</description>
-</property>
-
-<property>
- <name>fs.s3a.threads.keepalivetime</name>
- <value>60</value>
- <description>Number of seconds a thread can be idle before being
- terminated.</description>
-</property>
-
-<property>
- <name>fs.s3a.max.total.tasks</name>
- <value>32</value>
- <description>The number of operations which can be queued for execution.
- This is in addition to the number of active threads in fs.s3a.threads.max.
- </description>
-</property>
-
-<property>
- <name>fs.s3a.executor.capacity</name>
- <value>16</value>
- <description>The maximum number of submitted tasks which is a single
- operation (e.g. rename(), delete()) may submit simultaneously for
- execution -excluding the IO-heavy block uploads, whose capacity
- is set in "fs.s3a.fast.upload.active.blocks"
-
- All tasks are submitted to the shared thread pool whose size is
- set in "fs.s3a.threads.max"; the value of capacity should be less than that
- of the thread pool itself, as the goal is to stop a single operation
- from overloading that thread pool.
- </description>
-</property>
-
<property>
<name>fs.s3a.multipart.size</name>
<value>64M</value>
@@ -2232,43 +2177,33 @@ from VMs running on EC2.
</description>
</property>
-<property>
- <name>fs.s3a.threads.max</name>
- <value>10</value>
- <description>The total number of threads available in the filesystem for data
- uploads *or any other queued filesystem operation*.</description>
-</property>
-
-<property>
- <name>fs.s3a.max.total.tasks</name>
- <value>5</value>
- <description>The number of operations which can be queued for execution</description>
-</property>
-
-<property>
- <name>fs.s3a.threads.keepalivetime</name>
- <value>60</value>
- <description>Number of seconds a thread can be idle before being
- terminated.</description>
-</property>
```
### <a name="multipart_purge"></a>Cleaning up after partial Upload Failures
-There are two mechanisms for cleaning up after leftover multipart
+There are four mechanisms for cleaning up after leftover multipart
uploads:
+- AWS Lifecycle rules. This is the simplest and SHOULD be used unless there
+ are are good reasons, such as the store not supporting lifecycle rules.
- Hadoop s3guard CLI commands for listing and deleting uploads by their
age. Documented in the [S3Guard](./s3guard.html) section.
+- Setting `fs.s3a.directory.operations.purge.uploads` to `true` for automatic
+ scan and delete during directory rename and delete
- The configuration parameter `fs.s3a.multipart.purge`, covered below.
If a large stream write operation is interrupted, there may be
intermediate partitions uploaded to S3 —data which will be billed for.
+If an S3A committer job is halted partway through, again, there may be
+many incomplete multipart uploads in the output directory.
These charges can be reduced by enabling `fs.s3a.multipart.purge`,
-and setting a purge time in seconds, such as 86400 seconds —24 hours.
+and setting a purge time in seconds, such as 24 hours.
When an S3A FileSystem instance is instantiated with the purge time greater
than zero, it will, on startup, delete all outstanding partition requests
-older than this time.
+older than this time. However, this makes filesystem instantiate slow, especially
+against very large buckets, as a full scan is made.
+
+Consider avoiding this in future.
```xml
<property>
@@ -2280,7 +2215,7 @@ older than this time.
<property>
<name>fs.s3a.multipart.purge.age</name>
- <value>86400</value>
+ <value>24h</value>
<description>Minimum age in seconds of multipart uploads to purge</description>
</property>
```
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md
index 88e6e8a0b211..e1d334bbfc43 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md
@@ -196,40 +196,93 @@ Fix: Use one of the dedicated [S3A Committers](committers.md).
## <a name="tuning"></a> Options to Tune
-### <a name="pooling"></a> Thread and connection pool sizes.
+### <a name="pooling"></a> Thread and connection pool settings.
Each S3A client interacting with a single bucket, as a single user, has its
-own dedicated pool of open HTTP 1.1 connections alongside a pool of threads used
-for upload and copy operations.
+own dedicated pool of open HTTP connections alongside a pool of threads used
+for background/parallel operations in addition to the worker threads of the
+actual application.
+
The default pool sizes are intended to strike a balance between performance
and memory/thread use.
You can have a larger pool of (reused) HTTP connections and threads
-for parallel IO (especially uploads) by setting the properties
+for parallel IO (especially uploads, prefetching and vector reads) by setting the appropriate
+properties. Note: S3A Connectors have their own thread pools for job commit, but
+everything uses the same HTTP connection pool.
+| Property | Default | Meaning |
+|--------------------------------|---------|------------------------------------------------------------------|
+| `fs.s3a.threads.max` | `96` | Threads in the thread pool |
+| `fs.s3a.threads.keepalivetime` | `60s` | Expiry time for idle threads in the thread pool |
+| `fs.s3a.executor.capacity` | `16` | Maximum threads for any single operation |
+| `fs.s3a.max.total.tasks` | `16` | Extra tasks which can be queued excluding prefetching operations |
-| property | meaning | default |
-|----------|---------|---------|
-| `fs.s3a.threads.max`| Threads in the AWS transfer manager| 10 |
-| `fs.s3a.connection.maximum`| Maximum number of HTTP connections | 10|
-We recommend using larger values for processes which perform
-a lot of IO: `DistCp`, Spark Workers and similar.
+Network timeout options can be tuned to make the client fail faster *or* retry more.
+The choice is yours. Generally recovery is better, but sometimes fail-fast is more useful.
-```xml
-<property>
- <name>fs.s3a.threads.max</name>
- <value>20</value>
-</property>
-<property>
- <name>fs.s3a.connection.maximum</name>
- <value>20</value>
-</property>
-```
-Be aware, however, that processes which perform many parallel queries
-may consume large amounts of resources if each query is working with
-a different set of s3 buckets, or are acting on behalf of different users.
+| Property | Default | V2 | Meaning |
+|-----------------------------------------|---------|:----|-------------------------------------------------------|
+| `fs.s3a.connection.maximum` | `200` | | Connection pool size |
+| `fs.s3a.connection.keepalive` | `false` | `*` | Use TCP keepalive on open channels |
+| `fs.s3a.connection.acquisition.timeout` | `60s` | `*` | Timeout for waiting for a connection from the pool. |
+| `fs.s3a.connection.establish.timeout` | `30s` | | Time to establish the TCP/TLS connection |
+| `fs.s3a.connection.idle.time` | `60s` | `*` | Maximum time for idle HTTP connections in the pool |
+| `fs.s3a.connection.request.timeout` | `0` | | If greater than zero, maximum duration of any request |
+| `fs.s3a.connection.timeout` | `200s` | | Timeout for socket problems on a TCP channel |
+| `fs.s3a.connection.ttl` | `5m` | | Lifetime of HTTP connections from the pool |
+
+
+Units:
+1. The default unit for all these options except for `fs.s3a.threads.keepalivetime` is milliseconds, unless a time suffix is declared.
+2. Versions of Hadoop built with the AWS V1 SDK *only* support milliseconds rather than suffix values.
+ If configurations are intended to apply across hadoop releases, you MUST use milliseconds without a suffix.
+3. `fs.s3a.threads.keepalivetime` has a default unit of seconds on all hadoop releases.
+4. Options flagged as "V2" are new with the AWS V2 SDK; they are ignored on V1 releases.
+
+---
+
+There are some hard tuning decisions related to pool size and expiry.
+As servers add more cores and services add many more worker threads, a larger pool size is more and more important:
+the default values in `core-default.xml` have been slowly increased over time but should be treated as
+"the best", simply what is considered a good starting case.
+With Vectored IO adding multiple GET requests per Spark/Hive worker thread,
+and stream prefetching performing background block prefetch, larger pool and thread sizes are even more important.
+
+In large hive deployments, thread and connection pools of thousands have been known to have been set.
+
+Small pool: small value in `fs.s3a.connection.maximum`.
+* Keeps network/memory cost of having many S3A instances in the same process low.
+* But: limit on how many connections can be open at at a time.
+
+* Large Pool. More HTTP connections can be created and kept, but cost of keeping network connections increases
+unless idle time is reduced through `fs.s3a.connection.idle.time`.
+
+If exceptions are raised with about timeouts acquiring connections from the pool, this can be a symptom of
+* Heavy load. Increase pool size and acquisition timeout `fs.s3a.connection.acquisition.timeout`
+* Process failing to close open input streams from the S3 store.
+ Fix: Find uses of `open()`/`openFile()` and make sure that the streams are being `close()d`
+
+*Retirement of HTTP Connections.*
+
+Connections are retired from the pool by `fs.s3a.connection.idle.time`, the maximum time for idle connections,
+and `fs.s3a.connection.ttl`, the maximum life of any connection in the pool, even if it repeatedly reused.
+
+Limiting idle time saves on network connections, at the cost of requiring new connections on subsequent S3 operations.
+
+Limiting connection TTL is useful to spread across load balancers and recover from some network
+connection problems, including those caused by proxies.
+
+*Request timeout*: `fs.s3a.connection.request.timeout`
+
+If set, this sets an upper limit on any non-streaming API call (i.e. everything but `GET`).
+
+A timeout is good to detect and recover from failures.
+However, it also sets a limit on the duration of a POST/PUT of data
+-which, if after a timeout, will only be repeated, ultimately to failure.
+
### For large data uploads, tune the block size: `fs.s3a.block.size`
@@ -327,18 +380,6 @@ efficient in terms of HTTP connection use, and reduce the IOP rate against
the S3 bucket/shard.
```xml
-<property>
- <name>fs.s3a.threads.max</name>
- <value>20</value>
-</property>
-
-<property>
- <name>fs.s3a.connection.maximum</name>
- <value>30</value>
- <descriptiom>
- Make greater than both fs.s3a.threads.max and -numListstatusThreads
- </descriptiom>
-</property>
<property>
<name>fs.s3a.experimental.input.fadvise</name>
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AExceptionTranslation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AExceptionTranslation.java
index c4d9ca7bcc04..b26ca6889bd1 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AExceptionTranslation.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AExceptionTranslation.java
@@ -24,6 +24,7 @@ import static org.apache.hadoop.fs.s3a.S3ATestUtils.verifyExceptionClass;
import static org.apache.hadoop.fs.s3a.S3AUtils.*;
import static org.apache.hadoop.fs.s3a.audit.AuditIntegration.maybeTranslateAuditException;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.*;
+import static org.apache.hadoop.test.LambdaTestUtils.verifyCause;
import static org.junit.Assert.*;
import java.io.EOFException;
@@ -37,6 +38,8 @@ import java.util.function.Consumer;
import org.assertj.core.api.Assertions;
import software.amazon.awssdk.awscore.exception.AwsErrorDetails;
import software.amazon.awssdk.awscore.exception.AwsServiceException;
+import software.amazon.awssdk.core.exception.ApiCallAttemptTimeoutException;
+import software.amazon.awssdk.core.exception.ApiCallTimeoutException;
import software.amazon.awssdk.core.exception.SdkClientException;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.http.SdkHttpResponse;
@@ -44,11 +47,12 @@ import software.amazon.awssdk.services.s3.model.S3Exception;
import org.junit.Test;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.s3a.api.UnsupportedRequestException;
import org.apache.hadoop.fs.s3a.audit.AuditFailureException;
import org.apache.hadoop.fs.s3a.audit.AuditOperationRejectedException;
import org.apache.hadoop.fs.s3a.impl.ErrorTranslation;
-
+import org.apache.hadoop.io.retry.RetryPolicy;
import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
@@ -330,4 +334,51 @@ public class TestS3AExceptionTranslation {
.isNull();
}
+ /**
+ * 504 gateway timeout is translated to a {@link AWSApiCallTimeoutException}.
+ */
+ @Test
+ public void test504ToTimeout() throws Throwable {
+ AWSApiCallTimeoutException ex =
+ verifyExceptionClass(AWSApiCallTimeoutException.class,
+ translateException("test", "/", createS3Exception(504)));
+ verifyCause(S3Exception.class, ex);
+ }
+
+ /**
+ * SDK ApiCallTimeoutException is translated to a
+ * {@link AWSApiCallTimeoutException}.
+ */
+ @Test
+ public void testApiCallTimeoutExceptionToTimeout() throws Throwable {
+ AWSApiCallTimeoutException ex =
+ verifyExceptionClass(AWSApiCallTimeoutException.class,
+ translateException("test", "/",
+ ApiCallTimeoutException.builder()
+ .message("timeout")
+ .build()));
+ verifyCause(ApiCallTimeoutException.class, ex);
+ }
+
+ /**
+ * SDK ApiCallAttemptTimeoutException is translated to a
+ * {@link AWSApiCallTimeoutException}.
+ */
+ @Test
+ public void testApiCallAttemptTimeoutExceptionToTimeout() throws Throwable {
+ AWSApiCallTimeoutException ex =
+ verifyExceptionClass(AWSApiCallTimeoutException.class,
+ translateException("test", "/",
+ ApiCallAttemptTimeoutException.builder()
+ .message("timeout")
+ .build()));
+ verifyCause(ApiCallAttemptTimeoutException.class, ex);
+
+ // and confirm these timeouts are retried.
+ final S3ARetryPolicy retryPolicy = new S3ARetryPolicy(new Configuration(false));
+ Assertions.assertThat(retryPolicy.shouldRetry(ex, 0, 0, true).action)
+ .describedAs("retry policy for exception %s", ex)
+ .isEqualTo(RetryPolicy.RetryAction.RetryDecision.RETRY);
+ }
+
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestConnectionTimeouts.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestConnectionTimeouts.java
new file mode 100644
index 000000000000..2da70e6ef459
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestConnectionTimeouts.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FutureDataInputStreamBuilder;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.net.ConnectTimeoutException;
+
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.apache.hadoop.fs.s3a.Constants.CONNECTION_ACQUISITION_TIMEOUT;
+import static org.apache.hadoop.fs.s3a.Constants.CONNECTION_IDLE_TIME;
+import static org.apache.hadoop.fs.s3a.Constants.CONNECTION_TTL;
+import static org.apache.hadoop.fs.s3a.Constants.ESTABLISH_TIMEOUT;
+import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
+import static org.apache.hadoop.fs.s3a.Constants.MAXIMUM_CONNECTIONS;
+import static org.apache.hadoop.fs.s3a.Constants.MAX_ERROR_RETRIES;
+import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_ENABLED_KEY;
+import static org.apache.hadoop.fs.s3a.Constants.REQUEST_TIMEOUT;
+import static org.apache.hadoop.fs.s3a.Constants.RETRY_LIMIT;
+import static org.apache.hadoop.fs.s3a.Constants.SOCKET_TIMEOUT;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static org.apache.hadoop.fs.s3a.impl.ConfigurationHelper.setDurationAsMillis;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Test for timeout generation/handling, especially those related to connection
+ * pools.
+ * This test has proven a bit brittle to parallel test runs, so test cases should
+ * create their own FS instances.
+ * The likely cause is actually -Dprefetch test runs as these return connections to
+ * the pool.
+ * However, it is also important to have a non-brittle FS for creating the test file
+ * and teardow, again, this makes for a flaky test..
+ */
+public class ITestConnectionTimeouts extends AbstractS3ATestBase {
+
+ /**
+ * How big a file to create?
+ */
+ public static final int FILE_SIZE = 1024;
+
+ /**
+ * Create a configuration for an FS which has timeouts set to very low values
+ * and no retries.
+ * @return a configuration to use for the brittle FS.
+ */
+ private Configuration timingOutConfiguration() {
+ Configuration conf = new Configuration(getConfiguration());
+ removeBaseAndBucketOverrides(conf,
+ CONNECTION_TTL,
+ CONNECTION_ACQUISITION_TIMEOUT,
+ CONNECTION_IDLE_TIME,
+ ESTABLISH_TIMEOUT,
+ MAX_ERROR_RETRIES,
+ MAXIMUM_CONNECTIONS,
+ PREFETCH_ENABLED_KEY,
+ REQUEST_TIMEOUT,
+ SOCKET_TIMEOUT,
+ FS_S3A_CREATE_PERFORMANCE
+ );
+
+ // only one connection is allowed, and the establish timeout is low
+ conf.setInt(MAXIMUM_CONNECTIONS, 1);
+ conf.setInt(MAX_ERROR_RETRIES, 0);
+ // needed to ensure that streams are kept open.
+ // without this the tests is unreliable in batch runs.
+ conf.setBoolean(PREFETCH_ENABLED_KEY, false);
+ conf.setInt(RETRY_LIMIT, 0);
+ conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, true);
+ final Duration ms10 = Duration.ofMillis(10);
+ setDurationAsMillis(conf, CONNECTION_ACQUISITION_TIMEOUT, ms10);
+ setDurationAsMillis(conf, ESTABLISH_TIMEOUT, ms10);
+ return conf;
+ }
+
+ @Override
+ public void teardown() throws Exception {
+ AWSClientConfig.resetMinimumOperationDuration();
+ super.teardown();
+ }
+
+ /**
+ * Use up the connection pool and expect the failure to be handled.
+ */
+ @Test
+ public void testGeneratePoolTimeouts() throws Throwable {
+ byte[] data = dataset(FILE_SIZE, '0', 10);
+ AWSClientConfig.setMinimumOperationDuration(Duration.ZERO);
+ Configuration conf = timingOutConfiguration();
+ Path path = methodPath();
+ int streamsToCreate = 100;
+ List<FSDataInputStream> streams = new ArrayList<>(streamsToCreate);
+ final S3AFileSystem fs = getFileSystem();
+ // create the test file using the good fs, to avoid connection timeouts
+ // during setup.
+ ContractTestUtils.createFile(fs, path, true, data);
+ final FileStatus st = fs.getFileStatus(path);
+ try (FileSystem brittleFS = FileSystem.newInstance(fs.getUri(), conf)) {
+ intercept(ConnectTimeoutException.class, () -> {
+ for (int i = 0; i < streamsToCreate; i++) {
+ FutureDataInputStreamBuilder b = brittleFS.openFile(path);
+ b.withFileStatus(st);
+ b.opt(FS_OPTION_OPENFILE_READ_POLICY, FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE);
+
+ final FSDataInputStream in = b.build().get();
+ streams.add(in);
+ // kick off the read so forcing a GET.
+ in.read();
+ }
+ });
+ } finally {
+ streams.forEach(s -> {
+ IOUtils.cleanupWithLogger(LOG, s);
+ });
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestAwsClientConfig.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestAwsClientConfig.java
new file mode 100644
index 000000000000..2a9e05755ce6
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestAwsClientConfig.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+import java.time.Duration;
+import java.util.Arrays;
+
+import org.assertj.core.api.Assertions;
+import org.junit.After;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static org.apache.hadoop.fs.s3a.Constants.CONNECTION_ACQUISITION_TIMEOUT;
+import static org.apache.hadoop.fs.s3a.Constants.CONNECTION_IDLE_TIME;
+import static org.apache.hadoop.fs.s3a.Constants.CONNECTION_KEEPALIVE;
+import static org.apache.hadoop.fs.s3a.Constants.CONNECTION_TTL;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_CONNECTION_ACQUISITION_TIMEOUT_DURATION;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_CONNECTION_IDLE_TIME_DURATION;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_CONNECTION_KEEPALIVE;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_CONNECTION_TTL_DURATION;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_ESTABLISH_TIMEOUT_DURATION;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_MAXIMUM_CONNECTIONS;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_SOCKET_TIMEOUT_DURATION;
+import static org.apache.hadoop.fs.s3a.Constants.ESTABLISH_TIMEOUT;
+import static org.apache.hadoop.fs.s3a.Constants.MAXIMUM_CONNECTIONS;
+import static org.apache.hadoop.fs.s3a.Constants.MINIMUM_NETWORK_OPERATION_DURATION;
+import static org.apache.hadoop.fs.s3a.Constants.REQUEST_TIMEOUT;
+import static org.apache.hadoop.fs.s3a.Constants.SOCKET_TIMEOUT;
+import static org.apache.hadoop.fs.s3a.impl.AWSClientConfig.createApiConnectionSettings;
+import static org.apache.hadoop.fs.s3a.impl.AWSClientConfig.createConnectionSettings;
+import static org.apache.hadoop.fs.s3a.impl.ConfigurationHelper.enforceMinimumDuration;
+
+/**
+ * Unit tests for {@link AWSClientConfig}.
+ * These may play with the config timeout settings, so reset the timeouts
+ * during teardown.
+ * For isolation from any site settings, the tests create configurations
+ * without loading of defaut/site XML files.
+ */
+public class TestAwsClientConfig extends AbstractHadoopTestBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestAwsClientConfig.class);
+
+ @After
+ public void teardown() throws Exception {
+ AWSClientConfig.resetMinimumOperationDuration();
+ }
+
+ /**
+ * Create a new empty configuration.
+ * @return configuration.
+ */
+ private Configuration conf() {
+ return new Configuration(false);
+ }
+
+ /**
+ * Innermost duration enforcement, which is not applied if
+ * the minimum value is null.
+ */
+ @Test
+ public void testEnforceMinDuration() {
+ final Duration s10 = Duration.ofSeconds(10);
+ final Duration s1 = Duration.ofSeconds(1);
+
+ Assertions.assertThat(enforceMinimumDuration("key", s1, s10))
+ .describedAs("10s")
+ .isEqualTo(s10);
+
+ // and a null check
+ Assertions.assertThat(enforceMinimumDuration("key",
+ s1, null))
+ .describedAs("10s")
+ .isEqualTo(s1);
+ }
+
+ /**
+ * When loading a connection settings from an empty configuration, the
+ * correct default values are loaded.
+ */
+ @Test
+ public void testLoadUnsetValues() {
+ final AWSClientConfig.ConnectionSettings conn = createConnectionSettings(conf());
+ assertDuration(CONNECTION_ACQUISITION_TIMEOUT, DEFAULT_CONNECTION_ACQUISITION_TIMEOUT_DURATION,
+ conn.getAcquisitionTimeout());
+ assertDuration(CONNECTION_TTL, DEFAULT_CONNECTION_TTL_DURATION,
+ conn.getConnectionTTL());
+ assertDuration(CONNECTION_IDLE_TIME, DEFAULT_CONNECTION_IDLE_TIME_DURATION,
+ conn.getMaxIdleTime());
+ assertDuration(ESTABLISH_TIMEOUT, DEFAULT_ESTABLISH_TIMEOUT_DURATION,
+ conn.getEstablishTimeout());
+ assertDuration(SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT_DURATION,
+ conn.getSocketTimeout());
+ Assertions.assertThat(conn.getMaxConnections())
+ .describedAs(MAXIMUM_CONNECTIONS)
+ .isEqualTo(DEFAULT_MAXIMUM_CONNECTIONS);
+ Assertions.assertThat(conn.isKeepAlive())
+ .describedAs(CONNECTION_KEEPALIVE)
+ .isEqualTo(DEFAULT_CONNECTION_KEEPALIVE);
+ }
+
+ /**
+ * If we set a minimum duration that is bigger than the configured value,
+ * the minimum value wins.
+ * Some options have a minimum value of zero.
+ */
+ @Test
+ public void testMinimumDurationWins() {
+
+ final Configuration conf = conf();
+ setOptionsToValue("1s", conf,
+ CONNECTION_ACQUISITION_TIMEOUT,
+ CONNECTION_TTL,
+ CONNECTION_IDLE_TIME,
+ ESTABLISH_TIMEOUT,
+ SOCKET_TIMEOUT);
+ final AWSClientConfig.ConnectionSettings conn = createConnectionSettings(conf);
+ LOG.info("Connection settings: {}", conn);
+ assertDuration(CONNECTION_ACQUISITION_TIMEOUT, MINIMUM_NETWORK_OPERATION_DURATION,
+ conn.getAcquisitionTimeout());
+
+ assertDuration(ESTABLISH_TIMEOUT, MINIMUM_NETWORK_OPERATION_DURATION,
+ conn.getEstablishTimeout());
+ assertDuration(SOCKET_TIMEOUT, MINIMUM_NETWORK_OPERATION_DURATION,
+ conn.getSocketTimeout());
+
+ // those options with a minimum of zero
+ final Duration s1 = Duration.ofSeconds(1);
+ assertDuration(CONNECTION_TTL, s1, conn.getConnectionTTL());
+ assertDuration(CONNECTION_IDLE_TIME, s1, conn.getMaxIdleTime());
+ }
+
+ /**
+ * Assert that a a duration has the expected value.
+ * @param name option name for assertion text
+ * @param expected expected duration
+ * @param actual actual duration
+ */
+ private void assertDuration(String name, Duration expected, Duration actual) {
+ Assertions.assertThat(actual)
+ .describedAs("Duration of %s", name)
+ .isEqualTo(expected);
+ }
+
+ /**
+ * Test {@link AWSClientConfig#createApiConnectionSettings(Configuration)}.
+ */
+ @Test
+ public void testCreateApiConnectionSettings() {
+ final Configuration conf = conf();
+ conf.set(REQUEST_TIMEOUT, "1h");
+ final AWSClientConfig.ClientSettings settings =
+ createApiConnectionSettings(conf);
+ Assertions.assertThat(settings.getApiCallTimeout())
+ .describedAs("%s in %s", REQUEST_TIMEOUT, settings)
+ .isEqualTo(Duration.ofHours(1));
+ }
+
+ /**
+ * Set a list of keys to the same value.
+ * @param value value to set
+ * @param conf configuration to patch
+ * @param keys keys
+ */
+ private void setOptionsToValue(String value, Configuration conf, String... keys) {
+ Arrays.stream(keys).forEach(key -> conf.set(key, value));
+ }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestUnbufferDraining.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestUnbufferDraining.java
index fb75560a8075..b77ca97c7ddf 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestUnbufferDraining.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestUnbufferDraining.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs.s3a.performance;
import java.io.IOException;
+import java.time.Duration;
import org.assertj.core.api.Assertions;
import org.junit.Test;
@@ -34,6 +35,7 @@ import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.S3AInputPolicy;
import org.apache.hadoop.fs.s3a.S3AInputStream;
+import org.apache.hadoop.fs.s3a.impl.AWSClientConfig;
import org.apache.hadoop.fs.statistics.IOStatistics;
import org.apache.hadoop.io.IOUtils;
@@ -51,6 +53,7 @@ import static org.apache.hadoop.fs.s3a.Constants.REQUEST_TIMEOUT;
import static org.apache.hadoop.fs.s3a.Constants.RETRY_LIMIT;
import static org.apache.hadoop.fs.s3a.Constants.SOCKET_TIMEOUT;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static org.apache.hadoop.fs.s3a.impl.ConfigurationHelper.setDurationAsSeconds;
import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyStatisticCounterValue;
import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.retrieveIOStatistics;
import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_ABORTED;
@@ -119,16 +122,23 @@ public class ITestUnbufferDraining extends AbstractS3ACostTest {
// now create a new FS with minimal http capacity and recovery
// a separate one is used to avoid test teardown suffering
// from the lack of http connections and short timeouts.
- Configuration conf = getConfiguration();
- // kick off async drain for any data
- conf.setInt(ASYNC_DRAIN_THRESHOLD, 1);
- conf.setInt(MAXIMUM_CONNECTIONS, 2);
- conf.setInt(MAX_ERROR_RETRIES, 1);
- conf.setInt(ESTABLISH_TIMEOUT, 1000);
- conf.setInt(READAHEAD_RANGE, READAHEAD);
- conf.setInt(RETRY_LIMIT, 1);
-
- brittleFS = FileSystem.newInstance(getFileSystem().getUri(), conf);
+ try {
+ // allow small durations.
+ AWSClientConfig.setMinimumOperationDuration(Duration.ZERO);
+ Configuration conf = getConfiguration();
+ // kick off async drain for any data
+ conf.setInt(ASYNC_DRAIN_THRESHOLD, 1);
+ conf.setInt(MAXIMUM_CONNECTIONS, 2);
+ conf.setInt(MAX_ERROR_RETRIES, 1);
+ conf.setInt(READAHEAD_RANGE, READAHEAD);
+ conf.setInt(RETRY_LIMIT, 1);
+ setDurationAsSeconds(conf, ESTABLISH_TIMEOUT,
+ Duration.ofSeconds(1));
+
+ brittleFS = FileSystem.newInstance(getFileSystem().getUri(), conf);
+ } finally {
+ AWSClientConfig.resetMinimumOperationDuration();
+ }
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 17/17: preparing for 3.3.6-aws release
Posted by ah...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ahmar pushed a commit to branch feature-HADOOP-18073-sdk-v2-upgrade-3.6-aws
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit d4808360467e0ec54614e6657359af3a0bb13d2a
Author: Ahmar Suhail <ah...@amazon.co.uk>
AuthorDate: Tue Dec 5 08:20:11 2023 +0000
preparing for 3.3.6-aws release
---
hadoop-assemblies/pom.xml | 4 ++--
hadoop-build-tools/pom.xml | 2 +-
hadoop-client-modules/hadoop-client-api/pom.xml | 4 ++--
hadoop-client-modules/hadoop-client-check-invariants/pom.xml | 4 ++--
hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml | 4 ++--
hadoop-client-modules/hadoop-client-integration-tests/pom.xml | 4 ++--
hadoop-client-modules/hadoop-client-minicluster/pom.xml | 4 ++--
hadoop-client-modules/hadoop-client-runtime/pom.xml | 4 ++--
hadoop-client-modules/hadoop-client/pom.xml | 4 ++--
hadoop-client-modules/pom.xml | 2 +-
hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml | 4 ++--
hadoop-cloud-storage-project/hadoop-cos/pom.xml | 2 +-
hadoop-cloud-storage-project/pom.xml | 4 ++--
hadoop-common-project/hadoop-annotations/pom.xml | 4 ++--
hadoop-common-project/hadoop-auth-examples/pom.xml | 4 ++--
hadoop-common-project/hadoop-auth/pom.xml | 4 ++--
hadoop-common-project/hadoop-common/pom.xml | 4 ++--
hadoop-common-project/hadoop-kms/pom.xml | 4 ++--
hadoop-common-project/hadoop-minikdc/pom.xml | 4 ++--
hadoop-common-project/hadoop-nfs/pom.xml | 4 ++--
hadoop-common-project/hadoop-registry/pom.xml | 4 ++--
hadoop-common-project/pom.xml | 4 ++--
hadoop-dist/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 4 ++--
hadoop-hdfs-project/pom.xml | 4 ++--
.../hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml | 4 ++--
.../hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml | 4 ++--
.../hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml | 4 ++--
.../hadoop-mapreduce-client-hs-plugins/pom.xml | 4 ++--
.../hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml | 4 ++--
.../hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml | 4 ++--
.../hadoop-mapreduce-client-nativetask/pom.xml | 4 ++--
.../hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml | 4 ++--
.../hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/pom.xml | 4 ++--
hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml | 4 ++--
hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml | 4 ++--
hadoop-mapreduce-project/pom.xml | 4 ++--
hadoop-maven-plugins/pom.xml | 2 +-
hadoop-minicluster/pom.xml | 4 ++--
hadoop-project-dist/pom.xml | 4 ++--
hadoop-project/pom.xml | 4 ++--
hadoop-tools/hadoop-aliyun/pom.xml | 2 +-
hadoop-tools/hadoop-archive-logs/pom.xml | 4 ++--
hadoop-tools/hadoop-archives/pom.xml | 4 ++--
hadoop-tools/hadoop-aws/pom.xml | 4 ++--
hadoop-tools/hadoop-azure-datalake/pom.xml | 2 +-
hadoop-tools/hadoop-azure/pom.xml | 2 +-
hadoop-tools/hadoop-benchmark/pom.xml | 4 ++--
hadoop-tools/hadoop-datajoin/pom.xml | 4 ++--
hadoop-tools/hadoop-distcp/pom.xml | 4 ++--
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-blockgen/pom.xml | 4 ++--
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-dist/pom.xml | 4 ++--
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/pom.xml | 4 ++--
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/pom.xml | 4 ++--
hadoop-tools/hadoop-dynamometer/pom.xml | 4 ++--
hadoop-tools/hadoop-extras/pom.xml | 4 ++--
hadoop-tools/hadoop-fs2img/pom.xml | 4 ++--
hadoop-tools/hadoop-gridmix/pom.xml | 4 ++--
hadoop-tools/hadoop-kafka/pom.xml | 4 ++--
hadoop-tools/hadoop-openstack/pom.xml | 4 ++--
hadoop-tools/hadoop-pipes/pom.xml | 4 ++--
hadoop-tools/hadoop-resourceestimator/pom.xml | 2 +-
hadoop-tools/hadoop-rumen/pom.xml | 4 ++--
hadoop-tools/hadoop-sls/pom.xml | 4 ++--
hadoop-tools/hadoop-streaming/pom.xml | 4 ++--
hadoop-tools/hadoop-tools-dist/pom.xml | 4 ++--
hadoop-tools/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml | 4 ++--
.../hadoop-yarn-applications-catalog-docker/pom.xml | 2 +-
.../hadoop-yarn-applications-catalog-webapp/pom.xml | 2 +-
.../hadoop-yarn-applications/hadoop-yarn-applications-catalog/pom.xml | 2 +-
.../hadoop-yarn-applications-distributedshell/pom.xml | 4 ++--
.../hadoop-yarn-applications-mawo-core/pom.xml | 2 +-
.../hadoop-yarn-applications/hadoop-yarn-applications-mawo/pom.xml | 2 +-
.../hadoop-yarn-applications-unmanaged-am-launcher/pom.xml | 4 ++--
.../hadoop-yarn-services/hadoop-yarn-services-api/pom.xml | 2 +-
.../hadoop-yarn-services/hadoop-yarn-services-core/pom.xml | 2 +-
.../hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml | 2 +-
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml | 2 +-
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml | 4 ++--
.../hadoop-yarn-server-applicationhistoryservice/pom.xml | 4 ++--
.../hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml | 4 ++--
.../hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml | 4 ++--
.../hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml | 4 ++--
.../hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml | 4 ++--
.../hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml | 4 ++--
.../hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml | 4 ++--
.../hadoop-yarn-server-timeline-pluginstorage/pom.xml | 4 ++--
.../hadoop-yarn-server-timelineservice-documentstore/pom.xml | 2 +-
.../hadoop-yarn-server-timelineservice-hbase-tests/pom.xml | 4 ++--
.../hadoop-yarn-server-timelineservice-hbase-client/pom.xml | 2 +-
.../hadoop-yarn-server-timelineservice-hbase-common/pom.xml | 4 ++--
.../hadoop-yarn-server-timelineservice-hbase-server-1/pom.xml | 4 ++--
.../hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml | 4 ++--
.../hadoop-yarn-server-timelineservice-hbase-server/pom.xml | 4 ++--
.../hadoop-yarn-server-timelineservice-hbase/pom.xml | 4 ++--
.../hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml | 4 ++--
.../hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/pom.xml | 4 ++--
hadoop-yarn-project/pom.xml | 4 ++--
pom.xml | 4 ++--
112 files changed, 205 insertions(+), 205 deletions(-)
diff --git a/hadoop-assemblies/pom.xml b/hadoop-assemblies/pom.xml
index 33b9aa86961a..bebf12c90550 100644
--- a/hadoop-assemblies/pom.xml
+++ b/hadoop-assemblies/pom.xml
@@ -23,11 +23,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-assemblies</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop Assemblies</name>
<description>Apache Hadoop Assemblies</description>
diff --git a/hadoop-build-tools/pom.xml b/hadoop-build-tools/pom.xml
index 3cc154b5b9a3..a212f6c84116 100644
--- a/hadoop-build-tools/pom.xml
+++ b/hadoop-build-tools/pom.xml
@@ -18,7 +18,7 @@
<parent>
<artifactId>hadoop-main</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-build-tools</artifactId>
diff --git a/hadoop-client-modules/hadoop-client-api/pom.xml b/hadoop-client-modules/hadoop-client-api/pom.xml
index 77051050c200..9d7054e99199 100644
--- a/hadoop-client-modules/hadoop-client-api/pom.xml
+++ b/hadoop-client-modules/hadoop-client-api/pom.xml
@@ -18,11 +18,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-client-api</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<packaging>jar</packaging>
<description>Apache Hadoop Client</description>
diff --git a/hadoop-client-modules/hadoop-client-check-invariants/pom.xml b/hadoop-client-modules/hadoop-client-check-invariants/pom.xml
index 3183b822ad13..6fdbb6cfbb29 100644
--- a/hadoop-client-modules/hadoop-client-check-invariants/pom.xml
+++ b/hadoop-client-modules/hadoop-client-check-invariants/pom.xml
@@ -18,11 +18,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-client-check-invariants</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<packaging>pom</packaging>
<description>
diff --git a/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml b/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml
index 49596b4e74b2..46cf37c93586 100644
--- a/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml
+++ b/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml
@@ -18,11 +18,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-client-check-test-invariants</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<packaging>pom</packaging>
<description>
diff --git a/hadoop-client-modules/hadoop-client-integration-tests/pom.xml b/hadoop-client-modules/hadoop-client-integration-tests/pom.xml
index 70b97d58d23a..880bdc86023e 100644
--- a/hadoop-client-modules/hadoop-client-integration-tests/pom.xml
+++ b/hadoop-client-modules/hadoop-client-integration-tests/pom.xml
@@ -18,11 +18,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-client-integration-tests</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Checks that we can use the generated artifacts</description>
<name>Apache Hadoop Client Packaging Integration Tests</name>
diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
index 67e3a8b1b183..61d4091d6c01 100644
--- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml
+++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
@@ -18,11 +18,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-client-minicluster</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<packaging>jar</packaging>
<description>Apache Hadoop Minicluster for Clients</description>
diff --git a/hadoop-client-modules/hadoop-client-runtime/pom.xml b/hadoop-client-modules/hadoop-client-runtime/pom.xml
index 78153c1f0728..ed49e5133d35 100644
--- a/hadoop-client-modules/hadoop-client-runtime/pom.xml
+++ b/hadoop-client-modules/hadoop-client-runtime/pom.xml
@@ -18,11 +18,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-client-runtime</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<packaging>jar</packaging>
<description>Apache Hadoop Client</description>
diff --git a/hadoop-client-modules/hadoop-client/pom.xml b/hadoop-client-modules/hadoop-client/pom.xml
index cb78442e15f6..7cda1b2509a2 100644
--- a/hadoop-client-modules/hadoop-client/pom.xml
+++ b/hadoop-client-modules/hadoop-client/pom.xml
@@ -18,11 +18,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<artifactId>hadoop-client</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Client aggregation pom with dependencies exposed</description>
<name>Apache Hadoop Client Aggregator</name>
diff --git a/hadoop-client-modules/pom.xml b/hadoop-client-modules/pom.xml
index 68ab8e5b0957..1f0c3e8e2043 100644
--- a/hadoop-client-modules/pom.xml
+++ b/hadoop-client-modules/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-client-modules</artifactId>
diff --git a/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml b/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
index fa1b68a4c970..dbe4fbe4e54a 100644
--- a/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
+++ b/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
@@ -18,11 +18,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-cloud-storage</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<packaging>jar</packaging>
<description>Apache Hadoop Cloud Storage</description>
diff --git a/hadoop-cloud-storage-project/hadoop-cos/pom.xml b/hadoop-cloud-storage-project/hadoop-cos/pom.xml
index 7b4e4cba2309..47fdd4d4227b 100644
--- a/hadoop-cloud-storage-project/hadoop-cos/pom.xml
+++ b/hadoop-cloud-storage-project/hadoop-cos/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-cos</artifactId>
diff --git a/hadoop-cloud-storage-project/pom.xml b/hadoop-cloud-storage-project/pom.xml
index 218b87478050..f9ee08970542 100644
--- a/hadoop-cloud-storage-project/pom.xml
+++ b/hadoop-cloud-storage-project/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-cloud-storage-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Cloud Storage Project</description>
<name>Apache Hadoop Cloud Storage Project</name>
<packaging>pom</packaging>
diff --git a/hadoop-common-project/hadoop-annotations/pom.xml b/hadoop-common-project/hadoop-annotations/pom.xml
index 085598cfca21..54bd2f6a4eb2 100644
--- a/hadoop-common-project/hadoop-annotations/pom.xml
+++ b/hadoop-common-project/hadoop-annotations/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-annotations</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Annotations</description>
<name>Apache Hadoop Annotations</name>
<packaging>jar</packaging>
diff --git a/hadoop-common-project/hadoop-auth-examples/pom.xml b/hadoop-common-project/hadoop-auth-examples/pom.xml
index 7e9efe5ceb50..e0b6fcb9047f 100644
--- a/hadoop-common-project/hadoop-auth-examples/pom.xml
+++ b/hadoop-common-project/hadoop-auth-examples/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-auth-examples</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<packaging>war</packaging>
<name>Apache Hadoop Auth Examples</name>
diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml
index 280c5727e38f..d26ffa09b7c0 100644
--- a/hadoop-common-project/hadoop-auth/pom.xml
+++ b/hadoop-common-project/hadoop-auth/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-auth</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<packaging>jar</packaging>
<name>Apache Hadoop Auth</name>
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 7e0367bd2b31..a32612437e7a 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<artifactId>hadoop-common</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Common</description>
<name>Apache Hadoop Common</name>
<packaging>jar</packaging>
diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml
index 21f1cb3ea903..6253699bcd99 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -22,11 +22,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-kms</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<packaging>jar</packaging>
<name>Apache Hadoop KMS</name>
diff --git a/hadoop-common-project/hadoop-minikdc/pom.xml b/hadoop-common-project/hadoop-minikdc/pom.xml
index a84e3628fe96..0ea70b649d6f 100644
--- a/hadoop-common-project/hadoop-minikdc/pom.xml
+++ b/hadoop-common-project/hadoop-minikdc/pom.xml
@@ -18,12 +18,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-minikdc</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop MiniKDC</description>
<name>Apache Hadoop MiniKDC</name>
<packaging>jar</packaging>
diff --git a/hadoop-common-project/hadoop-nfs/pom.xml b/hadoop-common-project/hadoop-nfs/pom.xml
index baabb0fdc6ec..3fb1587ed82c 100644
--- a/hadoop-common-project/hadoop-nfs/pom.xml
+++ b/hadoop-common-project/hadoop-nfs/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-nfs</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<packaging>jar</packaging>
<name>Apache Hadoop NFS</name>
diff --git a/hadoop-common-project/hadoop-registry/pom.xml b/hadoop-common-project/hadoop-registry/pom.xml
index 204fbbe9da55..023bf3564e85 100644
--- a/hadoop-common-project/hadoop-registry/pom.xml
+++ b/hadoop-common-project/hadoop-registry/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-project</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-registry</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop Registry</name>
<dependencies>
diff --git a/hadoop-common-project/pom.xml b/hadoop-common-project/pom.xml
index b2a5fd3a4fa1..edc89fae39c3 100644
--- a/hadoop-common-project/pom.xml
+++ b/hadoop-common-project/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-common-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Common Project</description>
<name>Apache Hadoop Common Project</name>
<packaging>pom</packaging>
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index 04a42f050db7..3d444b3ac80d 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-dist</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Distribution</description>
<name>Apache Hadoop Distribution</name>
<packaging>jar</packaging>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
index c19393b6a936..cd778b862086 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
@@ -20,11 +20,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<artifactId>hadoop-hdfs-client</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop HDFS Client</description>
<name>Apache Hadoop HDFS Client</name>
<packaging>jar</packaging>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
index 4269baf49775..8d004eadd0bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
@@ -22,11 +22,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-hdfs-httpfs</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<packaging>jar</packaging>
<name>Apache Hadoop HttpFS</name>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
index 51c49fb046f4..696a603624da 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
@@ -20,11 +20,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<artifactId>hadoop-hdfs-native-client</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop HDFS Native Client</description>
<name>Apache Hadoop HDFS Native Client</name>
<packaging>jar</packaging>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
index 0cf95cb19f31..cee688cdd66b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
@@ -20,11 +20,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-hdfs-nfs</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop HDFS-NFS</description>
<name>Apache Hadoop HDFS-NFS</name>
<packaging>jar</packaging>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
index 8eb6d9d5193a..a06d404632d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
@@ -20,11 +20,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<artifactId>hadoop-hdfs-rbf</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop HDFS-RBF</description>
<name>Apache Hadoop HDFS-RBF</name>
<packaging>jar</packaging>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index c22238001086..3c0e9a3da8f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -20,11 +20,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<artifactId>hadoop-hdfs</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop HDFS</description>
<name>Apache Hadoop HDFS</name>
<packaging>jar</packaging>
diff --git a/hadoop-hdfs-project/pom.xml b/hadoop-hdfs-project/pom.xml
index d2d5172a712d..0700b2135581 100644
--- a/hadoop-hdfs-project/pom.xml
+++ b/hadoop-hdfs-project/pom.xml
@@ -20,11 +20,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-hdfs-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop HDFS Project</description>
<name>Apache Hadoop HDFS Project</name>
<packaging>pom</packaging>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
index d1db5ab375c7..5fa3fac698c6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-mapreduce-client-app</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop MapReduce App</name>
<properties>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
index ee0765925716..1cd971adfd08 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-mapreduce-client-common</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop MapReduce Common</name>
<properties>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
index 8e6ca9f3ecb9..b49fc2316d76 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop MapReduce Core</name>
<properties>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml
index dbc86c4d6346..dc6b4d4cfb29 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-mapreduce-client-hs-plugins</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop MapReduce HistoryServer Plugins</name>
<properties>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
index 8c4f011805f0..f35b566556a8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-mapreduce-client-hs</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop MapReduce HistoryServer</name>
<properties>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
index 292070050e1e..6e99f8fc4e0b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop MapReduce JobClient</name>
<properties>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
index 17fa5a8ffbef..106395f9d1f3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-mapreduce-client-nativetask</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop MapReduce NativeTask</name>
<properties>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml
index a09ca1d5818c..2e398f86eef5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-mapreduce-client-shuffle</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop MapReduce Shuffle</name>
<properties>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/pom.xml
index b3be58e8366a..4ff6e2d29f0b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/pom.xml
@@ -18,11 +18,11 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-mapreduce-client-uploader</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop MapReduce Uploader</name>
<dependencies>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
index 945d2256a231..f6263f58c1fa 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-mapreduce-client</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop MapReduce Client</name>
<packaging>pom</packaging>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
index 75a15f1e8caf..a00539ce70e0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-mapreduce-examples</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop MapReduce Examples</description>
<name>Apache Hadoop MapReduce Examples</name>
<packaging>jar</packaging>
diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml
index 9fce8dd6eb26..ba17bfd70660 100644
--- a/hadoop-mapreduce-project/pom.xml
+++ b/hadoop-mapreduce-project/pom.xml
@@ -18,11 +18,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-mapreduce</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<packaging>pom</packaging>
<name>Apache Hadoop MapReduce</name>
<url>https://hadoop.apache.org/</url>
diff --git a/hadoop-maven-plugins/pom.xml b/hadoop-maven-plugins/pom.xml
index 907edaa3bc15..94b489b3d8bf 100644
--- a/hadoop-maven-plugins/pom.xml
+++ b/hadoop-maven-plugins/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-maven-plugins</artifactId>
diff --git a/hadoop-minicluster/pom.xml b/hadoop-minicluster/pom.xml
index 7c189536b879..1839ce733248 100644
--- a/hadoop-minicluster/pom.xml
+++ b/hadoop-minicluster/pom.xml
@@ -18,11 +18,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-minicluster</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<packaging>jar</packaging>
<description>Apache Hadoop Mini-Cluster</description>
diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml
index 55ad5e47f045..d2db9837f4f5 100644
--- a/hadoop-project-dist/pom.xml
+++ b/hadoop-project-dist/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Project Dist POM</description>
<name>Apache Hadoop Project Dist POM</name>
<packaging>pom</packaging>
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index dd37935ad064..9145c8648628 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -20,10 +20,10 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-main</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Project POM</description>
<name>Apache Hadoop Project POM</name>
<packaging>pom</packaging>
diff --git a/hadoop-tools/hadoop-aliyun/pom.xml b/hadoop-tools/hadoop-aliyun/pom.xml
index 3f1d454254c4..f4b997c66d2f 100644
--- a/hadoop-tools/hadoop-aliyun/pom.xml
+++ b/hadoop-tools/hadoop-aliyun/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-aliyun</artifactId>
diff --git a/hadoop-tools/hadoop-archive-logs/pom.xml b/hadoop-tools/hadoop-archive-logs/pom.xml
index 79b3bab9309a..e1bfd27a1075 100644
--- a/hadoop-tools/hadoop-archive-logs/pom.xml
+++ b/hadoop-tools/hadoop-archive-logs/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-archive-logs</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Archive Logs</description>
<name>Apache Hadoop Archive Logs</name>
<packaging>jar</packaging>
diff --git a/hadoop-tools/hadoop-archives/pom.xml b/hadoop-tools/hadoop-archives/pom.xml
index d964983ba8da..a7ae1017c1e9 100644
--- a/hadoop-tools/hadoop-archives/pom.xml
+++ b/hadoop-tools/hadoop-archives/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-archives</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Archives</description>
<name>Apache Hadoop Archives</name>
<packaging>jar</packaging>
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 85880b8482f1..5cef13ba2dfc 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -19,11 +19,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-aws</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop Amazon Web Services support</name>
<description>
This module contains code to support integration with Amazon Web Services.
diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml b/hadoop-tools/hadoop-azure-datalake/pom.xml
index 77c9559949af..901e19c10bc3 100644
--- a/hadoop-tools/hadoop-azure-datalake/pom.xml
+++ b/hadoop-tools/hadoop-azure-datalake/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-azure-datalake</artifactId>
diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml
index ac7d8ad8e9d8..9530192900f1 100644
--- a/hadoop-tools/hadoop-azure/pom.xml
+++ b/hadoop-tools/hadoop-azure/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-azure</artifactId>
diff --git a/hadoop-tools/hadoop-benchmark/pom.xml b/hadoop-tools/hadoop-benchmark/pom.xml
index 6d7ba89d81f6..f59b347681c3 100644
--- a/hadoop-tools/hadoop-benchmark/pom.xml
+++ b/hadoop-tools/hadoop-benchmark/pom.xml
@@ -22,11 +22,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project/pom.xml</relativePath>
</parent>
<artifactId>hadoop-benchmark</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<packaging>jar</packaging>
<name>Apache Hadoop Common Benchmark</name>
diff --git a/hadoop-tools/hadoop-datajoin/pom.xml b/hadoop-tools/hadoop-datajoin/pom.xml
index df854e40fc57..34df5ce28ce0 100644
--- a/hadoop-tools/hadoop-datajoin/pom.xml
+++ b/hadoop-tools/hadoop-datajoin/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-datajoin</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Data Join</description>
<name>Apache Hadoop Data Join</name>
<packaging>jar</packaging>
diff --git a/hadoop-tools/hadoop-distcp/pom.xml b/hadoop-tools/hadoop-distcp/pom.xml
index 29f703d1ecc6..50b5f6a3a4ff 100644
--- a/hadoop-tools/hadoop-distcp/pom.xml
+++ b/hadoop-tools/hadoop-distcp/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-distcp</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Distributed Copy</description>
<name>Apache Hadoop Distributed Copy</name>
<packaging>jar</packaging>
diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-blockgen/pom.xml b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-blockgen/pom.xml
index 4c1b6a65c623..ffcd6e57d270 100644
--- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-blockgen/pom.xml
+++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-blockgen/pom.xml
@@ -19,11 +19,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-dynamometer-blockgen</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Dynamometer Block Listing Generator</description>
<name>Apache Hadoop Dynamometer Block Listing Generator</name>
<packaging>jar</packaging>
diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-dist/pom.xml b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-dist/pom.xml
index d6acaeb227e7..a9bc84ae5d31 100644
--- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-dist/pom.xml
+++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-dist/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../../hadoop-project-dist</relativePath>
</parent>
<artifactId>hadoop-dynamometer-dist</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Dynamometer Dist</description>
<name>Apache Hadoop Dynamometer Dist</name>
<packaging>jar</packaging>
diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/pom.xml b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/pom.xml
index 600a45ffcb04..228148fe83c8 100644
--- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/pom.xml
+++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/pom.xml
@@ -19,11 +19,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-dynamometer-infra</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Dynamometer Cluster Simulator</description>
<name>Apache Hadoop Dynamometer Cluster Simulator</name>
<packaging>jar</packaging>
diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/pom.xml b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/pom.xml
index 015ba423230c..03982219d20e 100644
--- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/pom.xml
+++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/pom.xml
@@ -19,11 +19,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-dynamometer-workload</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Dynamometer Workload Simulator</description>
<name>Apache Hadoop Dynamometer Workload Simulator</name>
<packaging>jar</packaging>
diff --git a/hadoop-tools/hadoop-dynamometer/pom.xml b/hadoop-tools/hadoop-dynamometer/pom.xml
index f8723071780a..06417628373c 100644
--- a/hadoop-tools/hadoop-dynamometer/pom.xml
+++ b/hadoop-tools/hadoop-dynamometer/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-dynamometer</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Dynamometer</description>
<name>Apache Hadoop Dynamometer</name>
<packaging>pom</packaging>
diff --git a/hadoop-tools/hadoop-extras/pom.xml b/hadoop-tools/hadoop-extras/pom.xml
index a2fd20a0589e..062ae8c74b24 100644
--- a/hadoop-tools/hadoop-extras/pom.xml
+++ b/hadoop-tools/hadoop-extras/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-extras</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Extras</description>
<name>Apache Hadoop Extras</name>
<packaging>jar</packaging>
diff --git a/hadoop-tools/hadoop-fs2img/pom.xml b/hadoop-tools/hadoop-fs2img/pom.xml
index 97ff2a4985d8..365adf6fddc1 100644
--- a/hadoop-tools/hadoop-fs2img/pom.xml
+++ b/hadoop-tools/hadoop-fs2img/pom.xml
@@ -17,12 +17,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-fs2img</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Image Generation Tool</description>
<name>Apache Hadoop Image Generation Tool</name>
<packaging>jar</packaging>
diff --git a/hadoop-tools/hadoop-gridmix/pom.xml b/hadoop-tools/hadoop-gridmix/pom.xml
index e7e8730ab72b..ad31989236a7 100644
--- a/hadoop-tools/hadoop-gridmix/pom.xml
+++ b/hadoop-tools/hadoop-gridmix/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-gridmix</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Gridmix</description>
<name>Apache Hadoop Gridmix</name>
<packaging>jar</packaging>
diff --git a/hadoop-tools/hadoop-kafka/pom.xml b/hadoop-tools/hadoop-kafka/pom.xml
index af0393b1f689..c73b016b4191 100644
--- a/hadoop-tools/hadoop-kafka/pom.xml
+++ b/hadoop-tools/hadoop-kafka/pom.xml
@@ -19,11 +19,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-kafka</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop Kafka Library support</name>
<description>
This module contains code to support integration with Kafka.
diff --git a/hadoop-tools/hadoop-openstack/pom.xml b/hadoop-tools/hadoop-openstack/pom.xml
index ef98f27c6d7a..138a9ee24a89 100644
--- a/hadoop-tools/hadoop-openstack/pom.xml
+++ b/hadoop-tools/hadoop-openstack/pom.xml
@@ -19,11 +19,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-openstack</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop OpenStack support</name>
<description>
This module used to contain code to support integration with OpenStack.
diff --git a/hadoop-tools/hadoop-pipes/pom.xml b/hadoop-tools/hadoop-pipes/pom.xml
index dfdf6a196df3..2300d8ddfa02 100644
--- a/hadoop-tools/hadoop-pipes/pom.xml
+++ b/hadoop-tools/hadoop-pipes/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-pipes</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Pipes</description>
<name>Apache Hadoop Pipes</name>
<packaging>pom</packaging>
diff --git a/hadoop-tools/hadoop-resourceestimator/pom.xml b/hadoop-tools/hadoop-resourceestimator/pom.xml
index bc5d2a072296..f5c874e80312 100644
--- a/hadoop-tools/hadoop-resourceestimator/pom.xml
+++ b/hadoop-tools/hadoop-resourceestimator/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-resourceestimator</artifactId>
diff --git a/hadoop-tools/hadoop-rumen/pom.xml b/hadoop-tools/hadoop-rumen/pom.xml
index 9e0c19e9dd2e..a5da9a8f131b 100644
--- a/hadoop-tools/hadoop-rumen/pom.xml
+++ b/hadoop-tools/hadoop-rumen/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-rumen</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Rumen</description>
<name>Apache Hadoop Rumen</name>
<packaging>jar</packaging>
diff --git a/hadoop-tools/hadoop-sls/pom.xml b/hadoop-tools/hadoop-sls/pom.xml
index c8a0e6d67f44..d0cb61c18439 100644
--- a/hadoop-tools/hadoop-sls/pom.xml
+++ b/hadoop-tools/hadoop-sls/pom.xml
@@ -19,11 +19,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-sls</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Scheduler Load Simulator</description>
<name>Apache Hadoop Scheduler Load Simulator</name>
<packaging>jar</packaging>
diff --git a/hadoop-tools/hadoop-streaming/pom.xml b/hadoop-tools/hadoop-streaming/pom.xml
index ff25dc078df9..5a439b79638e 100644
--- a/hadoop-tools/hadoop-streaming/pom.xml
+++ b/hadoop-tools/hadoop-streaming/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-streaming</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop MapReduce Streaming</description>
<name>Apache Hadoop MapReduce Streaming</name>
<packaging>jar</packaging>
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index 455c7a7a6964..327de28031a3 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<artifactId>hadoop-tools-dist</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Tools Dist</description>
<name>Apache Hadoop Tools Dist</name>
<packaging>jar</packaging>
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index b05e489261f8..2104c1ae65ea 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-tools</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Tools</description>
<name>Apache Hadoop Tools</name>
<packaging>pom</packaging>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
index 2bd4f2cbe5dd..3989e468c91a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-api</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop YARN API</name>
<properties>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-docker/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-docker/pom.xml
index ed3676d0f331..73857484bc43 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-docker/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-docker/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>hadoop-yarn-applications-catalog</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<name>Apache Hadoop YARN Application Catalog Docker Image</name>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml
index 154f19004bda..748ed08177e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>hadoop-yarn-applications-catalog</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<name>Apache Hadoop YARN Application Catalog Webapp</name>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/pom.xml
index 9b0074c55756..24a3cf0b63ba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hadoop-yarn-applications</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<groupId>org.apache.hadoop</groupId>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
index 07b372fb1433..b332b5b6e10c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn-applications</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-applications-distributedshell</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop YARN DistributedShell</name>
<properties>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/hadoop-yarn-applications-mawo-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/hadoop-yarn-applications-mawo-core/pom.xml
index a19b87c87dc9..712800d55040 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/hadoop-yarn-applications-mawo-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/hadoop-yarn-applications-mawo-core/pom.xml
@@ -15,7 +15,7 @@
<parent>
<artifactId>hadoop-yarn-applications-mawo</artifactId>
<groupId>org.apache.hadoop.applications.mawo</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/pom.xml
index af4258e9b9c1..04d665404db0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/pom.xml
@@ -15,7 +15,7 @@
<parent>
<artifactId>hadoop-yarn-applications</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
index 92b196f8a15b..6fdd4329b0f0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn-applications</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-applications-unmanaged-am-launcher</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop YARN Unmanaged Am Launcher</name>
<properties>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml
index 1b131ea07445..d4e83d9f3deb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-services</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<artifactId>hadoop-yarn-services-api</artifactId>
<name>Apache Hadoop YARN Services API</name>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
index e62384f201e0..43e0f57f612d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-services</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<artifactId>hadoop-yarn-services-core</artifactId>
<packaging>jar</packaging>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
index 48f48e539501..d56fd6c60eb5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hadoop-yarn-applications</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-services</artifactId>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
index 08612160c867..56512125bec4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-applications</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop YARN Applications</name>
<packaging>pom</packaging>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
index c78077275925..832293459e1c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
@@ -17,10 +17,10 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<artifactId>hadoop-yarn-client</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop YARN Client</name>
<properties>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
index 7560fdad8669..5c69552d887c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-common</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop YARN Common</name>
<properties>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml
index be0e1684cbd3..2a2d891ed28a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml
@@ -18,7 +18,7 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-csi</artifactId>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
index 8d2c9e872396..08969ae32b7c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-registry</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop YARN Registry</name>
<dependencies>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
index 046132f0c0aa..433d114a74b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
@@ -22,11 +22,11 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop YARN ApplicationHistoryService</name>
<properties>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
index ec93e9d4b5c7..accc3de6eb91 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-common</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop YARN Server Common</name>
<properties>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
index a13cd63ae84c..acddc7c9d6dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-nodemanager</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop YARN NodeManager</name>
<properties>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
index 190463d84aa1..660971ac5d6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop YARN ResourceManager</name>
<properties>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml
index 9aaf25d91a70..843e12527c21 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-router</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop YARN Router</name>
<properties>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml
index f44d20f5d790..9d5a2ebc17b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml
@@ -17,10 +17,10 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<artifactId>hadoop-yarn-server-sharedcachemanager</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop YARN SharedCacheManager</name>
<properties>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
index 8aca9d4b7fbf..e914b6f86a7c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
@@ -19,10 +19,10 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<artifactId>hadoop-yarn-server-tests</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop YARN Server Tests</name>
<properties>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml
index 059cbaf6a80b..b5ad72480885 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml
@@ -22,11 +22,11 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-timeline-pluginstorage</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop YARN Timeline Plugin Storage</name>
<properties>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/pom.xml
index feca209ccb57..c44eb9002edd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-timelineservice-documentstore</artifactId>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
index f185dd3206b2..8174dcc3767e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
@@ -22,11 +22,11 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-timelineservice-hbase-tests</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop YARN TimelineService HBase tests</name>
<properties>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml
index 23f3c5919a70..7e0d1448a8a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>hadoop-yarn-server-timelineservice-hbase</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-timelineservice-hbase-client</artifactId>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml
index 927c5c4333f0..9210627b5a53 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml
@@ -22,13 +22,13 @@
<parent>
<artifactId>hadoop-yarn-server-timelineservice-hbase</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-timelineservice-hbase-common</artifactId>
<name>Apache Hadoop YARN TimelineService HBase Common</name>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<properties>
<!-- Needed for generating FindBugs warnings using parent pom -->
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-1/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-1/pom.xml
index d7103540a88e..4abb4bbb10e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-1/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-1/pom.xml
@@ -22,13 +22,13 @@
<parent>
<artifactId>hadoop-yarn-server-timelineservice-hbase-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-timelineservice-hbase-server-1</artifactId>
<name>Apache Hadoop YARN TimelineService HBase Server 1.2</name>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<properties>
<!-- Needed for generating FindBugs warnings using parent pom -->
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml
index cae2627ecd97..066c2526e43d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml
@@ -22,13 +22,13 @@
<parent>
<artifactId>hadoop-yarn-server-timelineservice-hbase-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-timelineservice-hbase-server-2</artifactId>
<name>Apache Hadoop YARN TimelineService HBase Server 2.0</name>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<properties>
<!-- Needed for generating FindBugs warnings using parent pom -->
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/pom.xml
index f7656226cf66..5959afb3a4a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/pom.xml
@@ -22,12 +22,12 @@
<parent>
<artifactId>hadoop-yarn-server-timelineservice-hbase</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-timelineservice-hbase-server</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop YARN TimelineService HBase Servers</name>
<packaging>pom</packaging>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml
index d4cbf0be8038..0bdd6fab2344 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml
@@ -22,12 +22,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-timelineservice-hbase</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop YARN TimelineService HBase Backend</name>
<packaging>pom</packaging>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
index 424689893ca3..da843a2eab29 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
@@ -22,11 +22,11 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-timelineservice</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop YARN Timeline Service</name>
<properties>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
index 7cb159e46bcd..70c0af6350dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop YARN Web Proxy</name>
<properties>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
index f97a4e4599dc..b224b16395f0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop YARN Server</name>
<packaging>pom</packaging>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
index 7b495f78c73c..5e1ee01d21e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-site</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop YARN Site</name>
<packaging>pom</packaging>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
index bb15949968d6..2e0f8e5027d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
@@ -20,11 +20,11 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-ui</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<name>Apache Hadoop YARN UI</name>
<packaging>${packagingType}</packaging>
diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml
index 21ea3aba58ca..b735efb12243 100644
--- a/hadoop-yarn-project/hadoop-yarn/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/pom.xml
@@ -17,11 +17,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-yarn</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<packaging>pom</packaging>
<name>Apache Hadoop YARN</name>
diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml
index 1f45b3c23d80..a1a9f18ef389 100644
--- a/hadoop-yarn-project/pom.xml
+++ b/hadoop-yarn-project/pom.xml
@@ -18,11 +18,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-yarn-project</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<packaging>pom</packaging>
<name>Apache Hadoop YARN Project</name>
<url>https://hadoop.apache.org/yarn/</url>
diff --git a/pom.xml b/pom.xml
index a54d09f6ee47..6ea743d47659 100644
--- a/pom.xml
+++ b/pom.xml
@@ -18,7 +18,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-main</artifactId>
- <version>3.3.9-SNAPSHOT</version>
+ <version>3.3.6-aws</version>
<description>Apache Hadoop Main</description>
<name>Apache Hadoop Main</name>
<packaging>pom</packaging>
@@ -80,7 +80,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
<properties>
<!-- required as child projects with different version can't use ${project.version} -->
- <hadoop.version>3.3.9-SNAPSHOT</hadoop.version>
+ <hadoop.version>3.3.6-aws</hadoop.version>
<distMgmtSnapshotsId>apache.snapshots.https</distMgmtSnapshotsId>
<distMgmtSnapshotsName>Apache Development Snapshot Repository</distMgmtSnapshotsName>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 09/17: HADOOP-18945. S3A. IAMInstanceCredentialsProvider failing. (#6202)
Posted by ah...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ahmar pushed a commit to branch feature-HADOOP-18073-sdk-v2-upgrade-3.6-aws
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 8a1eb917ba674ad40ec7a88f5f1482cb78360bd2
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Mon Oct 23 14:24:30 2023 +0100
HADOOP-18945. S3A. IAMInstanceCredentialsProvider failing. (#6202)
This restores asynchronous retrieval/refresh of any AWS credentials provided by the
EC2 instance/container in which the process is running.
Contributed by Steve Loughran
---
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 4 +-
.../s3a/auth/IAMInstanceCredentialsProvider.java | 94 +++++++++++++++---
.../hadoop/fs/s3a/impl/ErrorTranslation.java | 6 +-
.../auth/TestIAMInstanceCredentialsProvider.java | 107 +++++++++++++++++++++
.../hadoop/fs/s3a/impl/TestErrorTranslation.java | 31 +++++-
5 files changed, 220 insertions(+), 22 deletions(-)
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index a8a7172d8a43..155bd8a6105a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -80,7 +80,7 @@ import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.isAbstract;
import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.isNotInstanceOf;
import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.unsupportedConstructor;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.*;
-import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractNetworkException;
+import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractIOException;
import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteIterator;
@@ -192,7 +192,7 @@ public final class S3AUtils {
return ioe;
}
// network problems covered by an IOE inside the exception chain.
- ioe = maybeExtractNetworkException(path, exception);
+ ioe = maybeExtractIOException(path, exception);
if (ioe != null) {
return ioe;
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java
index 2e39b275b4a4..080b79e7f20d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java
@@ -21,37 +21,69 @@ package org.apache.hadoop.fs.s3a.auth;
import java.io.Closeable;
import java.io.IOException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.HttpCredentialsProvider;
import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
import software.amazon.awssdk.core.exception.SdkClientException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractIOException;
+
/**
* This is an IAM credential provider which wraps
* an {@code ContainerCredentialsProvider}
* to provide credentials when the S3A connector is instantiated on AWS EC2
* or the AWS container services.
* <p>
- * When it fails to authenticate, it raises a
- * {@link NoAwsCredentialsException} which can be recognized by retry handlers
+ * The provider is initialized with async credential refresh enabled to be less
+ * brittle against transient network issues.
+ * <p>
+ * If the ContainerCredentialsProvider fails to authenticate, then an instance of
+ * {@link InstanceProfileCredentialsProvider} is created and attemped to
+ * be used instead, again with async credential refresh enabled.
+ * <p>
+ * If both credential providers fail, a {@link NoAwsCredentialsException}
+ * is thrown, which can be recognized by retry handlers
* as a non-recoverable failure.
* <p>
* It is implicitly public; marked evolving as we can change its semantics.
- *
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
public class IAMInstanceCredentialsProvider
implements AwsCredentialsProvider, Closeable {
- private final AwsCredentialsProvider containerCredentialsProvider =
- ContainerCredentialsProvider.builder().build();
+ private static final Logger LOG =
+ LoggerFactory.getLogger(IAMInstanceCredentialsProvider.class);
+
+ /**
+ * The credentials provider.
+ * Initially a container credentials provider, but if that fails
+ * fall back to the instance profile provider.
+ */
+ private HttpCredentialsProvider iamCredentialsProvider;
+
+ /**
+ * Is the container credentials provider in use?
+ */
+ private boolean isContainerCredentialsProvider;
+ /**
+ * Constructor.
+ * Build credentials provider with async refresh,
+ * mark {@link #isContainerCredentialsProvider} as true.
+ */
public IAMInstanceCredentialsProvider() {
+ isContainerCredentialsProvider = true;
+ iamCredentialsProvider = ContainerCredentialsProvider.builder()
+ .asyncCredentialUpdateEnabled(true)
+ .build();
}
/**
@@ -65,9 +97,16 @@ public class IAMInstanceCredentialsProvider
try {
return getCredentials();
} catch (SdkClientException e) {
+
+ // if the exception contains an IOE, extract it
+ // so its type is the immediate cause of this new exception.
+ Throwable t = e;
+ final IOException ioe = maybeExtractIOException("IAM endpoint", e);
+ if (ioe != null) {
+ t = ioe;
+ }
throw new NoAwsCredentialsException("IAMInstanceCredentialsProvider",
- e.getMessage(),
- e);
+ e.getMessage(), t);
}
}
@@ -78,23 +117,52 @@ public class IAMInstanceCredentialsProvider
*
* @return credentials
*/
- private AwsCredentials getCredentials() {
+ private synchronized AwsCredentials getCredentials() {
try {
- return containerCredentialsProvider.resolveCredentials();
+ return iamCredentialsProvider.resolveCredentials();
} catch (SdkClientException e) {
- return InstanceProfileCredentialsProvider.create().resolveCredentials();
+ LOG.debug("Failed to get credentials from container provider,", e);
+ if (isContainerCredentialsProvider) {
+ // create instance profile provider
+ LOG.debug("Switching to instance provider", e);
+
+ // close it to shut down any thread
+ iamCredentialsProvider.close();
+ isContainerCredentialsProvider = false;
+ iamCredentialsProvider = InstanceProfileCredentialsProvider.builder()
+ .asyncCredentialUpdateEnabled(true)
+ .build();
+ return iamCredentialsProvider.resolveCredentials();
+ } else {
+ // already using instance profile provider, so fail
+ throw e;
+ }
+
}
}
+ /**
+ * Is this a container credentials provider?
+ * @return true if the container credentials provider is in use;
+ * false for InstanceProfileCredentialsProvider
+ */
+ public boolean isContainerCredentialsProvider() {
+ return isContainerCredentialsProvider;
+ }
+
@Override
- public void close() throws IOException {
- // no-op.
+ public synchronized void close() throws IOException {
+ // this be true but just for safety...
+ if (iamCredentialsProvider != null) {
+ iamCredentialsProvider.close();
+ }
}
@Override
public String toString() {
return "IAMInstanceCredentialsProvider{" +
- "containerCredentialsProvider=" + containerCredentialsProvider +
+ "credentialsProvider=" + iamCredentialsProvider +
+ ", isContainerCredentialsProvider=" + isContainerCredentialsProvider +
'}';
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java
index 7b5190becc48..f8a1f907bb3b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java
@@ -79,7 +79,7 @@ public final class ErrorTranslation {
* @param thrown exception
* @return a translated exception or null.
*/
- public static IOException maybeExtractNetworkException(String path, Throwable thrown) {
+ public static IOException maybeExtractIOException(String path, Throwable thrown) {
if (thrown == null) {
return null;
@@ -100,7 +100,9 @@ public final class ErrorTranslation {
// as a new instance is created through reflection, the
// class of the returned instance will be that of the innermost,
// unless no suitable constructor is available.
- return wrapWithInnerIOE(path, thrown, (IOException) cause);
+ final IOException ioe = (IOException) cause;
+
+ return wrapWithInnerIOE(path, thrown, ioe);
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java
new file mode 100644
index 000000000000..c8986eab9b85
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.auth;
+
+import java.io.IOException;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+/**
+ * Unit tests for IAMInstanceCredentials provider.
+ * This is a bit tricky as we don't want to require running in EC2,
+ * but nor do we want a test which doesn't work in EC2.
+ */
+public class TestIAMInstanceCredentialsProvider extends AbstractHadoopTestBase {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestIAMInstanceCredentialsProvider.class);
+
+ /**
+ * Error string from
+ * software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider,
+ * if IAM resolution has been disabled: {@value}.
+ */
+ public static final String DISABLED =
+ "IMDS credentials have been disabled by environment variable or system property";
+
+ /**
+ * Test an immediate create/close.
+ */
+ @Test
+ public void testIAMInstanceCredentialsProviderClose() throws Throwable {
+ new IAMInstanceCredentialsProvider().close();
+ }
+
+ /**
+ * Test instantiation.
+ * Multiple outcomes depending on host setup.
+ * <ol>
+ * <li> In EC2: credentials resolved.
+ * Assert the credentials comes with a key.</li>
+ * <li> Not in EC2: NoAwsCredentialsException wraps network error trying
+ * to talk to the service.
+ * Assert wrapped exception is an IOE.</li>
+ * <li> IMDS resolution disabled by env var/sysprop.
+ * NoAwsCredentialsException raised doesn't contain an IOE.
+ * Require the message to contain the {@link #DISABLED} text.</li>j
+ * </ol>
+ */
+ @Test
+ public void testIAMInstanceCredentialsInstantiate() throws Throwable {
+ try (IAMInstanceCredentialsProvider provider = new IAMInstanceCredentialsProvider()) {
+ try {
+ final AwsCredentials credentials = provider.resolveCredentials();
+ // if we get here this test suite is running in a container/EC2
+ LOG.info("Credentials: retrieved from {}: key={}",
+ provider.isContainerCredentialsProvider() ? "container" : "EC2",
+ credentials.accessKeyId());
+ Assertions.assertThat(credentials.accessKeyId())
+ .describedAs("Access key from IMDS")
+ .isNotBlank();
+
+ // and if we get here, so does a second call
+ provider.resolveCredentials();
+ } catch (NoAwsCredentialsException expected) {
+ // this is expected if the test is not running in a container/EC2
+ LOG.info("Not running in a container/EC2");
+ LOG.info("Exception raised", expected);
+ // and we expect to have fallen back to InstanceProfileCredentialsProvider
+ Assertions.assertThat(provider.isContainerCredentialsProvider())
+ .describedAs("%s: shoud be using InstanceProfileCredentialsProvider")
+ .isFalse();
+ final Throwable cause = expected.getCause();
+ if (cause == null) {
+ throw expected;
+ }
+ if (!(cause instanceof IOException)
+ && !cause.toString().contains(DISABLED)) {
+ throw new AssertionError("Cause not a IOException", cause);
+ }
+ }
+ }
+ }
+
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java
index a1fdbabb9479..0f0b2c0c34bb 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java
@@ -19,8 +19,10 @@
package org.apache.hadoop.fs.s3a.impl;
import java.io.IOException;
+import java.io.UncheckedIOException;
import java.net.ConnectException;
import java.net.NoRouteToHostException;
+import java.net.SocketTimeoutException;
import java.net.UnknownHostException;
import java.util.Collections;
@@ -31,9 +33,10 @@ import software.amazon.awssdk.core.exception.SdkClientException;
import software.amazon.awssdk.core.retry.RetryPolicyContext;
import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.s3a.auth.NoAwsCredentialsException;
import org.apache.hadoop.test.AbstractHadoopTestBase;
-import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractNetworkException;
+import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractIOException;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.junit.Assert.assertTrue;
@@ -64,7 +67,7 @@ public class TestErrorTranslation extends AbstractHadoopTestBase {
new UnknownHostException("bottom")));
final IOException ioe = intercept(UnknownHostException.class, "top",
() -> {
- throw maybeExtractNetworkException("", thrown);
+ throw maybeExtractIOException("", thrown);
});
// the wrapped exception is the top level one: no stack traces have
@@ -79,7 +82,7 @@ public class TestErrorTranslation extends AbstractHadoopTestBase {
public void testNoRouteToHostExceptionExtraction() throws Throwable {
intercept(NoRouteToHostException.class, "top",
() -> {
- throw maybeExtractNetworkException("p2",
+ throw maybeExtractIOException("p2",
sdkException("top",
sdkException("middle",
new NoRouteToHostException("bottom"))));
@@ -90,17 +93,35 @@ public class TestErrorTranslation extends AbstractHadoopTestBase {
public void testConnectExceptionExtraction() throws Throwable {
intercept(ConnectException.class, "top",
() -> {
- throw maybeExtractNetworkException("p1",
+ throw maybeExtractIOException("p1",
sdkException("top",
sdkException("middle",
new ConnectException("bottom"))));
});
}
+
+ /**
+ * When there is an UncheckedIOException, its inner class is
+ * extracted.
+ */
+ @Test
+ public void testUncheckedIOExceptionExtraction() throws Throwable {
+ intercept(SocketTimeoutException.class, "top",
+ () -> {
+ final SdkClientException thrown = sdkException("top",
+ sdkException("middle",
+ new UncheckedIOException(
+ new SocketTimeoutException("bottom"))));
+ throw maybeExtractIOException("p1",
+ new NoAwsCredentialsException("IamProvider", thrown.toString(), thrown));
+ });
+ }
+
@Test
public void testNoConstructorExtraction() throws Throwable {
intercept(PathIOException.class, NoConstructorIOE.MESSAGE,
() -> {
- throw maybeExtractNetworkException("p1",
+ throw maybeExtractIOException("p1",
sdkException("top",
sdkException("middle",
new NoConstructorIOE())));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 02/17: HADOOP-18073. S3A: Upgrade AWS SDK to V2 (#5995)
Posted by ah...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ahmar pushed a commit to branch feature-HADOOP-18073-sdk-v2-upgrade-3.6-aws
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 1fbe185f9ba86977495d69e7c80a931628cb7ba0
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Mon Sep 11 14:30:25 2023 +0100
HADOOP-18073. S3A: Upgrade AWS SDK to V2 (#5995)
This patch migrates the S3A connector to use the V2 AWS SDK.
This is a significant change at the source code level.
Any applications using the internal extension/override points in
the filesystem connector are likely to break.
This includes but is not limited to:
- Code invoking methods on the S3AFileSystem class
which used classes from the V1 SDK.
- The ability to define the factory for the `AmazonS3` client, and
to retrieve it from the S3AFileSystem. There is a new factory
API and a special interface S3AInternals to access a limited
set of internal classes and operations.
- Delegation token and auditing extensions.
- Classes trying to integrate with the AWS SDK.
All standard V1 credential providers listed in the option
fs.s3a.aws.credentials.provider will be automatically remapped to their
V2 equivalent.
Other V1 Credential Providers are supported, but only if the V1 SDK is
added back to the classpath.
The SDK Signing plugin has changed; all v1 signers are incompatible.
There is no support for the S3 "v2" signing algorithm.
Finally, the aws-sdk-bundle JAR has been replaced by the shaded V2
equivalent, "bundle.jar", which is now exported by the hadoop-aws module.
Consult the document aws_sdk_upgrade for the full details.
Contributed by Ahmar Suhail + some bits by Steve Loughran
---
LICENSE-binary | 1 +
.../hadoop/fs/statistics/StoreStatisticNames.java | 4 +
.../src/main/resources/core-default.xml | 52 +-
hadoop-project/pom.xml | 22 +-
.../hadoop-aws/dev-support/findbugs-exclude.xml | 5 +
hadoop-tools/hadoop-aws/pom.xml | 16 +-
.../hadoop/fs/s3a/AWSBadRequestException.java | 4 +-
.../apache/hadoop/fs/s3a/AWSClientIOException.java | 18 +-
.../hadoop/fs/s3a/AWSCredentialProviderList.java | 110 +-
.../hadoop/fs/s3a/AWSNoResponseException.java | 9 +-
.../apache/hadoop/fs/s3a/AWSRedirectException.java | 4 +-
.../org/apache/hadoop/fs/s3a/AWSS3IOException.java | 25 +-
.../hadoop/fs/s3a/AWSServiceIOException.java | 37 +-
.../fs/s3a/AWSServiceThrottledException.java | 9 +-
.../hadoop/fs/s3a/AWSStatus500Exception.java | 9 +-
.../fs/s3a/AnonymousAWSCredentialsProvider.java | 18 +-
.../java/org/apache/hadoop/fs/s3a/ArnResource.java | 12 +-
.../java/org/apache/hadoop/fs/s3a/Constants.java | 16 +-
.../fs/s3a/CredentialInitializationException.java | 18 +-
.../hadoop/fs/s3a/DefaultS3ClientFactory.java | 430 +++-----
.../hadoop/fs/s3a/FailureInjectionPolicy.java | 2 +-
.../hadoop/fs/s3a/InconsistentAmazonS3Client.java | 345 ------
.../hadoop/fs/s3a/InconsistentS3ClientFactory.java | 54 -
.../java/org/apache/hadoop/fs/s3a/Invoker.java | 21 +-
.../java/org/apache/hadoop/fs/s3a/Listing.java | 53 +-
.../org/apache/hadoop/fs/s3a/MultipartUtils.java | 49 +-
.../fs/s3a/ProgressableProgressListener.java | 46 +-
.../apache/hadoop/fs/s3a/S3ABlockOutputStream.java | 124 ++-
.../org/apache/hadoop/fs/s3a/S3ADataBlocks.java | 8 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 1099 +++++++++++---------
.../org/apache/hadoop/fs/s3a/S3AInputStream.java | 116 +--
.../org/apache/hadoop/fs/s3a/S3AInternals.java | 118 +++
.../org/apache/hadoop/fs/s3a/S3ARetryPolicy.java | 56 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 678 ++++--------
.../org/apache/hadoop/fs/s3a/S3ClientFactory.java | 187 +++-
.../org/apache/hadoop/fs/s3a/S3ListRequest.java | 16 +-
.../org/apache/hadoop/fs/s3a/S3ListResult.java | 61 +-
.../apache/hadoop/fs/s3a/S3ObjectAttributes.java | 27 -
.../fs/s3a/SharedInstanceCredentialProvider.java | 1 -
.../fs/s3a/SimpleAWSCredentialsProvider.java | 31 +-
.../java/org/apache/hadoop/fs/s3a/Statistic.java | 5 +
.../fs/s3a/TemporaryAWSCredentialsProvider.java | 7 +-
.../java/org/apache/hadoop/fs/s3a/Tristate.java | 45 +-
.../java/org/apache/hadoop/fs/s3a/UploadInfo.java | 12 +-
.../apache/hadoop/fs/s3a/WriteOperationHelper.java | 202 ++--
.../org/apache/hadoop/fs/s3a/WriteOperations.java | 100 +-
.../hadoop/fs/s3a/adapter/AwsV1BindingSupport.java | 118 +++
.../V1ToV2AwsCredentialProviderAdapter.java | 165 +++
.../{UploadInfo.java => adapter/package-info.java} | 32 +-
.../apache/hadoop/fs/s3a/api/RequestFactory.java | 224 ++--
.../fs/s3a/audit/AWSAuditEventCallbacks.java | 105 +-
.../hadoop/fs/s3a/audit/AWSRequestAnalyzer.java | 135 +--
.../hadoop/fs/s3a/audit/AuditIntegration.java | 54 +-
.../hadoop/fs/s3a/audit/AuditManagerS3A.java | 21 +-
.../hadoop/fs/s3a/audit/S3AAuditConstants.java | 13 +-
.../fs/s3a/audit/impl/ActiveAuditManagerS3A.java | 419 +++++---
.../hadoop/fs/s3a/audit/impl/LoggingAuditor.java | 125 ++-
.../fs/s3a/audit/impl/NoopAuditManagerS3A.java | 16 +-
.../s3a/audit/impl/S3AInternalAuditConstants.java | 16 +-
.../fs/s3a/auth/AbstractAWSCredentialProvider.java | 13 +-
.../auth/AbstractSessionCredentialsProvider.java | 27 +-
.../fs/s3a/auth/AssumedRoleCredentialProvider.java | 95 +-
.../fs/s3a/auth/CredentialProviderListFactory.java | 303 ++++++
.../s3a/auth/IAMInstanceCredentialsProvider.java | 49 +-
.../fs/s3a/auth/MarshalledCredentialBinding.java | 58 +-
.../fs/s3a/auth/MarshalledCredentialProvider.java | 5 +-
.../hadoop/fs/s3a/auth/NoAuthWithAWSException.java | 2 +-
.../hadoop/fs/s3a/auth/STSClientFactory.java | 141 ++-
.../apache/hadoop/fs/s3a/auth/SignerFactory.java | 114 ++
.../apache/hadoop/fs/s3a/auth/SignerManager.java | 8 +-
.../delegation/EncryptionSecretOperations.java | 38 +-
.../fs/s3a/auth/delegation/RoleTokenBinding.java | 2 +-
.../s3a/auth/delegation/S3ADelegationTokens.java | 5 -
.../s3a/auth/delegation/SessionTokenBinding.java | 61 +-
.../hadoop/fs/s3a/commit/AbstractS3ACommitter.java | 9 +-
.../apache/hadoop/fs/s3a/commit/PutTracker.java | 4 +-
.../fs/s3a/commit/files/SinglePendingCommit.java | 13 +-
.../fs/s3a/commit/impl/CommitOperations.java | 54 +-
.../fs/s3a/commit/magic/MagicCommitTracker.java | 28 +-
.../AWSCannedACL.java} | 28 +-
.../apache/hadoop/fs/s3a/impl/AWSClientConfig.java | 394 +++++++
.../org/apache/hadoop/fs/s3a/impl/AWSHeaders.java | 98 ++
.../hadoop/fs/s3a/impl/BulkDeleteRetryHandler.java | 12 +-
.../hadoop/fs/s3a/impl/ChangeDetectionPolicy.java | 125 ++-
.../apache/hadoop/fs/s3a/impl/ChangeTracker.java | 71 +-
.../s3a/impl/ConfigureShadedAWSSocketFactory.java | 13 +-
.../org/apache/hadoop/fs/s3a/impl/CopyOutcome.java | 80 --
.../apache/hadoop/fs/s3a/impl/DeleteOperation.java | 17 +-
.../hadoop/fs/s3a/impl/ErrorTranslation.java | 14 +-
.../hadoop/fs/s3a/impl/HeaderProcessing.java | 206 ++--
.../fs/s3a/impl/InstantiationIOException.java | 180 ++++
.../hadoop/fs/s3a/impl/InternalConstants.java | 51 +-
...upport.java => MultiObjectDeleteException.java} | 67 +-
.../apache/hadoop/fs/s3a/impl/NetworkBinding.java | 11 +-
.../hadoop/fs/s3a/impl/OperationCallbacks.java | 15 +-
.../{Tristate.java => impl/ProgressListener.java} | 14 +-
.../ProgressListenerEvent.java} | 17 +-
.../apache/hadoop/fs/s3a/impl/RenameOperation.java | 23 +-
.../hadoop/fs/s3a/impl/RequestFactoryImpl.java | 663 ++++++------
.../hadoop/fs/s3a/impl/S3AMultipartUploader.java | 31 +-
.../hadoop/fs/s3a/impl/SDKStreamDrainer.java | 43 +-
.../org/apache/hadoop/fs/s3a/impl/V2Migration.java | 72 +-
.../hadoop/fs/s3a/prefetch/S3ARemoteObject.java | 54 +-
.../fs/s3a/prefetch/S3ARemoteObjectReader.java | 7 +-
.../apache/hadoop/fs/s3a/s3guard/S3GuardTool.java | 8 +-
.../hadoop/fs/s3a/select/BlockingEnumeration.java | 156 +++
.../apache/hadoop/fs/s3a/select/SelectBinding.java | 126 ++-
.../fs/s3a/select/SelectEventStreamPublisher.java | 124 +++
.../hadoop/fs/s3a/select/SelectInputStream.java | 31 +-
.../fs/s3a/select/SelectObjectContentHelper.java | 114 ++
.../statistics/impl/AwsStatisticsCollector.java | 166 ++-
.../org/apache/hadoop/fs/s3a/tools/MarkerTool.java | 17 +-
.../hadoop/fs/s3a/tools/MarkerToolOperations.java | 13 +-
.../fs/s3a/tools/MarkerToolOperationsImpl.java | 11 +-
.../markdown/tools/hadoop-aws/assumed_roles.md | 2 +-
.../src/site/markdown/tools/hadoop-aws/auditing.md | 48 +-
.../markdown/tools/hadoop-aws/aws_sdk_upgrade.md | 340 +++++-
.../tools/hadoop-aws/aws_sdk_v2_changelog.md | 340 ++++++
.../markdown/tools/hadoop-aws/delegation_tokens.md | 2 +-
.../src/site/markdown/tools/hadoop-aws/index.md | 142 +--
.../site/markdown/tools/hadoop-aws/s3_select.md | 43 +-
.../src/site/markdown/tools/hadoop-aws/testing.md | 69 +-
.../tools/hadoop-aws/troubleshooting_s3a.md | 95 +-
.../apache/hadoop/fs/s3a/AbstractS3AMockTest.java | 30 +-
.../apache/hadoop/fs/s3a/AbstractS3ATestBase.java | 8 +
.../apache/hadoop/fs/s3a/EncryptionTestUtils.java | 20 +-
.../fs/s3a/ITestS3AAWSCredentialsProvider.java | 164 +--
.../hadoop/fs/s3a/ITestS3ABucketExistence.java | 41 +-
.../apache/hadoop/fs/s3a/ITestS3ACannedACLs.java | 31 +-
.../fs/s3a/ITestS3AClientSideEncryptionKms.java | 6 +-
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 236 +++--
.../hadoop/fs/s3a/ITestS3AEncryptionSSEC.java | 2 +-
.../fs/s3a/ITestS3AEncryptionSSEKMSDefaultKey.java | 8 +-
.../ITestS3AEncryptionWithDefaultS3Settings.java | 3 +-
.../hadoop/fs/s3a/ITestS3AEndpointRegion.java | 200 ++--
.../hadoop/fs/s3a/ITestS3AFailureHandling.java | 28 +-
.../hadoop/fs/s3a/ITestS3AMiscOperations.java | 35 +-
.../hadoop/fs/s3a/ITestS3AMultipartUtils.java | 5 +-
.../hadoop/fs/s3a/ITestS3ARequesterPays.java | 2 +-
.../apache/hadoop/fs/s3a/ITestS3AStorageClass.java | 1 -
.../fs/s3a/ITestS3ATemporaryCredentials.java | 30 +-
.../apache/hadoop/fs/s3a/MockS3AFileSystem.java | 17 +-
.../apache/hadoop/fs/s3a/MockS3ClientFactory.java | 49 +-
.../apache/hadoop/fs/s3a/MultipartTestUtils.java | 35 +-
.../org/apache/hadoop/fs/s3a/S3ATestConstants.java | 5 +
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 17 +-
.../org/apache/hadoop/fs/s3a/TestArnResource.java | 9 +-
.../java/org/apache/hadoop/fs/s3a/TestInvoker.java | 105 +-
.../fs/s3a/TestS3AAWSCredentialsProvider.java | 266 ++---
.../hadoop/fs/s3a/TestS3ABlockOutputStream.java | 11 +-
.../apache/hadoop/fs/s3a/TestS3ADeleteOnExit.java | 25 +-
.../hadoop/fs/s3a/TestS3AExceptionTranslation.java | 208 +++-
.../apache/hadoop/fs/s3a/TestS3AGetFileStatus.java | 95 +-
.../hadoop/fs/s3a/TestS3AInputStreamRetry.java | 130 ++-
.../org/apache/hadoop/fs/s3a/TestS3AProxy.java | 18 +-
.../org/apache/hadoop/fs/s3a/TestS3AUnbuffer.java | 50 +-
.../hadoop/fs/s3a/TestStreamChangeTracker.java | 107 +-
.../fs/s3a/TestWildflyAndOpenSSLBinding.java | 9 +-
.../fs/s3a/adapter/TestV1CredentialsProvider.java | 222 ++++
.../hadoop/fs/s3a/audit/AbstractAuditingTest.java | 98 +-
.../hadoop/fs/s3a/audit/AuditTestSupport.java | 4 +-
.../hadoop/fs/s3a/audit/ITestAuditManager.java | 18 +-
...ler.java => SimpleAWSExecutionInterceptor.java} | 37 +-
.../hadoop/fs/s3a/audit/TestAuditIntegration.java | 82 +-
.../fs/s3a/audit/TestAuditSpanLifecycle.java | 11 +-
.../fs/s3a/audit/TestHttpReferrerAuditHeader.java | 55 +-
.../hadoop/fs/s3a/audit/TestLoggingAuditor.java | 41 +-
.../apache/hadoop/fs/s3a/auth/ITestAssumeRole.java | 36 +-
.../hadoop/fs/s3a/auth/ITestCustomSigner.java | 47 +-
.../apache/hadoop/fs/s3a/auth/RoleTestUtils.java | 7 +-
.../fs/s3a/auth/TestMarshalledCredentials.java | 10 +-
.../hadoop/fs/s3a/auth/TestSignerManager.java | 41 +-
.../auth/delegation/CountInvocationsProvider.java | 34 +-
.../ITestSessionDelegationInFilesystem.java | 30 +-
.../delegation/ITestSessionDelegationTokens.java | 28 +-
.../delegation/TestS3ADelegationTokenSupport.java | 1 -
.../fs/s3a/commit/AbstractITCommitProtocol.java | 4 +-
.../fs/s3a/commit/staging/StagingTestBase.java | 178 ++--
.../staging/TestDirectoryCommitterScale.java | 7 +-
.../s3a/commit/staging/TestStagingCommitter.java | 32 +-
.../staging/TestStagingPartitionedTaskCommit.java | 9 +-
.../fileContext/ITestS3AFileContextStatistics.java | 3 +-
.../fs/s3a/impl/ITestPartialRenamesDeletes.java | 1 -
.../hadoop/fs/s3a/impl/ITestRenameDeleteRace.java | 5 +-
.../hadoop/fs/s3a/impl/TestHeaderProcessing.java | 32 +-
.../hadoop/fs/s3a/impl/TestNetworkBinding.java | 43 -
.../hadoop/fs/s3a/impl/TestRequestFactory.java | 109 +-
.../hadoop/fs/s3a/impl/TestSDKStreamDrainer.java | 16 +-
.../performance/ITestDirectoryMarkerListing.java | 25 +-
.../fs/s3a/prefetch/MockS3ARemoteObject.java | 27 +-
.../hadoop/fs/s3a/prefetch/S3APrefetchFakes.java | 48 +-
.../fs/s3a/scale/AbstractSTestS3AHugeFiles.java | 25 +-
.../scale/ILoadTestS3ABulkDeleteThrottling.java | 13 +-
.../fs/s3a/scale/ITestS3ADirectoryPerformance.java | 22 +-
.../s3a/scale/ITestS3AHugeFilesStorageClass.java | 4 +-
.../hadoop/fs/s3a/select/AbstractS3SelectTest.java | 4 +-
.../hadoop/fs/s3a/select/ITestS3SelectLandsat.java | 2 +-
.../hadoop/fs/s3a/select/StreamPublisher.java | 89 ++
.../fs/s3a/select/TestBlockingEnumeration.java | 200 ++++
.../s3a/select/TestSelectEventStreamPublisher.java | 190 ++++
.../statistics/ITestAWSStatisticCollection.java | 4 +-
.../apache/hadoop/fs/s3a/test/ExtraAssertions.java | 2 +-
.../fs/s3a/test/MinimalOperationCallbacks.java | 14 +-
.../test/MinimalWriteOperationHelperCallbacks.java | 16 +-
.../hadoop/fs/s3a/tools/ITestMarkerTool.java | 7 -
205 files changed, 8900 insertions(+), 5946 deletions(-)
diff --git a/LICENSE-binary b/LICENSE-binary
index d84f311ce190..f2e7d9b28b26 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -364,6 +364,7 @@ org.objenesis:objenesis:2.6
org.xerial.snappy:snappy-java:1.1.10.4
org.yaml:snakeyaml:2.0
org.wildfly.openssl:wildfly-openssl:1.1.3.Final
+software.amazon.awssdk:bundle:jar:2.20.128
--------------------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java
index c04c1bb47fce..3a8927aba493 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java
@@ -407,6 +407,10 @@ public final class StoreStatisticNames {
public static final String MULTIPART_UPLOAD_LIST
= "multipart_upload_list";
+ /** Probe for store region: {@value}. */
+ public static final String STORE_REGION_PROBE
+ = "store_region_probe";
+
private StoreStatisticNames() {
}
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 1501ee4bc268..6af4f793732a 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1201,17 +1201,24 @@
<description>AWS secret key used by S3A file system. Omit for IAM role-based or provider-based authentication.</description>
</property>
+<property>
+ <name>fs.s3a.session.token</name>
+ <description>Session token, when using org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider
+ as one of the providers.
+ </description>
+</property>
+
<property>
<name>fs.s3a.aws.credentials.provider</name>
<value>
org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider,
org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,
- com.amazonaws.auth.EnvironmentVariableCredentialsProvider,
+ software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider,
org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider
</value>
<description>
Comma-separated class names of credential provider classes which implement
- com.amazonaws.auth.AWSCredentialsProvider.
+ software.amazon.awssdk.auth.credentials.AwsCredentialsProvider.
When S3A delegation tokens are not enabled, this list will be used
to directly authenticate with S3 and other AWS services.
@@ -1219,43 +1226,6 @@
token binding it may be used
to communicate wih the STS endpoint to request session/role
credentials.
-
- These are loaded and queried in sequence for a valid set of credentials.
- Each listed class must implement one of the following means of
- construction, which are attempted in order:
- * a public constructor accepting java.net.URI and
- org.apache.hadoop.conf.Configuration,
- * a public constructor accepting org.apache.hadoop.conf.Configuration,
- * a public static method named getInstance that accepts no
- arguments and returns an instance of
- com.amazonaws.auth.AWSCredentialsProvider, or
- * a public default constructor.
-
- Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows
- anonymous access to a publicly accessible S3 bucket without any credentials.
- Please note that allowing anonymous access to an S3 bucket compromises
- security and therefore is unsuitable for most use cases. It can be useful
- for accessing public data sets without requiring AWS credentials.
-
- If unspecified, then the default list of credential provider classes,
- queried in sequence, is:
- * org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider: looks
- for session login secrets in the Hadoop configuration.
- * org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider:
- Uses the values of fs.s3a.access.key and fs.s3a.secret.key.
- * com.amazonaws.auth.EnvironmentVariableCredentialsProvider: supports
- configuration of AWS access key ID and secret access key in
- environment variables named AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,
- and AWS_SESSION_TOKEN as documented in the AWS SDK.
- * org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider: picks up
- IAM credentials of any EC2 VM or AWS container in which the process is running.
- </description>
-</property>
-
-<property>
- <name>fs.s3a.session.token</name>
- <description>Session token, when using org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider
- as one of the providers.
</description>
</property>
@@ -1353,10 +1323,10 @@
Note: for job submission to actually collect these tokens,
Kerberos must be enabled.
- Options are:
+ Bindings available in hadoop-aws are:
org.apache.hadoop.fs.s3a.auth.delegation.SessionTokenBinding
org.apache.hadoop.fs.s3a.auth.delegation.FullCredentialsTokenBinding
- and org.apache.hadoop.fs.s3a.auth.delegation.RoleTokenBinding
+ org.apache.hadoop.fs.s3a.auth.delegation.RoleTokenBinding
</description>
</property>
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index c43f846331bb..77e12835e8bf 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -193,6 +193,8 @@
<surefire.fork.timeout>900</surefire.fork.timeout>
<aws-java-sdk.version>1.12.499</aws-java-sdk.version>
<hsqldb.version>2.7.1</hsqldb.version>
+ <aws-java-sdk-v2.version>2.20.128</aws-java-sdk-v2.version>
+ <aws.eventstream.version>1.0.1</aws.eventstream.version>
<frontend-maven-plugin.version>1.11.2</frontend-maven-plugin.version>
<jasmine-maven-plugin.version>2.1</jasmine-maven-plugin.version>
<phantomjs-maven-plugin.version>0.7</phantomjs-maven-plugin.version>
@@ -1103,15 +1105,31 @@
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-bundle</artifactId>
+ <artifactId>aws-java-sdk-core</artifactId>
<version>${aws-java-sdk.version}</version>
<exclusions>
<exclusion>
- <groupId>io.netty</groupId>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>bundle</artifactId>
+ <version>${aws-java-sdk-v2.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>software.amazon.eventstream</groupId>
+ <artifactId>eventstream</artifactId>
+ <version>${aws.eventstream.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.mina</groupId>
<artifactId>mina-core</artifactId>
diff --git a/hadoop-tools/hadoop-aws/dev-support/findbugs-exclude.xml b/hadoop-tools/hadoop-aws/dev-support/findbugs-exclude.xml
index 4bc00a3e5f16..17f7db631c3e 100644
--- a/hadoop-tools/hadoop-aws/dev-support/findbugs-exclude.xml
+++ b/hadoop-tools/hadoop-aws/dev-support/findbugs-exclude.xml
@@ -64,6 +64,11 @@
<Field name="futurePool"/>
<Bug pattern="IS2_INCONSISTENT_SYNC"/>
</Match>
+ <Match>
+ <Class name="org.apache.hadoop.fs.s3a.S3AFileSystem"/>
+ <Field name="s3AsyncClient"/>
+ <Bug pattern="IS2_INCONSISTENT_SYNC"/>
+ </Match>
<Match>
<Class name="org.apache.hadoop.fs.s3a.s3guard.S3GuardTool$BucketInfo"/>
<Method name="run"/>
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 207b1c0f9413..85880b8482f1 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -483,11 +483,25 @@
<scope>test</scope>
<type>test-jar</type>
</dependency>
+
+ <!-- The v1 SDK is used at compilation time for adapter classes in
+ org.apache.hadoop.fs.s3a.adapter. It is not needed at runtime
+ unless a non-standard v1 credential provider is declared. -->
<dependency>
<groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-bundle</artifactId>
+ <artifactId>aws-java-sdk-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>bundle</artifactId>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>software.amazon.eventstream</groupId>
+ <artifactId>eventstream</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java
index 482c5a1db7a1..c5867eeab4f4 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.AmazonServiceException;
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
/**
* A 400 "Bad Request" exception was received.
@@ -36,7 +36,7 @@ public class AWSBadRequestException extends AWSServiceIOException {
* @param cause the underlying cause
*/
public AWSBadRequestException(String operation,
- AmazonServiceException cause) {
+ AwsServiceException cause) {
super(operation, cause);
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSClientIOException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSClientIOException.java
index f31f3c79b355..43b636c6d576 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSClientIOException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSClientIOException.java
@@ -18,29 +18,28 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.SdkBaseException;
+import software.amazon.awssdk.core.exception.SdkException;
import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
import java.io.IOException;
/**
- * IOException equivalent of an {@link AmazonClientException}.
+ * IOException equivalent of an {@link SdkException}.
*/
public class AWSClientIOException extends IOException {
private final String operation;
public AWSClientIOException(String operation,
- SdkBaseException cause) {
+ SdkException cause) {
super(cause);
Preconditions.checkArgument(operation != null, "Null 'operation' argument");
Preconditions.checkArgument(cause != null, "Null 'cause' argument");
this.operation = operation;
}
- public AmazonClientException getCause() {
- return (AmazonClientException) super.getCause();
+ public SdkException getCause() {
+ return (SdkException) super.getCause();
}
@Override
@@ -48,4 +47,11 @@ public class AWSClientIOException extends IOException {
return operation + ": " + getCause().getMessage();
}
+ /**
+ * Query inner cause for retryability.
+ * @return what the cause says.
+ */
+ public boolean retryable() {
+ return getCause().retryable();
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java
index 75b09a4ed746..1a084cd2357d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.fs.s3a;
import java.io.Closeable;
+import java.io.IOException;
+import java.nio.file.AccessDeniedException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -27,21 +29,22 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.AnonymousAWSCredentials;
-import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.s3a.auth.NoAuthWithAWSException;
import org.apache.hadoop.fs.s3a.auth.NoAwsCredentialsException;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.core.exception.SdkException;
/**
* A list of providers.
@@ -51,17 +54,17 @@ import org.apache.hadoop.io.IOUtils;
* <ol>
* <li>Allows extra providers to be added dynamically.</li>
* <li>If any provider in the chain throws an exception other than
- * an {@link AmazonClientException}, that is rethrown, rather than
+ * an {@link SdkException}, that is rethrown, rather than
* swallowed.</li>
* <li>Has some more diagnostics.</li>
- * <li>On failure, the last "relevant" AmazonClientException raised is
+ * <li>On failure, the last "relevant" {@link SdkException} raised is
* rethrown; exceptions other than 'no credentials' have priority.</li>
- * <li>Special handling of {@link AnonymousAWSCredentials}.</li>
+ * <li>Special handling of {@link AnonymousCredentialsProvider}.</li>
* </ol>
*/
-@InterfaceAudience.Private
+@InterfaceAudience.LimitedPrivate("extensions")
@InterfaceStability.Evolving
-public final class AWSCredentialProviderList implements AWSCredentialsProvider,
+public final class AWSCredentialProviderList implements AwsCredentialsProvider,
AutoCloseable {
private static final Logger LOG = LoggerFactory.getLogger(
@@ -73,9 +76,9 @@ public final class AWSCredentialProviderList implements AWSCredentialsProvider,
CREDENTIALS_REQUESTED_WHEN_CLOSED
= "Credentials requested after provider list was closed";
- private final List<AWSCredentialsProvider> providers = new ArrayList<>(1);
+ private final List<AwsCredentialsProvider> providers = new ArrayList<>(1);
private boolean reuseLastProvider = true;
- private AWSCredentialsProvider lastProvider;
+ private AwsCredentialsProvider lastProvider;
private final AtomicInteger refCount = new AtomicInteger(1);
@@ -98,17 +101,17 @@ public final class AWSCredentialProviderList implements AWSCredentialsProvider,
* @param providers provider list.
*/
public AWSCredentialProviderList(
- Collection<AWSCredentialsProvider> providers) {
+ Collection<AwsCredentialsProvider> providers) {
this.providers.addAll(providers);
}
/**
- * Create with an initial list of providers.
+ * Create with an initial list of SDK V2 credential providers.
* @param name name for error messages, may be ""
* @param providerArgs provider list.
*/
public AWSCredentialProviderList(final String name,
- final AWSCredentialsProvider... providerArgs) {
+ final AwsCredentialsProvider... providerArgs) {
setName(name);
Collections.addAll(providers, providerArgs);
}
@@ -126,11 +129,11 @@ public final class AWSCredentialProviderList implements AWSCredentialsProvider,
}
/**
- * Add a new provider.
- * @param p provider
+ * Add a new SDK V2 provider.
+ * @param provider provider
*/
- public void add(AWSCredentialsProvider p) {
- providers.add(p);
+ public void add(AwsCredentialsProvider provider) {
+ providers.add(provider);
}
/**
@@ -142,16 +145,11 @@ public final class AWSCredentialProviderList implements AWSCredentialsProvider,
}
/**
- * Refresh all child entries.
+ * Was an implementation of the v1 refresh; now just
+ * a no-op.
*/
- @Override
+ @Deprecated
public void refresh() {
- if (isClosed()) {
- return;
- }
- for (AWSCredentialsProvider provider : providers) {
- provider.refresh();
- }
}
/**
@@ -160,7 +158,7 @@ public final class AWSCredentialProviderList implements AWSCredentialsProvider,
* @return a set of credentials (possibly anonymous), for authenticating.
*/
@Override
- public AWSCredentials getCredentials() {
+ public AwsCredentials resolveCredentials() {
if (isClosed()) {
LOG.warn(CREDENTIALS_REQUESTED_WHEN_CLOSED);
throw new NoAuthWithAWSException(name +
@@ -168,18 +166,18 @@ public final class AWSCredentialProviderList implements AWSCredentialsProvider,
}
checkNotEmpty();
if (reuseLastProvider && lastProvider != null) {
- return lastProvider.getCredentials();
+ return lastProvider.resolveCredentials();
}
- AmazonClientException lastException = null;
- for (AWSCredentialsProvider provider : providers) {
+ SdkException lastException = null;
+ for (AwsCredentialsProvider provider : providers) {
try {
- AWSCredentials credentials = provider.getCredentials();
+ AwsCredentials credentials = provider.resolveCredentials();
Preconditions.checkNotNull(credentials,
"Null credentials returned by %s", provider);
- if ((credentials.getAWSAccessKeyId() != null &&
- credentials.getAWSSecretKey() != null)
- || (credentials instanceof AnonymousAWSCredentials)) {
+ if ((credentials.accessKeyId() != null && credentials.secretAccessKey() != null) || (
+ provider instanceof AnonymousCredentialsProvider
+ || provider instanceof AnonymousAWSCredentialsProvider)) {
lastProvider = provider;
LOG.debug("Using credentials from {}", provider);
return credentials;
@@ -196,7 +194,7 @@ public final class AWSCredentialProviderList implements AWSCredentialsProvider,
}
LOG.debug("No credentials from {}: {}",
provider, e.toString());
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
lastException = e;
LOG.debug("No credentials provided by {}: {}",
provider, e.toString(), e);
@@ -222,14 +220,13 @@ public final class AWSCredentialProviderList implements AWSCredentialsProvider,
*
* @return providers
*/
- @VisibleForTesting
- List<AWSCredentialsProvider> getProviders() {
+ public List<AwsCredentialsProvider> getProviders() {
return providers;
}
/**
* Verify that the provider list is not empty.
- * @throws AmazonClientException if there are no providers.
+ * @throws SdkException if there are no providers.
*/
public void checkNotEmpty() {
if (providers.isEmpty()) {
@@ -255,9 +252,11 @@ public final class AWSCredentialProviderList implements AWSCredentialsProvider,
*/
@Override
public String toString() {
- return "AWSCredentialProviderList[" +
- name +
- "refcount= " + refCount.get() + ": [" +
+ return "AWSCredentialProviderList"
+ + " name=" + name
+ + "; refcount= " + refCount.get()
+ + "; size="+ providers.size()
+ + ": [" +
StringUtils.join(providers, ", ") + ']'
+ (lastProvider != null ? (" last provider: " + lastProvider) : "");
}
@@ -317,7 +316,7 @@ public final class AWSCredentialProviderList implements AWSCredentialsProvider,
}
// do this outside the synchronized block.
- for (AWSCredentialsProvider p : providers) {
+ for (AwsCredentialsProvider p : providers) {
if (p instanceof Closeable) {
IOUtils.closeStream((Closeable) p);
} else if (p instanceof AutoCloseable) {
@@ -333,4 +332,27 @@ public final class AWSCredentialProviderList implements AWSCredentialsProvider,
public int size() {
return providers.size();
}
+
+
+ /**
+ * Translate an exception if it or its inner exception is an
+ * {@link CredentialInitializationException}.
+ * If this condition is not met, null is returned.
+ * @param path path of operation.
+ * @param throwable exception
+ * @return a translated exception or null.
+ */
+ public static IOException maybeTranslateCredentialException(String path,
+ Throwable throwable) {
+ if (throwable instanceof CredentialInitializationException) {
+ // the exception raised by AWSCredentialProvider list if the
+ // credentials were not accepted,
+ return (AccessDeniedException)new AccessDeniedException(path, null,
+ throwable.toString()).initCause(throwable);
+ } else if (throwable.getCause() instanceof CredentialInitializationException) {
+ return maybeTranslateCredentialException(path, throwable.getCause());
+ } else {
+ return null;
+ }
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSNoResponseException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSNoResponseException.java
index e6a23b2361da..b8562714b1aa 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSNoResponseException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSNoResponseException.java
@@ -18,14 +18,19 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.AmazonServiceException;
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
/**
* Status code 443, no response from server. This is considered idempotent.
*/
public class AWSNoResponseException extends AWSServiceIOException {
public AWSNoResponseException(String operation,
- AmazonServiceException cause) {
+ AwsServiceException cause) {
super(operation, cause);
}
+
+ @Override
+ public boolean retryable() {
+ return true;
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSRedirectException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSRedirectException.java
index bb337ee5eebd..cb478482a8ed 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSRedirectException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSRedirectException.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.AmazonServiceException;
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
/**
* Request is redirected.
@@ -32,7 +32,7 @@ public class AWSRedirectException extends AWSServiceIOException {
* @param cause the underlying cause
*/
public AWSRedirectException(String operation,
- AmazonServiceException cause) {
+ AwsServiceException cause) {
super(operation, cause);
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSS3IOException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSS3IOException.java
index 014d217b6a4f..de1dd8b4a7a1 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSS3IOException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSS3IOException.java
@@ -18,14 +18,13 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.services.s3.model.AmazonS3Exception;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import java.util.Map;
-
/**
- * Wrap a {@link AmazonS3Exception} as an IOE, relaying all
+ * Wrap a {@link S3Exception} as an IOE, relaying all
* getters.
*/
@InterfaceAudience.Public
@@ -38,24 +37,12 @@ public class AWSS3IOException extends AWSServiceIOException {
* @param cause the underlying cause
*/
public AWSS3IOException(String operation,
- AmazonS3Exception cause) {
+ S3Exception cause) {
super(operation, cause);
}
- public AmazonS3Exception getCause() {
- return (AmazonS3Exception) super.getCause();
- }
-
- public String getErrorResponseXml() {
- return getCause().getErrorResponseXml();
- }
-
- public Map<String, String> getAdditionalDetails() {
- return getCause().getAdditionalDetails();
- }
-
- public String getExtendedRequestId() {
- return getCause().getExtendedRequestId();
+ public S3Exception getCause() {
+ return (S3Exception) super.getCause();
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceIOException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceIOException.java
index a9c2c9840203..434ec8df292a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceIOException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceIOException.java
@@ -18,13 +18,15 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.AmazonServiceException;
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
+import software.amazon.awssdk.awscore.exception.AwsErrorDetails;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+
/**
* A specific exception from AWS operations.
- * The exception must always be created with an {@link AmazonServiceException}.
+ * The exception must always be created with an {@link AwsServiceException}.
* The attributes of this exception can all be directly accessed.
*/
@InterfaceAudience.Public
@@ -37,36 +39,27 @@ public class AWSServiceIOException extends AWSClientIOException {
* @param cause the underlying cause
*/
public AWSServiceIOException(String operation,
- AmazonServiceException cause) {
+ AwsServiceException cause) {
super(operation, cause);
}
- public AmazonServiceException getCause() {
- return (AmazonServiceException) super.getCause();
- }
-
- public String getRequestId() {
- return getCause().getRequestId();
+ public AwsServiceException getCause() {
+ return (AwsServiceException) super.getCause();
}
- public String getServiceName() {
- return getCause().getServiceName();
+ public String requestId() {
+ return getCause().requestId();
}
- public String getErrorCode() {
- return getCause().getErrorCode();
+ public AwsErrorDetails awsErrorDetails() {
+ return getCause().awsErrorDetails();
}
- public int getStatusCode() {
- return getCause().getStatusCode();
+ public int statusCode() {
+ return getCause().statusCode();
}
- public String getRawResponseContent() {
- return getCause().getRawResponseContent();
+ public String extendedRequestId() {
+ return getCause().extendedRequestId();
}
-
- public boolean isRetryable() {
- return getCause().isRetryable();
- }
-
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceThrottledException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceThrottledException.java
index 131cea756224..5cd2eb9d320f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceThrottledException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceThrottledException.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.AmazonServiceException;
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
/**
* Exception raised when a service was throttled.
@@ -36,7 +36,12 @@ public class AWSServiceThrottledException extends AWSServiceIOException {
* @param cause the underlying cause
*/
public AWSServiceThrottledException(String operation,
- AmazonServiceException cause) {
+ AwsServiceException cause) {
super(operation, cause);
}
+
+ @Override
+ public boolean retryable() {
+ return true;
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java
index 83be294fac7c..ecfe5da1455c 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.AmazonServiceException;
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
/**
* A 500 response came back from a service.
@@ -31,7 +31,12 @@ import com.amazonaws.AmazonServiceException;
*/
public class AWSStatus500Exception extends AWSServiceIOException {
public AWSStatus500Exception(String operation,
- AmazonServiceException cause) {
+ AwsServiceException cause) {
super(operation, cause);
}
+
+ @Override
+ public boolean retryable() {
+ return true;
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java
index 564c03bf731d..dcfc2a03b123 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java
@@ -18,9 +18,10 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.AnonymousAWSCredentials;
-import com.amazonaws.auth.AWSCredentials;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -35,23 +36,18 @@ import org.apache.hadoop.classification.InterfaceStability;
* property fs.s3a.aws.credentials.provider. Therefore, changing the class name
* would be a backward-incompatible change.
*
- * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
- * as part of upgrading S3A to SDK V2. See HADOOP-18073.
*/
@InterfaceAudience.Private
@InterfaceStability.Stable
-@Deprecated
-public class AnonymousAWSCredentialsProvider implements AWSCredentialsProvider {
+public class AnonymousAWSCredentialsProvider implements AwsCredentialsProvider {
public static final String NAME
= "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider";
- public AWSCredentials getCredentials() {
- return new AnonymousAWSCredentials();
+ public AwsCredentials resolveCredentials() {
+ return AnonymousCredentialsProvider.create().resolveCredentials();
}
- public void refresh() {}
-
@Override
public String toString() {
return getClass().getSimpleName();
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java
index a85f26223ffc..98745b295b5d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.fs.s3a;
import javax.annotation.Nonnull;
-import com.amazonaws.arn.Arn;
+import software.amazon.awssdk.arns.Arn;
/**
* Represents an Arn Resource, this can be an accesspoint or bucket.
@@ -126,14 +126,14 @@ public final class ArnResource {
public static ArnResource accessPointFromArn(String arn) throws IllegalArgumentException {
Arn parsed = Arn.fromString(arn);
- if (parsed.getRegion().isEmpty() || parsed.getAccountId().isEmpty() ||
- parsed.getResourceAsString().isEmpty()) {
+ if (!parsed.region().isPresent() || !parsed.accountId().isPresent() ||
+ parsed.resourceAsString().isEmpty()) {
throw new IllegalArgumentException(
String.format("Access Point Arn %s has an invalid format or missing properties", arn));
}
- String resourceName = parsed.getResource().getResource();
- return new ArnResource(resourceName, parsed.getAccountId(), parsed.getRegion(),
- parsed.getPartition(), arn);
+ String resourceName = parsed.resource().resource();
+ return new ArnResource(resourceName, parsed.accountId().get(), parsed.region().get(),
+ parsed.partition(), arn);
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index edd63b5f263c..e52e4a711a6d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -140,7 +140,6 @@ public final class Constants {
public static final String ASSUMED_ROLE_POLICY =
"fs.s3a.assumed.role.policy";
- @SuppressWarnings("deprecation")
public static final String ASSUMED_ROLE_CREDENTIALS_DEFAULT =
SimpleAWSCredentialsProvider.NAME;
@@ -597,7 +596,7 @@ public final class Constants {
public static final String SIGNING_ALGORITHM_STS =
"fs.s3a." + Constants.AWS_SERVICE_IDENTIFIER_STS.toLowerCase()
- + "signing-algorithm";
+ + ".signing-algorithm";
public static final String S3N_FOLDER_SUFFIX = "_$folder$";
public static final String FS_S3A_BLOCK_SIZE = "fs.s3a.block.size";
@@ -739,14 +738,21 @@ public final class Constants {
public static final String STREAM_READ_GAUGE_INPUT_POLICY =
"stream_read_gauge_input_policy";
+ /**
+ * S3 Client Factory implementation class: {@value}.
+ * Unstable and incompatible between v1 and v2 SDK versions.
+ */
@InterfaceAudience.Private
@InterfaceStability.Unstable
public static final String S3_CLIENT_FACTORY_IMPL =
"fs.s3a.s3.client.factory.impl";
+ /**
+ * Default factory:
+ * {@code org.apache.hadoop.fs.s3a.DefaultS3ClientFactory}.
+ */
@InterfaceAudience.Private
@InterfaceStability.Unstable
- @SuppressWarnings("deprecation")
public static final Class<? extends S3ClientFactory>
DEFAULT_S3_CLIENT_FACTORY_IMPL =
DefaultS3ClientFactory.class;
@@ -1299,4 +1305,8 @@ public final class Constants {
*/
public static final int DEFAULT_PREFETCH_MAX_BLOCKS_COUNT = 4;
+ /**
+ * The bucket region header.
+ */
+ public static final String BUCKET_REGION_HEADER = "x-amz-bucket-region";
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CredentialInitializationException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CredentialInitializationException.java
index 2f0cfd37ad37..c6a1b8fcc713 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CredentialInitializationException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CredentialInitializationException.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.AmazonClientException;
+import software.amazon.awssdk.core.exception.SdkClientException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -26,22 +26,22 @@ import org.apache.hadoop.classification.InterfaceStability;
/**
* Exception which Hadoop's AWSCredentialsProvider implementations should
* throw when there is a problem with the credential setup. This
- * is a subclass of {@link AmazonClientException} which sets
- * {@link #isRetryable()} to false, so as to fail fast.
+ * is a subclass of {@link SdkClientException} which sets
+ * {@link #retryable()} to false, so as to fail fast.
* This is used in credential providers and elsewhere.
* When passed through {@code S3AUtils.translateException()} it
- * is mapped to an AccessDeniedException. As a result, the Invoker
- * code will automatically translate
+ * is mapped to an AccessDeniedException.
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
-public class CredentialInitializationException extends AmazonClientException {
+public class CredentialInitializationException extends SdkClientException {
+
public CredentialInitializationException(String message, Throwable t) {
- super(message, t);
+ super(builder().message(message).cause(t));
}
public CredentialInitializationException(String message) {
- super(message);
+ super(builder().message(message));
}
/**
@@ -49,7 +49,7 @@ public class CredentialInitializationException extends AmazonClientException {
* @return false, always.
*/
@Override
- public boolean isRetryable() {
+ public boolean retryable() {
return false;
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
index bab4ec896ff7..98c72d276628 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
@@ -20,32 +20,26 @@ package org.apache.hadoop.fs.s3a;
import java.io.IOException;
import java.net.URI;
+import java.net.URISyntaxException;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.SdkClientException;
-import com.amazonaws.client.builder.AwsClientBuilder;
-import com.amazonaws.handlers.RequestHandler2;
-import com.amazonaws.regions.RegionUtils;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3Builder;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.AmazonS3ClientBuilder;
-import com.amazonaws.services.s3.AmazonS3EncryptionClientV2Builder;
-import com.amazonaws.services.s3.AmazonS3EncryptionV2;
-import com.amazonaws.services.s3.S3ClientOptions;
-import com.amazonaws.services.s3.internal.ServiceUtils;
-import com.amazonaws.services.s3.model.CryptoConfigurationV2;
-import com.amazonaws.services.s3.model.CryptoMode;
-import com.amazonaws.services.s3.model.CryptoRangeGetMode;
-import com.amazonaws.services.s3.model.EncryptionMaterialsProvider;
-import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider;
-import com.amazonaws.util.AwsHostNameUtils;
-import com.amazonaws.util.RuntimeHttpUtils;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.fs.s3a.impl.AWSClientConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
+import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
+import software.amazon.awssdk.core.retry.RetryPolicy;
+import software.amazon.awssdk.http.apache.ApacheHttpClient;
+import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.S3BaseClientBuilder;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3Configuration;
+import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration;
+import software.amazon.awssdk.transfer.s3.S3TransferManager;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -54,15 +48,11 @@ import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.s3a.statistics.impl.AwsStatisticsCollector;
import org.apache.hadoop.fs.store.LogExactlyOnce;
-import static com.amazonaws.services.s3.Headers.REQUESTER_PAYS_HEADER;
-import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION;
-import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_CENTRAL_REGION;
-import static org.apache.hadoop.fs.s3a.Constants.EXPERIMENTAL_AWS_INTERNAL_THROTTLING;
-import static org.apache.hadoop.fs.s3a.Constants.EXPERIMENTAL_AWS_INTERNAL_THROTTLING_DEFAULT;
-import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY;
-import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm;
-import static org.apache.hadoop.fs.s3a.S3AUtils.getS3EncryptionKey;
-import static org.apache.hadoop.fs.s3a.S3AUtils.translateException;
+import static org.apache.hadoop.fs.s3a.impl.AWSHeaders.REQUESTER_PAYS_HEADER;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_SECURE_CONNECTIONS;
+import static org.apache.hadoop.fs.s3a.Constants.SECURE_CONNECTIONS;
+import static org.apache.hadoop.fs.s3a.Constants.AWS_SERVICE_IDENTIFIER_S3;
+
/**
* The default {@link S3ClientFactory} implementation.
@@ -71,12 +61,9 @@ import static org.apache.hadoop.fs.s3a.S3AUtils.translateException;
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-@SuppressWarnings("deprecation")
public class DefaultS3ClientFactory extends Configured
implements S3ClientFactory {
- private static final String S3_SERVICE_NAME = "s3";
-
private static final String REQUESTER_PAYS_HEADER_VALUE = "requester";
/**
@@ -85,310 +72,161 @@ public class DefaultS3ClientFactory extends Configured
protected static final Logger LOG =
LoggerFactory.getLogger(DefaultS3ClientFactory.class);
- /**
- * A one-off warning of default region chains in use.
- */
- private static final LogExactlyOnce WARN_OF_DEFAULT_REGION_CHAIN =
- new LogExactlyOnce(LOG);
-
- /**
- * Warning message printed when the SDK Region chain is in use.
- */
- private static final String SDK_REGION_CHAIN_IN_USE =
- "S3A filesystem client is using"
- + " the SDK region resolution chain.";
/** Exactly once log to inform about ignoring the AWS-SDK Warnings for CSE. */
private static final LogExactlyOnce IGNORE_CSE_WARN = new LogExactlyOnce(LOG);
- /** Bucket name. */
- private String bucket;
-
- /**
- * Create the client by preparing the AwsConf configuration
- * and then invoking {@code buildAmazonS3Client()}.
- */
@Override
- public AmazonS3 createS3Client(
+ public S3Client createS3Client(
final URI uri,
final S3ClientCreationParameters parameters) throws IOException {
- Configuration conf = getConf();
- bucket = uri.getHost();
- final ClientConfiguration awsConf = S3AUtils
- .createAwsConf(conf,
- bucket,
- Constants.AWS_SERVICE_IDENTIFIER_S3);
- // add any headers
- parameters.getHeaders().forEach((h, v) ->
- awsConf.addHeader(h, v));
- if (parameters.isRequesterPays()) {
- // All calls must acknowledge requester will pay via header.
- awsConf.addHeader(REQUESTER_PAYS_HEADER, REQUESTER_PAYS_HEADER_VALUE);
- }
+ Configuration conf = getConf();
+ String bucket = uri.getHost();
+
+ ApacheHttpClient.Builder httpClientBuilder = AWSClientConfig
+ .createHttpClientBuilder(conf)
+ .proxyConfiguration(AWSClientConfig.createProxyConfiguration(conf, bucket));
+ return configureClientBuilder(S3Client.builder(), parameters, conf, bucket)
+ .httpClientBuilder(httpClientBuilder)
+ .build();
+ }
- // When EXPERIMENTAL_AWS_INTERNAL_THROTTLING is false
- // throttling is explicitly disabled on the S3 client so that
- // all failures are collected in S3A instrumentation, and its
- // retry policy is the only one used.
- // This may cause problems in copy/rename.
- awsConf.setUseThrottleRetries(
- conf.getBoolean(EXPERIMENTAL_AWS_INTERNAL_THROTTLING,
- EXPERIMENTAL_AWS_INTERNAL_THROTTLING_DEFAULT));
+ @Override
+ public S3AsyncClient createS3AsyncClient(
+ final URI uri,
+ final S3ClientCreationParameters parameters) throws IOException {
- if (!StringUtils.isEmpty(parameters.getUserAgentSuffix())) {
- awsConf.setUserAgentSuffix(parameters.getUserAgentSuffix());
- }
+ Configuration conf = getConf();
+ String bucket = uri.getHost();
+
+ NettyNioAsyncHttpClient.Builder httpClientBuilder = AWSClientConfig
+ .createAsyncHttpClientBuilder(conf)
+ .proxyConfiguration(AWSClientConfig.createAsyncProxyConfiguration(conf, bucket));
+
+ MultipartConfiguration multipartConfiguration = MultipartConfiguration.builder()
+ .minimumPartSizeInBytes(parameters.getMinimumPartSize())
+ .thresholdInBytes(parameters.getMultiPartThreshold())
+ .build();
+
+ return configureClientBuilder(S3AsyncClient.builder(), parameters, conf, bucket)
+ .httpClientBuilder(httpClientBuilder)
+ .multipartConfiguration(multipartConfiguration)
+ .multipartEnabled(true)
+ .build();
+ }
- // Get the encryption method for this bucket.
- S3AEncryptionMethods encryptionMethods =
- getEncryptionAlgorithm(bucket, conf);
- try {
- // If CSE is enabled then build a S3EncryptionClient.
- if (S3AEncryptionMethods.CSE_KMS.getMethod()
- .equals(encryptionMethods.getMethod())) {
- return buildAmazonS3EncryptionClient(
- awsConf,
- parameters);
- } else {
- return buildAmazonS3Client(
- awsConf,
- parameters);
- }
- } catch (SdkClientException e) {
- // SDK refused to build.
- throw translateException("creating AWS S3 client", uri.toString(), e);
- }
+ @Override
+ public S3TransferManager createS3TransferManager(final S3AsyncClient s3AsyncClient) {
+ return S3TransferManager.builder()
+ .s3Client(s3AsyncClient)
+ .build();
}
/**
- * Create an {@link AmazonS3} client of type
- * {@link AmazonS3EncryptionV2} if CSE is enabled.
- *
- * @param awsConf AWS configuration.
- * @param parameters parameters.
- *
- * @return new AmazonS3 client.
- * @throws IOException if lookupPassword() has any problem.
+ * Configure a sync or async S3 client builder.
+ * This method handles all shared configuration.
+ * @param builder S3 client builder
+ * @param parameters parameter object
+ * @param conf configuration object
+ * @param bucket bucket name
+ * @return the builder object
+ * @param <BuilderT> S3 client builder type
+ * @param <ClientT> S3 client type
*/
- protected AmazonS3 buildAmazonS3EncryptionClient(
- final ClientConfiguration awsConf,
- final S3ClientCreationParameters parameters) throws IOException {
+ private <BuilderT extends S3BaseClientBuilder<BuilderT, ClientT>, ClientT> BuilderT configureClientBuilder(
+ BuilderT builder, S3ClientCreationParameters parameters, Configuration conf, String bucket)
+ throws IOException {
- AmazonS3 client;
- AmazonS3EncryptionClientV2Builder builder =
- new AmazonS3EncryptionClientV2Builder();
- Configuration conf = getConf();
+ Region region = parameters.getRegion();
+ LOG.debug("Using region {}", region);
- // CSE-KMS Method
- String kmsKeyId = getS3EncryptionKey(bucket, conf, true);
- // Check if kmsKeyID is not null
- Preconditions.checkArgument(!StringUtils.isBlank(kmsKeyId), "CSE-KMS "
- + "method requires KMS key ID. Use " + S3_ENCRYPTION_KEY
- + " property to set it. ");
-
- EncryptionMaterialsProvider materialsProvider =
- new KMSEncryptionMaterialsProvider(kmsKeyId);
- builder.withEncryptionMaterialsProvider(materialsProvider);
- //Configure basic params of a S3 builder.
- configureBasicParams(builder, awsConf, parameters);
-
- // Configuring endpoint.
- AmazonS3EncryptionClientV2Builder.EndpointConfiguration epr
- = createEndpointConfiguration(parameters.getEndpoint(),
- awsConf, getConf().getTrimmed(AWS_REGION));
- configureEndpoint(builder, epr);
-
- // Create cryptoConfig.
- CryptoConfigurationV2 cryptoConfigurationV2 =
- new CryptoConfigurationV2(CryptoMode.AuthenticatedEncryption)
- .withRangeGetMode(CryptoRangeGetMode.ALL);
- if (epr != null) {
- cryptoConfigurationV2
- .withAwsKmsRegion(RegionUtils.getRegion(epr.getSigningRegion()));
- LOG.debug("KMS region used: {}", cryptoConfigurationV2.getAwsKmsRegion());
+ URI endpoint = getS3Endpoint(parameters.getEndpoint(), conf);
+
+ if (endpoint != null) {
+ builder.endpointOverride(endpoint);
+ LOG.debug("Using endpoint {}", endpoint);
}
- builder.withCryptoConfiguration(cryptoConfigurationV2);
- client = builder.build();
- IGNORE_CSE_WARN.info("S3 client-side encryption enabled: Ignore S3-CSE "
- + "Warnings.");
- return client;
- }
+ S3Configuration serviceConfiguration = S3Configuration.builder()
+ .pathStyleAccessEnabled(parameters.isPathStyleAccess())
+ .build();
- /**
- * Use the Builder API to create an AWS S3 client.
- * <p>
- * This has a more complex endpoint configuration mechanism
- * which initially caused problems; the
- * {@code withForceGlobalBucketAccessEnabled(true)}
- * command is critical here.
- * @param awsConf AWS configuration
- * @param parameters parameters
- * @return new AmazonS3 client
- * @throws SdkClientException if the configuration is invalid.
- */
- protected AmazonS3 buildAmazonS3Client(
- final ClientConfiguration awsConf,
- final S3ClientCreationParameters parameters) {
- AmazonS3ClientBuilder b = AmazonS3Client.builder();
- configureBasicParams(b, awsConf, parameters);
-
- // endpoint set up is a PITA
- AwsClientBuilder.EndpointConfiguration epr
- = createEndpointConfiguration(parameters.getEndpoint(),
- awsConf, getConf().getTrimmed(AWS_REGION));
- configureEndpoint(b, epr);
- final AmazonS3 client = b.build();
- return client;
+ return builder
+ .overrideConfiguration(createClientOverrideConfiguration(parameters, conf))
+ .credentialsProvider(parameters.getCredentialSet())
+ .region(region)
+ .serviceConfiguration(serviceConfiguration);
}
/**
- * A method to configure basic AmazonS3Builder parameters.
- *
- * @param builder Instance of AmazonS3Builder used.
- * @param awsConf ClientConfiguration used.
- * @param parameters Parameters used to set in the builder.
+ * Create an override configuration for an S3 client.
+ * @param parameters parameter object
+ * @param conf configuration object
+ * @throws IOException any IOE raised, or translated exception
+ * @return the override configuration
*/
- private void configureBasicParams(AmazonS3Builder builder,
- ClientConfiguration awsConf, S3ClientCreationParameters parameters) {
- builder.withCredentials(parameters.getCredentialSet());
- builder.withClientConfiguration(awsConf);
- builder.withPathStyleAccessEnabled(parameters.isPathStyleAccess());
+ protected ClientOverrideConfiguration createClientOverrideConfiguration(
+ S3ClientCreationParameters parameters, Configuration conf) throws IOException {
+ final ClientOverrideConfiguration.Builder clientOverrideConfigBuilder =
+ AWSClientConfig.createClientConfigBuilder(conf, AWS_SERVICE_IDENTIFIER_S3);
- if (parameters.getMetrics() != null) {
- builder.withMetricsCollector(
- new AwsStatisticsCollector(parameters.getMetrics()));
- }
- if (parameters.getRequestHandlers() != null) {
- builder.withRequestHandlers(
- parameters.getRequestHandlers().toArray(new RequestHandler2[0]));
- }
- if (parameters.getMonitoringListener() != null) {
- builder.withMonitoringListener(parameters.getMonitoringListener());
- }
+ // add any headers
+ parameters.getHeaders().forEach((h, v) -> clientOverrideConfigBuilder.putHeader(h, v));
- }
+ if (parameters.isRequesterPays()) {
+ // All calls must acknowledge requester will pay via header.
+ clientOverrideConfigBuilder.putHeader(REQUESTER_PAYS_HEADER, REQUESTER_PAYS_HEADER_VALUE);
+ }
- /**
- * A method to configure endpoint and Region for an AmazonS3Builder.
- *
- * @param builder Instance of AmazonS3Builder used.
- * @param epr EndpointConfiguration used to set in builder.
- */
- private void configureEndpoint(
- AmazonS3Builder builder,
- AmazonS3Builder.EndpointConfiguration epr) {
- if (epr != null) {
- // an endpoint binding was constructed: use it.
- builder.withEndpointConfiguration(epr);
- } else {
- // no idea what the endpoint is, so tell the SDK
- // to work it out at the cost of an extra HEAD request
- builder.withForceGlobalBucketAccessEnabled(true);
- // HADOOP-17771 force set the region so the build process doesn't halt.
- String region = getConf().getTrimmed(AWS_REGION, AWS_S3_CENTRAL_REGION);
- LOG.debug("fs.s3a.endpoint.region=\"{}\"", region);
- if (!region.isEmpty()) {
- // there's either an explicit region or we have fallen back
- // to the central one.
- LOG.debug("Using default endpoint; setting region to {}", region);
- builder.setRegion(region);
- } else {
- // no region.
- // allow this if people really want it; it is OK to rely on this
- // when deployed in EC2.
- WARN_OF_DEFAULT_REGION_CHAIN.warn(SDK_REGION_CHAIN_IN_USE);
- LOG.debug(SDK_REGION_CHAIN_IN_USE);
- }
+ if (!StringUtils.isEmpty(parameters.getUserAgentSuffix())) {
+ clientOverrideConfigBuilder.putAdvancedOption(SdkAdvancedClientOption.USER_AGENT_SUFFIX,
+ parameters.getUserAgentSuffix());
}
- }
- /**
- * Configure classic S3 client.
- * <p>
- * This includes: endpoint, Path Access and possibly other
- * options.
- *
- * @param s3 S3 Client.
- * @param endPoint s3 endpoint, may be empty
- * @param pathStyleAccess enable path style access?
- * @return S3 client
- * @throws IllegalArgumentException if misconfigured
- */
- protected static AmazonS3 configureAmazonS3Client(AmazonS3 s3,
- final String endPoint,
- final boolean pathStyleAccess)
- throws IllegalArgumentException {
- if (!endPoint.isEmpty()) {
- try {
- s3.setEndpoint(endPoint);
- } catch (IllegalArgumentException e) {
- String msg = "Incorrect endpoint: " + e.getMessage();
- LOG.error(msg);
- throw new IllegalArgumentException(msg, e);
+ if (parameters.getExecutionInterceptors() != null) {
+ for (ExecutionInterceptor interceptor : parameters.getExecutionInterceptors()) {
+ clientOverrideConfigBuilder.addExecutionInterceptor(interceptor);
}
}
- if (pathStyleAccess) {
- LOG.debug("Enabling path style access!");
- s3.setS3ClientOptions(S3ClientOptions.builder()
- .setPathStyleAccess(true)
- .build());
+
+ if (parameters.getMetrics() != null) {
+ clientOverrideConfigBuilder.addMetricPublisher(
+ new AwsStatisticsCollector(parameters.getMetrics()));
}
- return s3;
+
+ final RetryPolicy.Builder retryPolicyBuilder = AWSClientConfig.createRetryPolicyBuilder(conf);
+ clientOverrideConfigBuilder.retryPolicy(retryPolicyBuilder.build());
+
+ return clientOverrideConfigBuilder.build();
}
/**
- * Given an endpoint string, return an endpoint config, or null, if none
- * is needed.
- * <p>
- * This is a pretty painful piece of code. It is trying to replicate
- * what AwsClient.setEndpoint() does, because you can't
- * call that setter on an AwsClient constructed via
- * the builder, and you can't pass a metrics collector
- * down except through the builder.
- * <p>
- * Note also that AWS signing is a mystery which nobody fully
- * understands, especially given all problems surface in a
- * "400 bad request" response, which, like all security systems,
- * provides minimal diagnostics out of fear of leaking
- * secrets.
+ * Given a endpoint string, create the endpoint URI.
*
* @param endpoint possibly null endpoint.
- * @param awsConf config to build the URI from.
- * @param awsRegion AWS S3 Region if the corresponding config is set.
- * @return a configuration for the S3 client builder.
+ * @param conf config to build the URI from.
+ * @return an endpoint uri
*/
- @VisibleForTesting
- public static AwsClientBuilder.EndpointConfiguration
- createEndpointConfiguration(
- final String endpoint, final ClientConfiguration awsConf,
- String awsRegion) {
- LOG.debug("Creating endpoint configuration for \"{}\"", endpoint);
+ private static URI getS3Endpoint(String endpoint, final Configuration conf) {
+
+ boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS);
+
+ String protocol = secureConnections ? "https" : "http";
+
if (endpoint == null || endpoint.isEmpty()) {
- // the default endpoint...we should be using null at this point.
- LOG.debug("Using default endpoint -no need to generate a configuration");
+ // don't set an endpoint if none is configured, instead let the SDK figure it out.
return null;
}
- final URI epr = RuntimeHttpUtils.toUri(endpoint, awsConf);
- LOG.debug("Endpoint URI = {}", epr);
- String region = awsRegion;
- if (StringUtils.isBlank(region)) {
- if (!ServiceUtils.isS3USStandardEndpoint(endpoint)) {
- LOG.debug("Endpoint {} is not the default; parsing", epr);
- region = AwsHostNameUtils.parseRegion(
- epr.getHost(),
- S3_SERVICE_NAME);
- } else {
- // US-east, set region == null.
- LOG.debug("Endpoint {} is the standard one; declare region as null",
- epr);
- region = null;
- }
+ if (!endpoint.contains("://")) {
+ endpoint = String.format("%s://%s", protocol, endpoint);
+ }
+
+ try {
+ return new URI(endpoint);
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException(e);
}
- LOG.debug("Region for endpoint {}, URI {} is determined as {}",
- endpoint, epr, region);
- return new AwsClientBuilder.EndpointConfiguration(endpoint, region);
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/FailureInjectionPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/FailureInjectionPolicy.java
index f3e00cc61d76..f5c9c43a49cb 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/FailureInjectionPolicy.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/FailureInjectionPolicy.java
@@ -36,7 +36,7 @@ public class FailureInjectionPolicy {
public static final String DEFAULT_DELAY_KEY_SUBSTRING = "DELAY_LISTING_ME";
private static final Logger LOG =
- LoggerFactory.getLogger(InconsistentAmazonS3Client.class);
+ LoggerFactory.getLogger(FailureInjectionPolicy.class);
/**
* Probability of throttling a request.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java
deleted file mode 100644
index 317f2c785763..000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java
+++ /dev/null
@@ -1,345 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import java.util.List;
-import java.util.concurrent.atomic.AtomicLong;
-
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.AmazonServiceException;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.SdkClientException;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
-import com.amazonaws.services.s3.model.DeleteObjectRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsResult;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
-import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ListObjectsV2Request;
-import com.amazonaws.services.s3.model.ListObjectsV2Result;
-import com.amazonaws.services.s3.model.MultipartUploadListing;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.PutObjectResult;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.amazonaws.services.s3.model.UploadPartRequest;
-import com.amazonaws.services.s3.model.UploadPartResult;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * A wrapper around {@link com.amazonaws.services.s3.AmazonS3} that injects
- * failures.
- * It used to also inject inconsistency, but this was removed with S3Guard;
- * what is retained is the ability to throttle AWS operations and for the
- * input stream to be inconsistent.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class InconsistentAmazonS3Client extends AmazonS3Client {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(InconsistentAmazonS3Client.class);
-
- private FailureInjectionPolicy policy;
-
- /**
- * Counter of failures since last reset.
- */
- private final AtomicLong failureCounter = new AtomicLong(0);
-
-
- /**
- * Instantiate.
- * This subclasses a deprecated constructor of the parent
- * {@code AmazonS3Client} class; we can't use the builder API because,
- * that only creates the consistent client.
- * @param credentials credentials to auth.
- * @param clientConfiguration connection settings
- * @param conf hadoop configuration.
- */
- @SuppressWarnings("deprecation")
- public InconsistentAmazonS3Client(AWSCredentialsProvider credentials,
- ClientConfiguration clientConfiguration, Configuration conf) {
- super(credentials, clientConfiguration);
- policy = new FailureInjectionPolicy(conf);
- }
-
- /**
- * A way for tests to patch in a different fault injection policy at runtime.
- * @param fs filesystem under test
- * @param policy failure injection settings to set
- * @throws Exception on failure
- */
- public static void setFailureInjectionPolicy(S3AFileSystem fs,
- FailureInjectionPolicy policy) throws Exception {
- AmazonS3 s3 = fs.getAmazonS3ClientForTesting("s3guard");
- InconsistentAmazonS3Client ic = InconsistentAmazonS3Client.castFrom(s3);
- ic.replacePolicy(policy);
- }
-
- private void replacePolicy(FailureInjectionPolicy pol) {
- this.policy = pol;
- }
-
- @Override
- public String toString() {
- return String.format("Inconsistent S3 Client: %s; failure count %d",
- policy, failureCounter.get());
- }
-
- /**
- * Convenience function for test code to cast from supertype.
- * @param c supertype to cast from
- * @return subtype, not null
- * @throws Exception on error
- */
- public static InconsistentAmazonS3Client castFrom(AmazonS3 c) throws
- Exception {
- InconsistentAmazonS3Client ic = null;
- if (c instanceof InconsistentAmazonS3Client) {
- ic = (InconsistentAmazonS3Client) c;
- }
- Preconditions.checkNotNull(ic, "Not an instance of " +
- "InconsistentAmazonS3Client");
- return ic;
- }
-
- @Override
- public DeleteObjectsResult deleteObjects(DeleteObjectsRequest
- deleteObjectsRequest)
- throws AmazonClientException, AmazonServiceException {
- maybeFail();
- return super.deleteObjects(deleteObjectsRequest);
- }
-
- @Override
- public void deleteObject(DeleteObjectRequest deleteObjectRequest)
- throws AmazonClientException, AmazonServiceException {
- String key = deleteObjectRequest.getKey();
- LOG.debug("key {}", key);
- maybeFail();
- super.deleteObject(deleteObjectRequest);
- }
-
- /* We should only need to override this version of putObject() */
- @Override
- public PutObjectResult putObject(PutObjectRequest putObjectRequest)
- throws AmazonClientException, AmazonServiceException {
- LOG.debug("key {}", putObjectRequest.getKey());
- maybeFail();
- return super.putObject(putObjectRequest);
- }
-
- /* We should only need to override these versions of listObjects() */
- @Override
- public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
- throws AmazonClientException, AmazonServiceException {
- maybeFail();
- return super.listObjects(listObjectsRequest);
- }
-
- /* consistent listing with possibility of failing. */
- @Override
- public ListObjectsV2Result listObjectsV2(ListObjectsV2Request request)
- throws AmazonClientException, AmazonServiceException {
- maybeFail();
- return super.listObjectsV2(request);
- }
-
-
- @Override
- public CompleteMultipartUploadResult completeMultipartUpload(
- CompleteMultipartUploadRequest completeMultipartUploadRequest)
- throws SdkClientException, AmazonServiceException {
- maybeFail();
- return super.completeMultipartUpload(completeMultipartUploadRequest);
- }
-
- @Override
- public UploadPartResult uploadPart(UploadPartRequest uploadPartRequest)
- throws SdkClientException, AmazonServiceException {
- maybeFail();
- return super.uploadPart(uploadPartRequest);
- }
-
- @Override
- public InitiateMultipartUploadResult initiateMultipartUpload(
- InitiateMultipartUploadRequest initiateMultipartUploadRequest)
- throws SdkClientException, AmazonServiceException {
- maybeFail();
- return super.initiateMultipartUpload(initiateMultipartUploadRequest);
- }
-
- @Override
- public MultipartUploadListing listMultipartUploads(
- ListMultipartUploadsRequest listMultipartUploadsRequest)
- throws SdkClientException, AmazonServiceException {
- maybeFail();
- return super.listMultipartUploads(listMultipartUploadsRequest);
- }
-
- /**
- * Set the probability of throttling a request.
- * @param throttleProbability the probability of a request being throttled.
- */
- public void setThrottleProbability(float throttleProbability) {
- policy.setThrottleProbability(throttleProbability);
- }
-
- /**
- * Conditionally fail the operation.
- * @param errorMsg description of failure
- * @param statusCode http status code for error
- * @throws AmazonClientException if the client chooses to fail
- * the request.
- */
- private void maybeFail(String errorMsg, int statusCode)
- throws AmazonClientException {
- // code structure here is to line up for more failures later
- AmazonServiceException ex = null;
- if (FailureInjectionPolicy.trueWithProbability(policy.getThrottleProbability())) {
- // throttle the request
- ex = new AmazonServiceException(errorMsg
- + " count = " + (failureCounter.get() + 1), null);
- ex.setStatusCode(statusCode);
- }
-
- int failureLimit = policy.getFailureLimit();
- if (ex != null) {
- long count = failureCounter.incrementAndGet();
- if (failureLimit == 0
- || (failureLimit > 0 && count < failureLimit)) {
- throw ex;
- }
- }
- }
-
- private void maybeFail() {
- maybeFail("throttled", 503);
- }
-
- /**
- * Set the limit on failures before all operations pass through.
- * This resets the failure count.
- * @param limit limit; "0" means "no limit"
- */
- public void setFailureLimit(int limit) {
- policy.setFailureLimit(limit);
- failureCounter.set(0);
- }
-
- @Override
- public S3Object getObject(GetObjectRequest var1) throws SdkClientException,
- AmazonServiceException {
- maybeFail();
- return super.getObject(var1);
- }
-
- @Override
- public S3Object getObject(String bucketName, String key)
- throws SdkClientException, AmazonServiceException {
- maybeFail();
- return super.getObject(bucketName, key);
-
- }
-
- /** Since ObjectListing is immutable, we just override it with wrapper. */
- @SuppressWarnings("serial")
- private static class CustomObjectListing extends ObjectListing {
-
- private final List<S3ObjectSummary> customListing;
- private final List<String> customPrefixes;
-
- CustomObjectListing(ObjectListing rawListing,
- List<S3ObjectSummary> customListing,
- List<String> customPrefixes) {
- super();
- this.customListing = customListing;
- this.customPrefixes = customPrefixes;
-
- this.setBucketName(rawListing.getBucketName());
- this.setCommonPrefixes(rawListing.getCommonPrefixes());
- this.setDelimiter(rawListing.getDelimiter());
- this.setEncodingType(rawListing.getEncodingType());
- this.setMarker(rawListing.getMarker());
- this.setMaxKeys(rawListing.getMaxKeys());
- this.setNextMarker(rawListing.getNextMarker());
- this.setPrefix(rawListing.getPrefix());
- this.setTruncated(rawListing.isTruncated());
- }
-
- @Override
- public List<S3ObjectSummary> getObjectSummaries() {
- return customListing;
- }
-
- @Override
- public List<String> getCommonPrefixes() {
- return customPrefixes;
- }
- }
-
- @SuppressWarnings("serial")
- private static class CustomListObjectsV2Result extends ListObjectsV2Result {
-
- private final List<S3ObjectSummary> customListing;
- private final List<String> customPrefixes;
-
- CustomListObjectsV2Result(ListObjectsV2Result raw,
- List<S3ObjectSummary> customListing, List<String> customPrefixes) {
- super();
- this.customListing = customListing;
- this.customPrefixes = customPrefixes;
-
- this.setBucketName(raw.getBucketName());
- this.setCommonPrefixes(raw.getCommonPrefixes());
- this.setDelimiter(raw.getDelimiter());
- this.setEncodingType(raw.getEncodingType());
- this.setStartAfter(raw.getStartAfter());
- this.setMaxKeys(raw.getMaxKeys());
- this.setContinuationToken(raw.getContinuationToken());
- this.setPrefix(raw.getPrefix());
- this.setTruncated(raw.isTruncated());
- }
-
- @Override
- public List<S3ObjectSummary> getObjectSummaries() {
- return customListing;
- }
-
- @Override
- public List<String> getCommonPrefixes() {
- return customPrefixes;
- }
- }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentS3ClientFactory.java
deleted file mode 100644
index 4bfcc8aba3af..000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentS3ClientFactory.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.services.s3.AmazonS3;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * S3 Client factory used for testing with eventual consistency fault injection.
- * This client is for testing <i>only</i>; it is in the production
- * {@code hadoop-aws} module to enable integration tests to use this
- * just by editing the Hadoop configuration used to bring up the client.
- *
- * The factory uses the older constructor-based instantiation/configuration
- * of the client, so does not wire up metrics, handlers etc.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class InconsistentS3ClientFactory extends DefaultS3ClientFactory {
-
- @Override
- protected AmazonS3 buildAmazonS3Client(
- final ClientConfiguration awsConf,
- final S3ClientCreationParameters parameters) {
- LOG.warn("** FAILURE INJECTION ENABLED. Do not run in production! **");
- LOG.warn("List inconsistency is no longer emulated; only throttling and read errors");
- InconsistentAmazonS3Client s3
- = new InconsistentAmazonS3Client(
- parameters.getCredentialSet(), awsConf, getConf());
- configureAmazonS3Client(s3,
- parameters.getEndpoint(),
- parameters.isPathStyleAccess());
- return s3;
- }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java
index 67c8e7d809cc..9b2c95a90c76 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java
@@ -24,8 +24,7 @@ import java.util.Optional;
import java.util.concurrent.Future;
import javax.annotation.Nullable;
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.SdkBaseException;
+import software.amazon.awssdk.core.exception.SdkException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,6 +38,7 @@ import org.apache.hadoop.util.functional.FutureIO;
import org.apache.hadoop.util.functional.InvocationRaisingIOE;
import org.apache.hadoop.util.Preconditions;
+
import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.invokeTrackingDuration;
/**
@@ -120,7 +120,7 @@ public class Invoker {
throws IOException {
try (DurationInfo ignored = new DurationInfo(LOG, false, "%s", action)) {
return operation.apply();
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw S3AUtils.translateException(action, path, e);
}
}
@@ -145,7 +145,7 @@ public class Invoker {
throws IOException {
try {
return invokeTrackingDuration(tracker, operation);
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw S3AUtils.translateException(action, path, e);
}
}
@@ -170,7 +170,7 @@ public class Invoker {
/**
*
- * Wait for a future, translating AmazonClientException into an IOException.
+ * Wait for a future, translating SdkException into an IOException.
* @param action action to execute (used in error messages)
* @param path path of work (used in error messages)
* @param future future to await for
@@ -186,7 +186,7 @@ public class Invoker {
throws IOException {
try (DurationInfo ignored = new DurationInfo(LOG, false, "%s", action)) {
return FutureIO.awaitFuture(future);
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw S3AUtils.translateException(action, path, e);
}
}
@@ -444,7 +444,7 @@ public class Invoker {
* @param operation operation to execute
* @return the result of the call
* @throws IOException any IOE raised
- * @throws SdkBaseException any AWS exception raised
+ * @throws SdkException any AWS exception raised
* @throws RuntimeException : these are never caught and retries.
*/
@Retries.RetryRaw
@@ -466,7 +466,7 @@ public class Invoker {
}
// execute the operation, returning if successful
return operation.apply();
- } catch (IOException | SdkBaseException e) {
+ } catch (IOException | SdkException e) {
caught = e;
}
// you only get here if the operation didn't complete
@@ -479,7 +479,7 @@ public class Invoker {
translated = (IOException) caught;
} else {
translated = S3AUtils.translateException(text, "",
- (SdkBaseException)caught);
+ (SdkException) caught);
}
try {
@@ -518,11 +518,10 @@ public class Invoker {
if (caught instanceof IOException) {
throw (IOException) caught;
} else {
- throw (SdkBaseException) caught;
+ throw (SdkException) caught;
}
}
-
/**
* Execute an operation; any exception raised is simply caught and
* logged at debug.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
index 6c39cc4b6424..490deaaab04d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
@@ -18,7 +18,8 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
+import software.amazon.awssdk.services.s3.model.CommonPrefix;
+import software.amazon.awssdk.services.s3.model.S3Object;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.VisibleForTesting;
@@ -277,19 +278,19 @@ public class Listing extends AbstractStoreOperation {
}
/**
- * Interface to implement by the logic deciding whether to accept a summary
+ * Interface to implement the logic deciding whether to accept a s3Object
* entry or path as a valid file or directory.
*/
interface FileStatusAcceptor {
/**
- * Predicate to decide whether or not to accept a summary entry.
+ * Predicate to decide whether or not to accept a s3Object entry.
* @param keyPath qualified path to the entry
- * @param summary summary entry
+ * @param s3Object s3Object entry
* @return true if the entry is accepted (i.e. that a status entry
* should be generated.
*/
- boolean accept(Path keyPath, S3ObjectSummary summary);
+ boolean accept(Path keyPath, S3Object s3Object);
/**
* Predicate to decide whether or not to accept a prefix.
@@ -451,21 +452,21 @@ public class Listing extends AbstractStoreOperation {
int added = 0, ignored = 0;
// list to fill in with results. Initial size will be list maximum.
List<S3AFileStatus> stats = new ArrayList<>(
- objects.getObjectSummaries().size() +
+ objects.getS3Objects().size() +
objects.getCommonPrefixes().size());
// objects
- for (S3ObjectSummary summary : objects.getObjectSummaries()) {
- String key = summary.getKey();
+ for (S3Object s3Object : objects.getS3Objects()) {
+ String key = s3Object.key();
Path keyPath = getStoreContext().getContextAccessors().keyToPath(key);
if (LOG.isDebugEnabled()) {
- LOG.debug("{}: {}", keyPath, stringify(summary));
+ LOG.debug("{}: {}", keyPath, stringify(s3Object));
}
// Skip over keys that are ourselves and old S3N _$folder$ files
- if (acceptor.accept(keyPath, summary) && filter.accept(keyPath)) {
- S3AFileStatus status = createFileStatus(keyPath, summary,
+ if (acceptor.accept(keyPath, s3Object) && filter.accept(keyPath)) {
+ S3AFileStatus status = createFileStatus(keyPath, s3Object,
listingOperationCallbacks.getDefaultBlockSize(keyPath),
getStoreContext().getUsername(),
- summary.getETag(), null, isCSEEnabled);
+ s3Object.eTag(), null, isCSEEnabled);
LOG.debug("Adding: {}", status);
stats.add(status);
added++;
@@ -476,11 +477,11 @@ public class Listing extends AbstractStoreOperation {
}
// prefixes: always directories
- for (String prefix : objects.getCommonPrefixes()) {
+ for (CommonPrefix prefix : objects.getCommonPrefixes()) {
Path keyPath = getStoreContext()
.getContextAccessors()
- .keyToPath(prefix);
- if (acceptor.accept(keyPath, prefix) && filter.accept(keyPath)) {
+ .keyToPath(prefix.prefix());
+ if (acceptor.accept(keyPath, prefix.prefix()) && filter.accept(keyPath)) {
S3AFileStatus status = new S3AFileStatus(Tristate.FALSE, keyPath,
getStoreContext().getUsername());
LOG.debug("Adding directory: {}", status);
@@ -731,18 +732,18 @@ public class Listing extends AbstractStoreOperation {
}
/**
- * Reject a summary entry if the key path is the qualified Path, or
+ * Reject a s3Object entry if the key path is the qualified Path, or
* it ends with {@code "_$folder$"}.
* @param keyPath key path of the entry
- * @param summary summary entry
+ * @param s3Object s3Object entry
* @return true if the entry is accepted (i.e. that a status entry
* should be generated.
*/
@Override
- public boolean accept(Path keyPath, S3ObjectSummary summary) {
+ public boolean accept(Path keyPath, S3Object s3Object) {
return !keyPath.equals(qualifiedPath)
- && !summary.getKey().endsWith(S3N_FOLDER_SUFFIX)
- && !objectRepresentsDirectory(summary.getKey());
+ && !s3Object.key().endsWith(S3N_FOLDER_SUFFIX)
+ && !objectRepresentsDirectory(s3Object.key());
}
/**
@@ -767,8 +768,8 @@ public class Listing extends AbstractStoreOperation {
*/
static class AcceptAllButS3nDirs implements FileStatusAcceptor {
- public boolean accept(Path keyPath, S3ObjectSummary summary) {
- return !summary.getKey().endsWith(S3N_FOLDER_SUFFIX);
+ public boolean accept(Path keyPath, S3Object s3Object) {
+ return !s3Object.key().endsWith(S3N_FOLDER_SUFFIX);
}
public boolean accept(Path keyPath, String prefix) {
@@ -799,17 +800,17 @@ public class Listing extends AbstractStoreOperation {
}
/**
- * Reject a summary entry if the key path is the qualified Path, or
+ * Reject a s3Object entry if the key path is the qualified Path, or
* it ends with {@code "_$folder$"}.
* @param keyPath key path of the entry
- * @param summary summary entry
+ * @param s3Object s3Object entry
* @return true if the entry is accepted (i.e. that a status entry
* should be generated.)
*/
@Override
- public boolean accept(Path keyPath, S3ObjectSummary summary) {
+ public boolean accept(Path keyPath, S3Object s3Object) {
return !keyPath.equals(qualifiedPath) &&
- !summary.getKey().endsWith(S3N_FOLDER_SUFFIX);
+ !s3Object.key().endsWith(S3N_FOLDER_SUFFIX);
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java
index d8c820cd8a12..296ec18dcf18 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java
@@ -23,13 +23,14 @@ import java.util.ListIterator;
import java.util.NoSuchElementException;
import javax.annotation.Nullable;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
-import com.amazonaws.services.s3.model.MultipartUpload;
-import com.amazonaws.services.s3.model.MultipartUploadListing;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest;
+import software.amazon.awssdk.services.s3.model.ListMultipartUploadsResponse;
+import software.amazon.awssdk.services.s3.model.MultipartUpload;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.s3a.api.RequestFactory;
import org.apache.hadoop.fs.s3a.impl.StoreContext;
@@ -43,7 +44,7 @@ import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDura
* MultipartUtils upload-specific functions for use by S3AFileSystem and Hadoop
* CLI.
* The Audit span active when
- * {@link #listMultipartUploads(StoreContext, AmazonS3, String, int)}
+ * {@link #listMultipartUploads(StoreContext, S3Client, String, int)}
* was invoked is retained for all subsequent operations.
*/
public final class MultipartUtils {
@@ -67,7 +68,7 @@ public final class MultipartUtils {
*/
static MultipartUtils.UploadIterator listMultipartUploads(
final StoreContext storeContext,
- AmazonS3 s3,
+ S3Client s3,
@Nullable String prefix,
int maxKeys)
throws IOException {
@@ -84,14 +85,14 @@ public final class MultipartUtils {
* at the time the iterator was constructed.
*/
static class ListingIterator implements
- RemoteIterator<MultipartUploadListing> {
+ RemoteIterator<ListMultipartUploadsResponse> {
private final String prefix;
private final RequestFactory requestFactory;
private final int maxKeys;
- private final AmazonS3 s3;
+ private final S3Client s3;
private final Invoker invoker;
private final AuditSpan auditSpan;
@@ -101,7 +102,7 @@ public final class MultipartUtils {
/**
* Most recent listing results.
*/
- private MultipartUploadListing listing;
+ private ListMultipartUploadsResponse listing;
/**
* Indicator that this is the first listing.
@@ -114,7 +115,7 @@ public final class MultipartUtils {
private int listCount = 0;
ListingIterator(final StoreContext storeContext,
- AmazonS3 s3,
+ S3Client s3,
@Nullable String prefix,
int maxKeys) throws IOException {
this.storeContext = storeContext;
@@ -153,7 +154,7 @@ public final class MultipartUtils {
*/
@Override
@Retries.RetryTranslated
- public MultipartUploadListing next() throws IOException {
+ public ListMultipartUploadsResponse next() throws IOException {
if (firstListing) {
firstListing = false;
} else {
@@ -171,32 +172,34 @@ public final class MultipartUtils {
public String toString() {
return "Upload iterator: prefix " + prefix
+ "; list count " + listCount
- + "; upload count " + listing.getMultipartUploads().size()
+ + "; upload count " + listing.uploads().size()
+ "; isTruncated=" + listing.isTruncated();
}
@Retries.RetryTranslated
private void requestNextBatch() throws IOException {
try (AuditSpan span = auditSpan.activate()) {
- ListMultipartUploadsRequest req = requestFactory
- .newListMultipartUploadsRequest(prefix);
+ ListMultipartUploadsRequest.Builder requestBuilder = requestFactory
+ .newListMultipartUploadsRequestBuilder(prefix);
if (!firstListing) {
- req.setKeyMarker(listing.getNextKeyMarker());
- req.setUploadIdMarker(listing.getNextUploadIdMarker());
+ requestBuilder.keyMarker(listing.nextKeyMarker());
+ requestBuilder.uploadIdMarker(listing.nextUploadIdMarker());
}
- req.setMaxUploads(maxKeys);
+ requestBuilder.maxUploads(maxKeys);
+
+ ListMultipartUploadsRequest request = requestBuilder.build();
LOG.debug("[{}], Requesting next {} uploads prefix {}, " +
"next key {}, next upload id {}", listCount, maxKeys, prefix,
- req.getKeyMarker(), req.getUploadIdMarker());
+ request.keyMarker(), request.uploadIdMarker());
listCount++;
listing = invoker.retry("listMultipartUploads", prefix, true,
trackDurationOfOperation(storeContext.getInstrumentation(),
MULTIPART_UPLOAD_LIST.getSymbol(),
- () -> s3.listMultipartUploads(req)));
+ () -> s3.listMultipartUploads(requestBuilder.build())));
LOG.debug("Listing found {} upload(s)",
- listing.getMultipartUploads().size());
+ listing.uploads().size());
LOG.debug("New listing state: {}", this);
}
}
@@ -216,14 +219,14 @@ public final class MultipartUtils {
*/
private ListingIterator lister;
/** Current listing: the last upload listing we fetched. */
- private MultipartUploadListing listing;
+ private ListMultipartUploadsResponse listing;
/** Iterator over the current listing. */
private ListIterator<MultipartUpload> batchIterator;
@Retries.RetryTranslated
public UploadIterator(
final StoreContext storeContext,
- AmazonS3 s3,
+ S3Client s3,
int maxKeys,
@Nullable String prefix)
throws IOException {
@@ -249,7 +252,7 @@ public final class MultipartUtils {
private boolean requestNextBatch() throws IOException {
if (lister.hasNext()) {
listing = lister.next();
- batchIterator = listing.getMultipartUploads().listIterator();
+ batchIterator = listing.uploads().listIterator();
return batchIterator.hasNext();
}
return false;
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProgressableProgressListener.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProgressableProgressListener.java
index 0ce022aa8858..7ee6c55c191b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProgressableProgressListener.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProgressableProgressListener.java
@@ -18,59 +18,55 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.event.ProgressEvent;
-import com.amazonaws.event.ProgressEventType;
-import com.amazonaws.event.ProgressListener;
-import com.amazonaws.services.s3.transfer.Upload;
+import software.amazon.awssdk.transfer.s3.model.ObjectTransfer;
+import software.amazon.awssdk.transfer.s3.progress.TransferListener;
import org.apache.hadoop.util.Progressable;
import org.slf4j.Logger;
-import static com.amazonaws.event.ProgressEventType.TRANSFER_COMPLETED_EVENT;
-import static com.amazonaws.event.ProgressEventType.TRANSFER_PART_STARTED_EVENT;
/**
* Listener to progress from AWS regarding transfers.
*/
-public class ProgressableProgressListener implements ProgressListener {
+public class ProgressableProgressListener implements TransferListener {
private static final Logger LOG = S3AFileSystem.LOG;
private final S3AFileSystem fs;
private final String key;
private final Progressable progress;
private long lastBytesTransferred;
- private final Upload upload;
/**
* Instantiate.
* @param fs filesystem: will be invoked with statistics updates
* @param key key for the upload
- * @param upload source of events
* @param progress optional callback for progress.
*/
public ProgressableProgressListener(S3AFileSystem fs,
String key,
- Upload upload,
Progressable progress) {
this.fs = fs;
this.key = key;
- this.upload = upload;
this.progress = progress;
this.lastBytesTransferred = 0;
}
@Override
- public void progressChanged(ProgressEvent progressEvent) {
- if (progress != null) {
- progress.progress();
- }
+ public void transferInitiated(TransferListener.Context.TransferInitiated context) {
+ fs.incrementWriteOperations();
+ }
+
+ @Override
+ public void transferComplete(TransferListener.Context.TransferComplete context) {
+ fs.incrementWriteOperations();
+ }
+
+ @Override
+ public void bytesTransferred(TransferListener.Context.BytesTransferred context) {
- // There are 3 http ops here, but this should be close enough for now
- ProgressEventType pet = progressEvent.getEventType();
- if (pet == TRANSFER_PART_STARTED_EVENT ||
- pet == TRANSFER_COMPLETED_EVENT) {
- fs.incrementWriteOperations();
+ if(progress != null) {
+ progress.progress();
}
- long transferred = upload.getProgress().getBytesTransferred();
+ long transferred = context.progressSnapshot().transferredBytes();
long delta = transferred - lastBytesTransferred;
fs.incrementPutProgressStatistics(key, delta);
lastBytesTransferred = transferred;
@@ -79,11 +75,13 @@ public class ProgressableProgressListener implements ProgressListener {
/**
* Method to invoke after upload has completed.
* This can handle race conditions in setup/teardown.
+ * @param upload upload which has just completed.
* @return the number of bytes which were transferred after the notification
*/
- public long uploadCompleted() {
- long delta = upload.getProgress().getBytesTransferred() -
- lastBytesTransferred;
+ public long uploadCompleted(ObjectTransfer upload) {
+
+ long delta =
+ upload.progress().snapshot().transferredBytes() - lastBytesTransferred;
if (delta > 0) {
LOG.debug("S3A write delta changed after finished: {} bytes", delta);
fs.incrementPutProgressStatistics(key, delta);
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
index 43a2b7e0dbd5..de0f59154e99 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
@@ -31,15 +31,16 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
-import com.amazonaws.SdkBaseException;
-import com.amazonaws.event.ProgressEvent;
-import com.amazonaws.event.ProgressEventType;
-import com.amazonaws.event.ProgressListener;
-import com.amazonaws.services.s3.model.PartETag;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.PutObjectResult;
-import com.amazonaws.services.s3.model.UploadPartRequest;
-
+import software.amazon.awssdk.core.exception.SdkException;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.model.CompletedPart;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartResponse;
+
+import org.apache.hadoop.fs.s3a.impl.ProgressListener;
+import org.apache.hadoop.fs.s3a.impl.ProgressListenerEvent;
import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
import org.apache.hadoop.fs.statistics.IOStatisticsAggregator;
import org.apache.hadoop.util.Preconditions;
@@ -69,6 +70,7 @@ import org.apache.hadoop.util.Progressable;
import static java.util.Objects.requireNonNull;
import static org.apache.hadoop.fs.s3a.S3AUtils.*;
import static org.apache.hadoop.fs.s3a.Statistic.*;
+import static org.apache.hadoop.fs.s3a.impl.ProgressListenerEvent.*;
import static org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext.EMPTY_BLOCK_OUTPUT_STREAM_STATISTICS;
import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDuration;
import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfInvocation;
@@ -192,7 +194,7 @@ class S3ABlockOutputStream extends OutputStream implements
this.executorService = MoreExecutors.listeningDecorator(
builder.executorService);
this.multiPartUpload = null;
- final Progressable progress = builder.progress;
+ Progressable progress = builder.progress;
this.progressListener = (progress instanceof ProgressListener) ?
(ProgressListener) progress
: new ProgressableListener(progress);
@@ -439,7 +441,7 @@ class S3ABlockOutputStream extends OutputStream implements
uploadCurrentBlock(true);
}
// wait for the partial uploads to finish
- final List<PartETag> partETags =
+ final List<CompletedPart> partETags =
multiPartUpload.waitForAllPartUploads();
bytes = bytesSubmitted;
@@ -597,27 +599,28 @@ class S3ABlockOutputStream extends OutputStream implements
final PutObjectRequest putObjectRequest = uploadData.hasFile() ?
writeOperationHelper.createPutObjectRequest(
key,
- uploadData.getFile(),
- builder.putOptions)
+ uploadData.getFile().length(),
+ builder.putOptions,
+ true)
: writeOperationHelper.createPutObjectRequest(
key,
- uploadData.getUploadStream(),
size,
- builder.putOptions);
- BlockUploadProgress callback =
- new BlockUploadProgress(
- block, progressListener, now());
- putObjectRequest.setGeneralProgressListener(callback);
+ builder.putOptions,
+ false);
+
+ BlockUploadProgress progressCallback =
+ new BlockUploadProgress(block, progressListener, now());
statistics.blockUploadQueued(size);
- ListenableFuture<PutObjectResult> putObjectResult =
+ ListenableFuture<PutObjectResponse> putObjectResult =
executorService.submit(() -> {
try {
// the putObject call automatically closes the input
// stream afterwards.
- return writeOperationHelper.putObject(
- putObjectRequest,
- builder.putOptions,
- statistics);
+ PutObjectResponse response =
+ writeOperationHelper.putObject(putObjectRequest, builder.putOptions, uploadData,
+ uploadData.hasFile(), statistics);
+ progressCallback.progressChanged(REQUEST_BYTE_TRANSFER_EVENT);
+ return response;
} finally {
cleanupWithLogger(LOG, uploadData, block);
}
@@ -761,7 +764,7 @@ class S3ABlockOutputStream extends OutputStream implements
*/
private class MultiPartUpload {
private final String uploadId;
- private final List<ListenableFuture<PartETag>> partETagsFutures;
+ private final List<ListenableFuture<CompletedPart>> partETagsFutures;
private int partsSubmitted;
private int partsUploaded;
private long bytesSubmitted;
@@ -866,18 +869,19 @@ class S3ABlockOutputStream extends OutputStream implements
final int currentPartNumber = partETagsFutures.size() + 1;
final UploadPartRequest request;
final S3ADataBlocks.BlockUploadData uploadData;
+ final RequestBody requestBody;
try {
uploadData = block.startUpload();
- request = writeOperationHelper.newUploadPartRequest(
+ requestBody = uploadData.hasFile()
+ ? RequestBody.fromFile(uploadData.getFile())
+ : RequestBody.fromInputStream(uploadData.getUploadStream(), size);
+
+ request = writeOperationHelper.newUploadPartRequestBuilder(
key,
uploadId,
currentPartNumber,
- size,
- uploadData.getUploadStream(),
- uploadData.getFile(),
- 0L);
- request.setLastPart(isLast);
- } catch (SdkBaseException aws) {
+ size).build();
+ } catch (SdkException aws) {
// catch and translate
IOException e = translateException("upload", key, aws);
// failure to start the upload.
@@ -888,28 +892,38 @@ class S3ABlockOutputStream extends OutputStream implements
noteUploadFailure(e);
throw e;
}
- BlockUploadProgress callback =
- new BlockUploadProgress(
- block, progressListener, now());
- request.setGeneralProgressListener(callback);
+
+ BlockUploadProgress progressCallback =
+ new BlockUploadProgress(block, progressListener, now());
+
statistics.blockUploadQueued(block.dataSize());
- ListenableFuture<PartETag> partETagFuture =
+ ListenableFuture<CompletedPart> partETagFuture =
executorService.submit(() -> {
// this is the queued upload operation
// do the upload
try {
LOG.debug("Uploading part {} for id '{}'",
currentPartNumber, uploadId);
- PartETag partETag = writeOperationHelper.uploadPart(request, statistics)
- .getPartETag();
+
+ progressCallback.progressChanged(TRANSFER_PART_STARTED_EVENT);
+
+ UploadPartResponse response = writeOperationHelper
+ .uploadPart(request, requestBody, statistics);
LOG.debug("Completed upload of {} to part {}",
- block, partETag.getETag());
+ block, response.eTag());
LOG.debug("Stream statistics of {}", statistics);
partsUploaded++;
- return partETag;
+
+ progressCallback.progressChanged(TRANSFER_PART_COMPLETED_EVENT);
+
+ return CompletedPart.builder()
+ .eTag(response.eTag())
+ .partNumber(currentPartNumber)
+ .build();
} catch (IOException e) {
// save immediately.
noteUploadFailure(e);
+ progressCallback.progressChanged(TRANSFER_PART_FAILED_EVENT);
throw e;
} finally {
// close the stream and block
@@ -924,7 +938,7 @@ class S3ABlockOutputStream extends OutputStream implements
* @return list of results
* @throws IOException IO Problems
*/
- private List<PartETag> waitForAllPartUploads() throws IOException {
+ private List<CompletedPart> waitForAllPartUploads() throws IOException {
LOG.debug("Waiting for {} uploads to complete", partETagsFutures.size());
try {
return Futures.allAsList(partETagsFutures).get();
@@ -948,7 +962,7 @@ class S3ABlockOutputStream extends OutputStream implements
*/
private void cancelAllActiveFutures() {
LOG.debug("Cancelling futures");
- for (ListenableFuture<PartETag> future : partETagsFutures) {
+ for (ListenableFuture<CompletedPart> future : partETagsFutures) {
future.cancel(true);
}
}
@@ -960,7 +974,7 @@ class S3ABlockOutputStream extends OutputStream implements
* @param partETags list of partial uploads
* @throws IOException on any problem
*/
- private void complete(List<PartETag> partETags)
+ private void complete(List<CompletedPart> partETags)
throws IOException {
maybeRethrowUploadFailure();
AtomicInteger errorCount = new AtomicInteger(0);
@@ -1005,22 +1019,24 @@ class S3ABlockOutputStream extends OutputStream implements
}
}
+
/**
* The upload progress listener registered for events returned
* during the upload of a single block.
* It updates statistics and handles the end of the upload.
* Transfer failures are logged at WARN.
*/
- private final class BlockUploadProgress implements ProgressListener {
+ private final class BlockUploadProgress {
+
private final S3ADataBlocks.DataBlock block;
private final ProgressListener nextListener;
private final Instant transferQueueTime;
private Instant transferStartTime;
+ private long size;
/**
* Track the progress of a single block upload.
* @param block block to monitor
- * @param nextListener optional next progress listener
* @param transferQueueTime time the block was transferred
* into the queue
*/
@@ -1029,20 +1045,17 @@ class S3ABlockOutputStream extends OutputStream implements
Instant transferQueueTime) {
this.block = block;
this.transferQueueTime = transferQueueTime;
+ this.size = block.dataSize();
this.nextListener = nextListener;
}
- @Override
- public void progressChanged(ProgressEvent progressEvent) {
- ProgressEventType eventType = progressEvent.getEventType();
- long bytesTransferred = progressEvent.getBytesTransferred();
+ public void progressChanged(ProgressListenerEvent eventType) {
- long size = block.dataSize();
switch (eventType) {
case REQUEST_BYTE_TRANSFER_EVENT:
// bytes uploaded
- statistics.bytesTransferred(bytesTransferred);
+ statistics.bytesTransferred(size);
break;
case TRANSFER_PART_STARTED_EVENT:
@@ -1057,6 +1070,7 @@ class S3ABlockOutputStream extends OutputStream implements
statistics.blockUploadCompleted(
Duration.between(transferStartTime, now()),
size);
+ statistics.bytesTransferred(size);
break;
case TRANSFER_PART_FAILED_EVENT:
@@ -1071,13 +1085,13 @@ class S3ABlockOutputStream extends OutputStream implements
}
if (nextListener != null) {
- nextListener.progressChanged(progressEvent);
+ nextListener.progressChanged(eventType, size);
}
}
}
/**
- * Bridge from AWS {@code ProgressListener} to Hadoop {@link Progressable}.
+ * Bridge from {@link ProgressListener} to Hadoop {@link Progressable}.
*/
private static class ProgressableListener implements ProgressListener {
private final Progressable progress;
@@ -1086,7 +1100,7 @@ class S3ABlockOutputStream extends OutputStream implements
this.progress = progress;
}
- public void progressChanged(ProgressEvent progressEvent) {
+ public void progressChanged(ProgressListenerEvent eventType, int bytesTransferred) {
if (progress != null) {
progress.progress();
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
index ddaf059b055d..5aa43245c2f0 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
@@ -47,7 +47,7 @@ import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
* Set of classes to support output streaming into blocks which are then
* uploaded as to S3 as a single PUT, or as part of a multipart request.
*/
-final class S3ADataBlocks {
+public final class S3ADataBlocks {
private static final Logger LOG =
LoggerFactory.getLogger(S3ADataBlocks.class);
@@ -101,7 +101,7 @@ final class S3ADataBlocks {
* It can be one of a file or an input stream.
* When closed, any stream is closed. Any source file is untouched.
*/
- static final class BlockUploadData implements Closeable {
+ public static final class BlockUploadData implements Closeable {
private final File file;
private final InputStream uploadStream;
@@ -109,7 +109,7 @@ final class S3ADataBlocks {
* File constructor; input stream will be null.
* @param file file to upload
*/
- BlockUploadData(File file) {
+ public BlockUploadData(File file) {
Preconditions.checkArgument(file.exists(), "No file: " + file);
this.file = file;
this.uploadStream = null;
@@ -119,7 +119,7 @@ final class S3ADataBlocks {
* Stream constructor, file field will be null.
* @param uploadStream stream to upload
*/
- BlockUploadData(InputStream uploadStream) {
+ public BlockUploadData(InputStream uploadStream) {
Preconditions.checkNotNull(uploadStream, "rawUploadStream");
this.uploadStream = uploadStream;
this.file = null;
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 07a14f63aa41..d0a32148c9d5 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -21,17 +21,20 @@ package org.apache.hadoop.fs.s3a;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InterruptedIOException;
import java.io.UncheckedIOException;
import java.net.URI;
import java.nio.file.AccessDeniedException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
@@ -41,6 +44,7 @@ import java.util.Set;
import java.util.Objects;
import java.util.TreeSet;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
@@ -48,43 +52,53 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.Nullable;
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.AmazonServiceException;
-import com.amazonaws.SdkBaseException;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.Headers;
-import com.amazonaws.services.s3.model.CannedAccessControlList;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
-import com.amazonaws.services.s3.model.CopyObjectRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsResult;
-import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
-import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ListObjectsV2Request;
-import com.amazonaws.services.s3.model.MultiObjectDeleteException;
-import com.amazonaws.services.s3.model.MultipartUpload;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.PutObjectResult;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.SelectObjectContentRequest;
-import com.amazonaws.services.s3.model.SelectObjectContentResult;
-import com.amazonaws.services.s3.model.StorageClass;
-import com.amazonaws.services.s3.model.UploadPartRequest;
-import com.amazonaws.services.s3.model.UploadPartResult;
-import com.amazonaws.services.s3.transfer.Copy;
-import com.amazonaws.services.s3.transfer.TransferManager;
-import com.amazonaws.services.s3.transfer.TransferManagerConfiguration;
-import com.amazonaws.services.s3.transfer.Upload;
-import com.amazonaws.services.s3.transfer.model.CopyResult;
-import com.amazonaws.services.s3.transfer.model.UploadResult;
-import com.amazonaws.event.ProgressListener;
-
+import software.amazon.awssdk.core.ResponseInputStream;
+import software.amazon.awssdk.core.exception.SdkException;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse;
+import software.amazon.awssdk.services.s3.model.GetBucketLocationRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
+import software.amazon.awssdk.services.s3.model.HeadBucketResponse;
+import software.amazon.awssdk.services.s3.model.MultipartUpload;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
+import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
+import software.amazon.awssdk.services.s3.model.CopyObjectResponse;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
+import software.amazon.awssdk.services.s3.model.NoSuchBucketException;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
+import software.amazon.awssdk.services.s3.model.S3Error;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest;
+import software.amazon.awssdk.services.s3.model.SelectObjectContentResponseHandler;
+import software.amazon.awssdk.services.s3.model.StorageClass;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartResponse;
+import software.amazon.awssdk.transfer.s3.model.CompletedCopy;
+import software.amazon.awssdk.transfer.s3.model.CompletedFileUpload;
+import software.amazon.awssdk.transfer.s3.model.Copy;
+import software.amazon.awssdk.transfer.s3.S3TransferManager;
+import software.amazon.awssdk.transfer.s3.model.CopyRequest;
+import software.amazon.awssdk.transfer.s3.model.FileUpload;
+import software.amazon.awssdk.transfer.s3.model.UploadFileRequest;
+
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.impl.prefetch.ExecutorServiceFuturePool;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -108,11 +122,12 @@ import org.apache.hadoop.fs.s3a.audit.AuditSpanS3A;
import org.apache.hadoop.fs.s3a.auth.SignerManager;
import org.apache.hadoop.fs.s3a.auth.delegation.DelegationOperations;
import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider;
+import org.apache.hadoop.fs.s3a.impl.AWSCannedACL;
+import org.apache.hadoop.fs.s3a.impl.AWSHeaders;
import org.apache.hadoop.fs.s3a.impl.BulkDeleteRetryHandler;
import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy;
import org.apache.hadoop.fs.s3a.impl.ContextAccessors;
import org.apache.hadoop.fs.s3a.impl.CopyFromLocalOperation;
-import org.apache.hadoop.fs.s3a.impl.CopyOutcome;
import org.apache.hadoop.fs.s3a.impl.CreateFileBuilder;
import org.apache.hadoop.fs.s3a.impl.DeleteOperation;
import org.apache.hadoop.fs.s3a.impl.DirectoryPolicy;
@@ -122,6 +137,7 @@ import org.apache.hadoop.fs.s3a.impl.HeaderProcessing;
import org.apache.hadoop.fs.s3a.impl.InternalConstants;
import org.apache.hadoop.fs.s3a.impl.ListingOperationCallbacks;
import org.apache.hadoop.fs.s3a.impl.MkdirOperation;
+import org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteException;
import org.apache.hadoop.fs.s3a.impl.OpenFileSupport;
import org.apache.hadoop.fs.s3a.impl.OperationCallbacks;
import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
@@ -131,7 +147,6 @@ import org.apache.hadoop.fs.s3a.impl.S3AMultipartUploaderBuilder;
import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum;
import org.apache.hadoop.fs.s3a.impl.StoreContext;
import org.apache.hadoop.fs.s3a.impl.StoreContextBuilder;
-import org.apache.hadoop.fs.s3a.impl.V2Migration;
import org.apache.hadoop.fs.s3a.prefetch.S3APrefetchingInputStream;
import org.apache.hadoop.fs.s3a.tools.MarkerToolOperations;
import org.apache.hadoop.fs.s3a.tools.MarkerToolOperationsImpl;
@@ -141,6 +156,7 @@ import org.apache.hadoop.fs.statistics.IOStatistics;
import org.apache.hadoop.fs.statistics.IOStatisticsSource;
import org.apache.hadoop.fs.statistics.IOStatisticsContext;
import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+import org.apache.hadoop.fs.store.LogExactlyOnce;
import org.apache.hadoop.fs.store.audit.AuditEntryPoint;
import org.apache.hadoop.fs.store.audit.ActiveThreadSpanSource;
import org.apache.hadoop.fs.store.audit.AuditSpan;
@@ -213,6 +229,7 @@ import static org.apache.hadoop.fs.s3a.Listing.toLocatedFileStatusIterator;
import static org.apache.hadoop.fs.s3a.S3AUtils.*;
import static org.apache.hadoop.fs.s3a.Statistic.*;
import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.INITIALIZE_SPAN;
+import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.createAWSCredentialProviderList;
import static org.apache.hadoop.fs.s3a.auth.RolePolicies.STATEMENT_ALLOW_KMS_RW;
import static org.apache.hadoop.fs.s3a.auth.RolePolicies.allowS3Operations;
import static org.apache.hadoop.fs.s3a.auth.delegation.S3ADelegationTokens.TokenIssuingPolicy.NoTokensAvailable;
@@ -224,14 +241,14 @@ import static org.apache.hadoop.fs.s3a.impl.CreateFileBuilder.OPTIONS_CREATE_FIL
import static org.apache.hadoop.fs.s3a.impl.CreateFileBuilder.OPTIONS_CREATE_FILE_OVERWRITE;
import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isObjectNotFound;
import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isUnknownBucket;
-import static org.apache.hadoop.fs.s3a.impl.InternalConstants.AP_INACCESSIBLE;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.AP_REQUIRED_EXCEPTION;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.ARN_BUCKET_OPTION;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DEFAULT_UPLOAD_PART_COUNT_LIMIT;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DELETE_CONSIDERED_IDEMPOTENT;
-import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_403;
-import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_404;
+import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_301_MOVED_PERMANENTLY;
+import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_403_FORBIDDEN;
+import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_404_NOT_FOUND;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.UPLOAD_PART_COUNT_LIMIT;
import static org.apache.hadoop.fs.s3a.impl.NetworkBinding.fixBucketRegion;
import static org.apache.hadoop.fs.s3a.impl.NetworkBinding.logDnsLookup;
@@ -275,7 +292,9 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
private URI uri;
private Path workingDir;
private String username;
- private AmazonS3 s3;
+ private S3Client s3Client;
+ /** Async client is used for transfer manager and s3 select. */
+ private S3AsyncClient s3AsyncClient;
// initial callback policy is fail-once; it's there just to assist
// some mock tests and other codepaths trying to call the low level
// APIs on an uninitialized filesystem.
@@ -294,7 +313,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
private Listing listing;
private long partSize;
private boolean enableMultiObjectsDelete;
- private TransferManager transfers;
+ private S3TransferManager transferManager;
private ExecutorService boundedThreadPool;
private ThreadPoolExecutor unboundedThreadPool;
@@ -313,10 +332,12 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
private int executorCapacity;
private long multiPartThreshold;
public static final Logger LOG = LoggerFactory.getLogger(S3AFileSystem.class);
+ /** Exactly once log to warn about setting the region in config to avoid probe. */
+ private static final LogExactlyOnce SET_REGION_WARNING = new LogExactlyOnce(LOG);
private static final Logger PROGRESS =
LoggerFactory.getLogger("org.apache.hadoop.fs.s3a.S3AFileSystem.Progress");
private LocalDirAllocator directoryAllocator;
- private CannedAccessControlList cannedACL;
+ private String cannedACL;
/**
* This must never be null; until initialized it just declares that there
@@ -364,6 +385,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
private AWSCredentialProviderList credentials;
private SignerManager signerManager;
+ private S3AInternals s3aInternals;
/**
* Page size for deletions.
@@ -429,6 +451,8 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
*/
private String scheme = FS_S3A;
+ private final static Map<String, Region> BUCKET_REGIONS = new HashMap<>();
+
/** Add any deprecated keys. */
@SuppressWarnings("deprecation")
private static void addDeprecatedKeys() {
@@ -510,6 +534,8 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
super.initialize(uri, conf);
setConf(conf);
+ s3aInternals = createS3AInternals();
+
// look for encryption data
// DT Bindings may override this
setEncryptionSecrets(
@@ -584,9 +610,6 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
// the encryption algorithms)
bindAWSClient(name, delegationTokensEnabled);
- initTransferManager();
-
-
// This initiates a probe against S3 for the bucket existing.
doBucketProbing();
@@ -654,7 +677,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
AWS_S3_VECTOR_ACTIVE_RANGE_READS, DEFAULT_AWS_S3_VECTOR_ACTIVE_RANGE_READS, 1);
vectoredIOContext = populateVectoredIOContext(conf);
scheme = (this.uri != null && this.uri.getScheme() != null) ? this.uri.getScheme() : FS_S3A;
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
// amazon client exception: stop all services then throw the translation
cleanupWithLogger(LOG, span);
stopAllServices();
@@ -706,7 +729,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* bucket existence check is not done to improve performance of
* S3AFileSystem initialization. When set to 1 or 2, bucket existence check
* will be performed which is potentially slow.
- * If 3 or higher: warn and use the v2 check.
+ * If 3 or higher: warn and skip check.
* Also logging DNS address of the s3 endpoint if the bucket probe value is
* greater than 0 else skipping it for increased performance.
* @throws UnknownStoreException the bucket is absent
@@ -723,18 +746,14 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
LOG.debug("skipping check for bucket existence");
break;
case 1:
- logDnsLookup(getConf());
- verifyBucketExists();
- break;
case 2:
logDnsLookup(getConf());
- verifyBucketExistsV2();
+ verifyBucketExists();
break;
default:
// we have no idea what this is, assume it is from a later release.
- LOG.warn("Unknown bucket probe option {}: {}; falling back to check #2",
+ LOG.warn("Unknown bucket probe option {}: {}; skipping check for bucket existence",
S3A_BUCKET_PROBE, bucketProbe);
- verifyBucketExistsV2();
break;
}
}
@@ -827,54 +846,37 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
}
/**
- * Verify that the bucket exists. This does not check permissions,
- * not even read access.
+ * Verify that the bucket exists.
* Retry policy: retrying, translated.
* @throws UnknownStoreException the bucket is absent
* @throws IOException any other problem talking to S3
*/
@Retries.RetryTranslated
- protected void verifyBucketExists()
- throws UnknownStoreException, IOException {
- if (!invoker.retry("doesBucketExist", bucket, true,
- trackDurationOfOperation(getDurationTrackerFactory(),
- STORE_EXISTS_PROBE.getSymbol(),
- () -> s3.doesBucketExist(bucket)))) {
- throw new UnknownStoreException("s3a://" + bucket + "/", " Bucket does "
- + "not exist");
- }
- }
+ protected void verifyBucketExists() throws UnknownStoreException, IOException {
- /**
- * Verify that the bucket exists. This will correctly throw an exception
- * when credentials are invalid.
- * Retry policy: retrying, translated.
- * @throws UnknownStoreException the bucket is absent
- * @throws IOException any other problem talking to S3
- */
- @Retries.RetryTranslated
- protected void verifyBucketExistsV2()
- throws UnknownStoreException, IOException {
- if (!invoker.retry("doesBucketExistV2", bucket, true,
- trackDurationOfOperation(getDurationTrackerFactory(),
- STORE_EXISTS_PROBE.getSymbol(),
- () -> {
- // Bug in SDK always returns `true` for AccessPoint ARNs with `doesBucketExistV2()`
- // expanding implementation to use ARNs and buckets correctly
+ if(!trackDurationAndSpan(
+ STORE_EXISTS_PROBE, bucket, null, () ->
+ invoker.retry("doesBucketExist", bucket, true, () -> {
try {
- s3.getBucketAcl(bucket);
- } catch (AmazonServiceException ex) {
- int statusCode = ex.getStatusCode();
- if (statusCode == SC_404 ||
- (statusCode == SC_403 && ex.getMessage().contains(AP_INACCESSIBLE))) {
+ if (BUCKET_REGIONS.containsKey(bucket)) {
+ return true;
+ }
+ s3Client.headBucket(HeadBucketRequest.builder().bucket(bucket).build());
+ return true;
+ } catch (AwsServiceException ex) {
+ int statusCode = ex.statusCode();
+ if (statusCode == SC_404_NOT_FOUND ||
+ (statusCode == SC_403_FORBIDDEN && accessPoint != null)) {
return false;
}
}
return true;
}))) {
- throw new UnknownStoreException("s3a://" + bucket + "/", " Bucket does "
- + "not exist");
+
+ throw new UnknownStoreException("s3a://" + bucket + "/",
+ " Bucket does " + "not exist. " + "Accessing with " + ENDPOINT + " set to "
+ + getConf().getTrimmed(ENDPOINT, null));
}
}
@@ -915,7 +917,6 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @param dtEnabled are delegation tokens enabled?
* @throws IOException failure.
*/
- @SuppressWarnings("deprecation")
private void bindAWSClient(URI name, boolean dtEnabled) throws IOException {
Configuration conf = getConf();
credentials = null;
@@ -928,7 +929,6 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
// with it if so.
LOG.debug("Using delegation tokens");
- V2Migration.v1DelegationTokenCredentialProvidersUsed();
S3ADelegationTokens tokens = new S3ADelegationTokens();
this.delegationTokens = Optional.of(tokens);
tokens.bindToFileSystem(getCanonicalUri(),
@@ -955,7 +955,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
uaSuffix = tokens.getUserAgentField();
} else {
// DT support is disabled, so create the normal credential chain
- credentials = createAWSCredentialProviderSet(name, conf);
+ credentials = createAWSCredentialProviderList(name, conf);
}
LOG.debug("Using credential provider {}", credentials);
Class<? extends S3ClientFactory> s3ClientFactoryClass = conf.getClass(
@@ -966,8 +966,14 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
? conf.getTrimmed(ENDPOINT, DEFAULT_ENDPOINT)
: accessPoint.getEndpoint();
- S3ClientFactory.S3ClientCreationParameters parameters = null;
- parameters = new S3ClientFactory.S3ClientCreationParameters()
+ String configuredRegion = accessPoint == null
+ ? conf.getTrimmed(AWS_REGION)
+ : accessPoint.getRegion();
+
+ Region region = getS3Region(configuredRegion);
+
+ S3ClientFactory.S3ClientCreationParameters parameters =
+ new S3ClientFactory.S3ClientCreationParameters()
.withCredentialSet(credentials)
.withPathUri(name)
.withEndpoint(endpoint)
@@ -975,11 +981,97 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
.withPathStyleAccess(conf.getBoolean(PATH_STYLE_ACCESS, false))
.withUserAgentSuffix(uaSuffix)
.withRequesterPays(conf.getBoolean(ALLOW_REQUESTER_PAYS, DEFAULT_ALLOW_REQUESTER_PAYS))
- .withRequestHandlers(auditManager.createRequestHandlers());
+ .withExecutionInterceptors(auditManager.createExecutionInterceptors())
+ .withMinimumPartSize(partSize)
+ .withMultipartThreshold(multiPartThreshold)
+ .withTransferManagerExecutor(unboundedThreadPool)
+ .withRegion(region);
- s3 = ReflectionUtils.newInstance(s3ClientFactoryClass, conf)
- .createS3Client(getUri(),
- parameters);
+ S3ClientFactory clientFactory = ReflectionUtils.newInstance(s3ClientFactoryClass, conf);
+ s3Client = clientFactory.createS3Client(getUri(), parameters);
+ createS3AsyncClient(clientFactory, parameters);
+ transferManager = clientFactory.createS3TransferManager(getS3AsyncClient());
+ }
+
+ /**
+ * Creates and configures the S3AsyncClient.
+ * Uses synchronized method to suppress spotbugs error.
+ *
+ * @param clientFactory factory used to create S3AsyncClient
+ * @param parameters parameter object
+ * @throws IOException on any IO problem
+ */
+ private void createS3AsyncClient(S3ClientFactory clientFactory,
+ S3ClientFactory.S3ClientCreationParameters parameters) throws IOException {
+ s3AsyncClient = clientFactory.createS3AsyncClient(getUri(), parameters);
+ }
+
+ /**
+ * Get the bucket region.
+ *
+ * @param region AWS S3 Region set in the config. This property may not be set, in which case
+ * ask S3 for the region.
+ * @return region of the bucket.
+ */
+ private Region getS3Region(String region) throws IOException {
+
+ if (!StringUtils.isBlank(region)) {
+ return Region.of(region);
+ }
+
+ Region cachedRegion = BUCKET_REGIONS.get(bucket);
+
+ if (cachedRegion != null) {
+ LOG.debug("Got region {} for bucket {} from cache", cachedRegion, bucket);
+ return cachedRegion;
+ }
+
+ Region s3Region = trackDurationAndSpan(STORE_REGION_PROBE, bucket, null,
+ () -> invoker.retry("getS3Region", bucket, true, () -> {
+ try {
+
+ SET_REGION_WARNING.warn(
+ "Getting region for bucket {} from S3, this will slow down FS initialisation. "
+ + "To avoid this, set the region using property {}", bucket,
+ FS_S3A_BUCKET_PREFIX + bucket + ".endpoint.region");
+
+ // build a s3 client with region eu-west-1 that can be used to get the region of the
+ // bucket. Using eu-west-1, as headBucket() doesn't work with us-east-1. This is because
+ // us-east-1 uses the endpoint s3.amazonaws.com, which resolves bucket.s3.amazonaws.com
+ // to the actual region the bucket is in. As the request is signed with us-east-1 and
+ // not the bucket's region, it fails.
+ S3Client getRegionS3Client =
+ S3Client.builder().region(Region.EU_WEST_1).credentialsProvider(credentials)
+ .build();
+
+ HeadBucketResponse headBucketResponse =
+ getRegionS3Client.headBucket(HeadBucketRequest.builder().bucket(bucket).build());
+
+ Region bucketRegion = Region.of(
+ headBucketResponse.sdkHttpResponse().headers().get(BUCKET_REGION_HEADER).get(0));
+ BUCKET_REGIONS.put(bucket, bucketRegion);
+
+ return bucketRegion;
+ } catch (S3Exception exception) {
+ if (exception.statusCode() == SC_301_MOVED_PERMANENTLY) {
+ Region bucketRegion = Region.of(
+ exception.awsErrorDetails().sdkHttpResponse().headers().get(BUCKET_REGION_HEADER)
+ .get(0));
+ BUCKET_REGIONS.put(bucket, bucketRegion);
+
+ return bucketRegion;
+ }
+
+ if (exception.statusCode() == SC_404_NOT_FOUND) {
+ throw new UnknownStoreException("s3a://" + bucket + "/",
+ " Bucket does " + "not exist");
+ }
+
+ throw exception;
+ }
+ }));
+
+ return s3Region;
}
/**
@@ -1079,12 +1171,14 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
.toUpperCase(Locale.US);
StorageClass storageClass = null;
if (!storageClassConf.isEmpty()) {
- try {
- storageClass = StorageClass.fromValue(storageClassConf);
- } catch (IllegalArgumentException e) {
+ storageClass = StorageClass.fromValue(storageClassConf);
+
+ if (storageClass.equals(StorageClass.UNKNOWN_TO_SDK_VERSION)) {
LOG.warn("Unknown storage class property {}: {}; falling back to default storage class",
STORAGE_CLASS, storageClassConf);
+ storageClass = null;
}
+
} else {
LOG.debug("Unset storage class property {}; falling back to default storage class",
STORAGE_CLASS);
@@ -1110,6 +1204,14 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
return requestFactory;
}
+ /**
+ * Get the S3 Async client.
+ * @return the async s3 client.
+ */
+ private S3AsyncClient getS3AsyncClient() {
+ return s3AsyncClient;
+ }
+
/**
* Implementation of all operations used by delegation tokens.
*/
@@ -1150,22 +1252,10 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
return encryptionSecrets;
}
- private void initTransferManager() {
- TransferManagerConfiguration transferConfiguration =
- new TransferManagerConfiguration();
- transferConfiguration.setMinimumUploadPartSize(partSize);
- transferConfiguration.setMultipartUploadThreshold(multiPartThreshold);
- transferConfiguration.setMultipartCopyPartSize(partSize);
- transferConfiguration.setMultipartCopyThreshold(multiPartThreshold);
-
- transfers = new TransferManager(s3, unboundedThreadPool);
- transfers.setConfiguration(transferConfiguration);
- }
-
private void initCannedAcls(Configuration conf) {
String cannedACLName = conf.get(CANNED_ACL, DEFAULT_CANNED_ACL);
if (!cannedACLName.isEmpty()) {
- cannedACL = CannedAccessControlList.valueOf(cannedACLName);
+ cannedACL = AWSCannedACL.valueOf(cannedACLName).toString();
} else {
cannedACL = null;
}
@@ -1198,12 +1288,22 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
public void abortOutstandingMultipartUploads(long seconds)
throws IOException {
Preconditions.checkArgument(seconds >= 0);
- Date purgeBefore =
- new Date(new Date().getTime() - seconds * 1000);
+ Instant purgeBefore =
+ Instant.now().minusSeconds(seconds);
LOG.debug("Purging outstanding multipart uploads older than {}",
purgeBefore);
invoker.retry("Purging multipart uploads", bucket, true,
- () -> transfers.abortMultipartUploads(bucket, purgeBefore));
+ () -> {
+ MultipartUtils.UploadIterator uploadIterator =
+ MultipartUtils.listMultipartUploads(createStoreContext(), s3Client, null, maxKeys);
+
+ while (uploadIterator.hasNext()) {
+ MultipartUpload upload = uploadIterator.next();
+ if (upload.initiated().compareTo(purgeBefore) < 0) {
+ abortMultipartUpload(upload);
+ }
+ }
+ });
}
/**
@@ -1252,81 +1352,122 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
}
/**
- * Returns the S3 client used by this filesystem.
- * This is for internal use within the S3A code itself.
- * @return AmazonS3Client
+ * Set the client -used in mocking tests to force in a different client.
+ * @param client client.
*/
- private AmazonS3 getAmazonS3Client() {
- return s3;
+ protected void setAmazonS3Client(S3Client client) {
+ Preconditions.checkNotNull(client, "clientV2");
+ LOG.debug("Setting S3V2 client to {}", client);
+ s3Client = client;
}
/**
- * Returns the S3 client used by this filesystem.
- * <i>Warning: this must only be used for testing, as it bypasses core
- * S3A operations. </i>
- * @param reason a justification for requesting access.
- * @return AmazonS3Client
+ * S3AInternals method.
+ * {@inheritDoc}.
*/
- @VisibleForTesting
- public AmazonS3 getAmazonS3ClientForTesting(String reason) {
- LOG.warn("Access to S3A client requested, reason {}", reason);
- V2Migration.v1S3ClientRequested();
- return s3;
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ public String getBucketLocation() throws IOException {
+ return s3aInternals.getBucketLocation(bucket);
}
/**
- * Set the client -used in mocking tests to force in a different client.
- * @param client client.
+ * Create the S3AInternals; left as something mocking
+ * subclasses may want to override.
+ * @return the internal implementation
*/
- protected void setAmazonS3Client(AmazonS3 client) {
- Preconditions.checkNotNull(client, "client");
- LOG.debug("Setting S3 client to {}", client);
- s3 = client;
-
- // Need to use a new TransferManager that uses the new client.
- // Also, using a new TransferManager requires a new threadpool as the old
- // TransferManager will shut the thread pool down when it is garbage
- // collected.
- initThreadPools(getConf());
- initTransferManager();
+ protected S3AInternals createS3AInternals() {
+ return new S3AInternalsImpl();
}
/**
- * Get the region of a bucket.
- * Invoked from StoreContext; consider an entry point.
- * @return the region in which a bucket is located
- * @throws AccessDeniedException if the caller lacks permission.
- * @throws IOException on any failure.
+ * Get the S3AInternals.
+ * @return the internal implementation
*/
- @Retries.RetryTranslated
- @InterfaceAudience.LimitedPrivate("diagnostics")
- public String getBucketLocation() throws IOException {
- return getBucketLocation(bucket);
+ public S3AInternals getS3AInternals() {
+ return s3aInternals;
}
/**
- * Get the region of a bucket; fixing up the region so it can be used
- * in the builders of other AWS clients.
- * Requires the caller to have the AWS role permission
- * {@code s3:GetBucketLocation}.
- * Retry policy: retrying, translated.
- * @param bucketName the name of the bucket
- * @return the region in which a bucket is located
- * @throws AccessDeniedException if the caller lacks permission.
- * @throws IOException on any failure.
+ * Implementation of the S3A Internals operations; pulled out of S3AFileSystem to
+ * force code accessing it to call {@link #getS3AInternals()}.
*/
- @VisibleForTesting
- @AuditEntryPoint
- @Retries.RetryTranslated
- public String getBucketLocation(String bucketName) throws IOException {
- final String region = trackDurationAndSpan(
- STORE_EXISTS_PROBE, bucketName, null, () ->
- invoker.retry("getBucketLocation()", bucketName, true, () ->
- // If accessPoint then region is known from Arn
- accessPoint != null
- ? accessPoint.getRegion()
- : s3.getBucketLocation(bucketName)));
- return fixBucketRegion(region);
+ private final class S3AInternalsImpl implements S3AInternals {
+
+ @Override
+ public S3Client getAmazonS3Client(String reason) {
+ LOG.debug("Access to S3 client requested, reason {}", reason);
+ return s3Client;
+ }
+
+ /**
+ * S3AInternals method.
+ * {@inheritDoc}.
+ */
+ @Override
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ public String getBucketLocation() throws IOException {
+ return s3aInternals.getBucketLocation(bucket);
+ }
+
+ /**
+ * S3AInternals method.
+ * {@inheritDoc}.
+ */
+ @Override
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ public String getBucketLocation(String bucketName) throws IOException {
+ final String region = trackDurationAndSpan(
+ STORE_EXISTS_PROBE, bucketName, null, () ->
+ invoker.retry("getBucketLocation()", bucketName, true, () ->
+ // If accessPoint then region is known from Arn
+ accessPoint != null
+ ? accessPoint.getRegion()
+ : s3Client.getBucketLocation(GetBucketLocationRequest.builder()
+ .bucket(bucketName)
+ .build())
+ .locationConstraintAsString()));
+ return fixBucketRegion(region);
+ }
+
+ /**
+ * S3AInternals method.
+ * {@inheritDoc}.
+ */
+ @Override
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ public HeadObjectResponse getObjectMetadata(Path path) throws IOException {
+ return trackDurationAndSpan(INVOCATION_GET_FILE_STATUS, path, () ->
+ S3AFileSystem.this.getObjectMetadata(makeQualified(path), null, invoker,
+ "getObjectMetadata"));
+ }
+
+ /**
+ * S3AInternals method.
+ * {@inheritDoc}.
+ */
+ @Override
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ public HeadBucketResponse getBucketMetadata() throws IOException {
+ return S3AFileSystem.this.getBucketMetadata();
+ }
+
+ /**
+ * Get a shared copy of the AWS credentials, with its reference
+ * counter updated.
+ * Caller is required to call {@code close()} on this after
+ * they have finished using it.
+ * @param purpose what is this for? This is initially for logging
+ * @return a reference to shared credentials.
+ */
+ public AWSCredentialProviderList shareCredentials(final String purpose) {
+ LOG.debug("Sharing credentials for: {}", purpose);
+ return credentials.share();
+ }
}
/**
@@ -1349,7 +1490,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
}
/**
- * Get the encryption algorithm of this endpoint.
+ * Get the encryption algorithm of this connector.
* @return the encryption algorithm.
*/
public S3AEncryptionMethods getS3EncryptionAlgorithm() {
@@ -1396,6 +1537,8 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* Get the bucket of this filesystem.
* @return the bucket
*/
+ @InterfaceAudience.Public
+ @InterfaceStability.Stable
public String getBucket() {
return bucket;
}
@@ -1413,7 +1556,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* Get the canned ACL of this FS.
* @return an ACL, if any
*/
- CannedAccessControlList getCannedACL() {
+ String getCannedACL() {
return cannedACL;
}
@@ -1643,18 +1786,18 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
}
@Override
- public GetObjectRequest newGetRequest(final String key) {
+ public GetObjectRequest.Builder newGetRequestBuilder(final String key) {
// active the audit span used for the operation
try (AuditSpan span = auditSpan.activate()) {
- return getRequestFactory().newGetObjectRequest(key);
+ return getRequestFactory().newGetObjectRequestBuilder(key);
}
}
@Override
- public S3Object getObject(GetObjectRequest request) {
+ public ResponseInputStream<GetObjectResponse> getObject(GetObjectRequest request) {
// active the audit span used for the operation
try (AuditSpan span = auditSpan.activate()) {
- return s3.getObject(request);
+ return s3Client.getObject(request);
}
}
@@ -1681,18 +1824,19 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
implements WriteOperationHelper.WriteOperationHelperCallbacks {
@Override
- public SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request) {
- return s3.selectObjectContent(request);
+ public CompletableFuture<Void> selectObjectContent(
+ SelectObjectContentRequest request,
+ SelectObjectContentResponseHandler responseHandler) {
+ return getS3AsyncClient().selectObjectContent(request, responseHandler);
}
@Override
- public CompleteMultipartUploadResult completeMultipartUpload(
+ public CompleteMultipartUploadResponse completeMultipartUpload(
CompleteMultipartUploadRequest request) {
- return s3.completeMultipartUpload(request);
+ return s3Client.completeMultipartUpload(request);
}
}
-
/**
* Create the read context for reading from the referenced file,
* using FS state as well as the status.
@@ -2054,7 +2198,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
innerRename(src, dst));
LOG.debug("Copied {} bytes", bytesCopied);
return true;
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw translateException("rename(" + src +", " + dst + ")", src, e);
} catch (RenameFailedException e) {
LOG.info("{}", e.getMessage());
@@ -2165,7 +2309,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* This operation throws an exception on any failure which needs to be
* reported and downgraded to a failure.
* Retries: retry translated, assuming all operations it is called do
- * so. For safely, consider catch and handle AmazonClientException
+ * so. For safely, consider catch and handle SdkException
* because this is such a complex method there's a risk it could surface.
* @param source path to be renamed
* @param dest new path after rename
@@ -2176,12 +2320,12 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @return the number of bytes copied.
* @throws FileNotFoundException there's no source file.
* @throws IOException on IO failure.
- * @throws AmazonClientException on failures inside the AWS SDK
+ * @throws SdkException on failures inside the AWS SDK
*/
@Retries.RetryMixed
private long innerRename(Path source, Path dest)
throws RenameFailedException, FileNotFoundException, IOException,
- AmazonClientException {
+ SdkException {
Path src = qualify(source);
Path dst = qualify(dest);
@@ -2276,7 +2420,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
}
@Override
- public CopyResult copyFile(final String srcKey,
+ public CopyObjectResponse copyFile(final String srcKey,
final String destKey,
final S3ObjectAttributes srcAttributes,
final S3AReadOpContext readContext) throws IOException {
@@ -2287,9 +2431,9 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
@Override
public void removeKeys(
- final List<DeleteObjectsRequest.KeyVersion> keysToDelete,
+ final List<ObjectIdentifier> keysToDelete,
final boolean deleteFakeDir)
- throws MultiObjectDeleteException, AmazonClientException, IOException {
+ throws MultiObjectDeleteException, SdkException, IOException {
auditSpan.activate();
S3AFileSystem.this.removeKeys(keysToDelete, deleteFakeDir);
}
@@ -2388,21 +2532,17 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* Low-level call to get at the object metadata.
* This method is used in some external applications and so
* must be viewed as a public entry point.
- * Auditing: An audit entry point.
+ * @deprecated use S3AInternals API.
* @param path path to the object. This will be qualified.
* @return metadata
* @throws IOException IO and object access problems.
*/
- @VisibleForTesting
@AuditEntryPoint
@InterfaceAudience.LimitedPrivate("utilities")
@Retries.RetryTranslated
- @InterfaceStability.Evolving
- public ObjectMetadata getObjectMetadata(Path path) throws IOException {
- V2Migration.v1GetObjectMetadataCalled();
- return trackDurationAndSpan(INVOCATION_GET_FILE_STATUS, path, () ->
- getObjectMetadata(makeQualified(path), null, invoker,
- "getObjectMetadata"));
+ @Deprecated
+ public HeadObjectResponse getObjectMetadata(Path path) throws IOException {
+ return getS3AInternals().getObjectMetadata(path);
}
/**
@@ -2415,7 +2555,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @throws IOException IO and object access problems.
*/
@Retries.RetryTranslated
- private ObjectMetadata getObjectMetadata(Path path,
+ private HeadObjectResponse getObjectMetadata(Path path,
ChangeTracker changeTracker, Invoker changeInvoker, String operation)
throws IOException {
String key = pathToKey(path);
@@ -2628,7 +2768,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
@Retries.RetryRaw
@VisibleForTesting
@InterfaceAudience.LimitedPrivate("external utilities")
- ObjectMetadata getObjectMetadata(String key) throws IOException {
+ HeadObjectResponse getObjectMetadata(String key) throws IOException {
return getObjectMetadata(key, null, invoker, "getObjectMetadata");
}
@@ -2645,28 +2785,28 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @throws RemoteFileChangedException if an unexpected version is detected
*/
@Retries.RetryRaw
- protected ObjectMetadata getObjectMetadata(String key,
+ protected HeadObjectResponse getObjectMetadata(String key,
ChangeTracker changeTracker,
Invoker changeInvoker,
String operation) throws IOException {
- ObjectMetadata meta = changeInvoker.retryUntranslated("GET " + key, true,
+ HeadObjectResponse response = changeInvoker.retryUntranslated("GET " + key, true,
() -> {
- GetObjectMetadataRequest request
- = getRequestFactory().newGetObjectMetadataRequest(key);
+ HeadObjectRequest.Builder requestBuilder =
+ getRequestFactory().newHeadObjectRequestBuilder(key);
incrementStatistic(OBJECT_METADATA_REQUESTS);
DurationTracker duration = getDurationTrackerFactory()
.trackDuration(ACTION_HTTP_HEAD_REQUEST.getSymbol());
try {
LOG.debug("HEAD {} with change tracker {}", key, changeTracker);
if (changeTracker != null) {
- changeTracker.maybeApplyConstraint(request);
+ changeTracker.maybeApplyConstraint(requestBuilder);
}
- ObjectMetadata objectMetadata = s3.getObjectMetadata(request);
+ HeadObjectResponse headObjectResponse = s3Client.headObject(requestBuilder.build());
if (changeTracker != null) {
- changeTracker.processMetadata(objectMetadata, operation);
+ changeTracker.processMetadata(headObjectResponse, operation);
}
- return objectMetadata;
- } catch(AmazonServiceException ase) {
+ return headObjectResponse;
+ } catch (AwsServiceException ase) {
if (!isObjectNotFound(ase)) {
// file not found is not considered a failure of the call,
// so only switch the duration tracker to update failure
@@ -2680,7 +2820,28 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
}
});
incrementReadOperations();
- return meta;
+ return response;
+ }
+
+ /**
+ * Request bucket metadata.
+ * @return the metadata
+ * @throws UnknownStoreException the bucket is absent
+ * @throws IOException any other problem talking to S3
+ */
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ protected HeadBucketResponse getBucketMetadata() throws IOException {
+ final HeadBucketResponse response = trackDurationAndSpan(STORE_EXISTS_PROBE, bucket, null,
+ () -> invoker.retry("getBucketMetadata()", bucket, true, () -> {
+ try {
+ return s3Client.headBucket(
+ getRequestFactory().newHeadBucketRequestBuilder(bucket).build());
+ } catch (NoSuchBucketException e) {
+ throw new UnknownStoreException("s3a://" + bucket + "/", " Bucket does " + "not exist");
+ }
+ }));
+ return response;
}
/**
@@ -2709,9 +2870,9 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
OBJECT_LIST_REQUEST,
() -> {
if (useListV1) {
- return S3ListResult.v1(s3.listObjects(request.getV1()));
+ return S3ListResult.v1(s3Client.listObjects(request.getV1()));
} else {
- return S3ListResult.v2(s3.listObjectsV2(request.getV2()));
+ return S3ListResult.v2(s3Client.listObjectsV2(request.getV2()));
}
}));
}
@@ -2754,15 +2915,21 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
OBJECT_CONTINUE_LIST_REQUEST,
() -> {
if (useListV1) {
- return S3ListResult.v1(
- s3.listNextBatchOfObjects(
- getRequestFactory()
- .newListNextBatchOfObjectsRequest(
- prevResult.getV1())));
+ List<S3Object> prevListResult = prevResult.getV1().contents();
+
+ // Next markers are only present when a delimiter is specified.
+ String nextMarker;
+ if (prevResult.getV1().nextMarker() != null) {
+ nextMarker = prevResult.getV1().nextMarker();
+ } else {
+ nextMarker = prevListResult.get(prevListResult.size() - 1).key();
+ }
+
+ return S3ListResult.v1(s3Client.listObjects(
+ request.getV1().toBuilder().marker(nextMarker).build()));
} else {
- request.getV2().setContinuationToken(prevResult.getV2()
- .getNextContinuationToken());
- return S3ListResult.v2(s3.listObjectsV2(request.getV2()));
+ return S3ListResult.v2(s3Client.listObjectsV2(request.getV2().toBuilder()
+ .continuationToken(prevResult.getV2().nextContinuationToken()).build()));
}
}));
}
@@ -2792,14 +2959,14 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
*
* Retry policy: retry untranslated; delete considered idempotent.
* @param key key to blob to delete.
- * @throws AmazonClientException problems working with S3
+ * @throws SdkException problems working with S3
* @throws InvalidRequestException if the request was rejected due to
* a mistaken attempt to delete the root directory.
*/
@VisibleForTesting
@Retries.RetryRaw
protected void deleteObject(String key)
- throws AmazonClientException, IOException {
+ throws SdkException, IOException {
blockRootDelete(key);
incrementWriteOperations();
try (DurationInfo ignored =
@@ -2811,8 +2978,9 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
incrementStatistic(OBJECT_DELETE_OBJECTS);
trackDurationOfInvocation(getDurationTrackerFactory(),
OBJECT_DELETE_REQUEST.getSymbol(),
- () -> s3.deleteObject(getRequestFactory()
- .newDeleteObjectRequest(key)));
+ () -> s3Client.deleteObject(getRequestFactory()
+ .newDeleteObjectRequestBuilder(key)
+ .build()));
return null;
});
}
@@ -2825,14 +2993,14 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @param f path path to delete
* @param key key of entry
* @param isFile is the path a file (used for instrumentation only)
- * @throws AmazonClientException problems working with S3
+ * @throws SdkException problems working with S3
* @throws IOException from invoker signature only -should not be raised.
*/
@Retries.RetryRaw
void deleteObjectAtPath(Path f,
String key,
boolean isFile)
- throws AmazonClientException, IOException {
+ throws SdkException, IOException {
if (isFile) {
instrumentation.fileDeleted(1);
} else {
@@ -2874,66 +3042,58 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @return the AWS response
* @throws MultiObjectDeleteException one or more of the keys could not
* be deleted.
- * @throws AmazonClientException amazon-layer failure.
+ * @throws SdkException amazon-layer failure.
*/
@Retries.RetryRaw
- private DeleteObjectsResult deleteObjects(DeleteObjectsRequest deleteRequest)
- throws MultiObjectDeleteException, AmazonClientException, IOException {
+ private DeleteObjectsResponse deleteObjects(DeleteObjectsRequest deleteRequest)
+ throws MultiObjectDeleteException, SdkException, IOException {
incrementWriteOperations();
BulkDeleteRetryHandler retryHandler =
new BulkDeleteRetryHandler(createStoreContext());
- int keyCount = deleteRequest.getKeys().size();
- try(DurationInfo ignored =
+ int keyCount = deleteRequest.delete().objects().size();
+ try (DurationInfo ignored =
new DurationInfo(LOG, false, "DELETE %d keys",
keyCount)) {
- return invoker.retryUntranslated("delete",
- DELETE_CONSIDERED_IDEMPOTENT,
- (text, e, r, i) -> {
- // handle the failure
- retryHandler.bulkDeleteRetried(deleteRequest, e);
- },
- // duration is tracked in the bulk delete counters
- trackDurationOfOperation(getDurationTrackerFactory(),
- OBJECT_BULK_DELETE_REQUEST.getSymbol(), () -> {
- incrementStatistic(OBJECT_DELETE_OBJECTS, keyCount);
- return s3.deleteObjects(deleteRequest);
- }));
- } catch (MultiObjectDeleteException e) {
- // one or more of the keys could not be deleted.
- // log and rethrow
- List<MultiObjectDeleteException.DeleteError> errors = e.getErrors();
- LOG.debug("Partial failure of delete, {} errors", errors.size(), e);
- for (MultiObjectDeleteException.DeleteError error : errors) {
- LOG.debug("{}: \"{}\" - {}",
- error.getKey(), error.getCode(), error.getMessage());
+ DeleteObjectsResponse response =
+ invoker.retryUntranslated("delete", DELETE_CONSIDERED_IDEMPOTENT,
+ (text, e, r, i) -> {
+ // handle the failure
+ retryHandler.bulkDeleteRetried(deleteRequest, e);
+ },
+ // duration is tracked in the bulk delete counters
+ trackDurationOfOperation(getDurationTrackerFactory(),
+ OBJECT_BULK_DELETE_REQUEST.getSymbol(), () -> {
+ incrementStatistic(OBJECT_DELETE_OBJECTS, keyCount);
+ return s3Client.deleteObjects(deleteRequest);
+ }));
+
+ if (!response.errors().isEmpty()) {
+ // one or more of the keys could not be deleted.
+ // log and then throw
+ List<S3Error> errors = response.errors();
+ LOG.debug("Partial failure of delete, {} errors", errors.size());
+ for (S3Error error : errors) {
+ LOG.debug("{}: \"{}\" - {}", error.key(), error.code(), error.message());
+ }
+ throw new MultiObjectDeleteException(errors);
}
- throw e;
+
+ return response;
}
}
/**
- * Create a putObject request.
+ * Create a putObject request builder.
* Adds the ACL and metadata
* @param key key of object
- * @param metadata metadata header
- * @param srcfile source file
+ * @param length length of object to be uploaded
+ * @param isDirectoryMarker true if object to be uploaded is a directory marker
* @return the request
*/
- public PutObjectRequest newPutObjectRequest(String key,
- ObjectMetadata metadata, File srcfile) {
- return requestFactory.newPutObjectRequest(key, metadata, null, srcfile);
- }
-
- /**
- * Create a new object metadata instance.
- * Any standard metadata headers are added here, for example:
- * encryption.
- *
- * @param length length of data to set in header.
- * @return a new metadata instance
- */
- public ObjectMetadata newObjectMetadata(long length) {
- return requestFactory.newObjectMetadata(length);
+ public PutObjectRequest.Builder newPutObjectRequestBuilder(String key,
+ long length,
+ boolean isDirectoryMarker) {
+ return requestFactory.newPutObjectRequestBuilder(key, null, length, isDirectoryMarker);
}
/**
@@ -2950,15 +3110,24 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* Retry policy: N/A: the transfer manager is performing the upload.
* Auditing: must be inside an audit span.
* @param putObjectRequest the request
+ * @param file the file to be uploaded
+ * @param listener the progress listener for the request
* @return the upload initiated
*/
@Retries.OnceRaw
- public UploadInfo putObject(PutObjectRequest putObjectRequest) {
+ public UploadInfo putObject(PutObjectRequest putObjectRequest, File file,
+ ProgressableProgressListener listener) {
long len = getPutRequestLength(putObjectRequest);
- LOG.debug("PUT {} bytes to {} via transfer manager ",
- len, putObjectRequest.getKey());
+ LOG.debug("PUT {} bytes to {} via transfer manager ", len, putObjectRequest.key());
incrementPutStartStatistics(len);
- Upload upload = transfers.upload(putObjectRequest);
+
+ FileUpload upload = transferManager.uploadFile(
+ UploadFileRequest.builder()
+ .putObjectRequest(putObjectRequest)
+ .source(file)
+ .addTransferListener(listener)
+ .build());
+
return new UploadInfo(upload, len);
}
@@ -2973,30 +3142,37 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @param putObjectRequest the request
* @param putOptions put object options
* @param durationTrackerFactory factory for duration tracking
+ * @param uploadData data to be uploaded
+ * @param isFile represents if data to be uploaded is a file
* @return the upload initiated
- * @throws AmazonClientException on problems
+ * @throws SdkException on problems
*/
@VisibleForTesting
@Retries.OnceRaw("For PUT; post-PUT actions are RetryExceptionsSwallowed")
- PutObjectResult putObjectDirect(PutObjectRequest putObjectRequest,
+ PutObjectResponse putObjectDirect(PutObjectRequest putObjectRequest,
PutObjectOptions putOptions,
+ S3ADataBlocks.BlockUploadData uploadData, boolean isFile,
DurationTrackerFactory durationTrackerFactory)
- throws AmazonClientException {
+ throws SdkException {
long len = getPutRequestLength(putObjectRequest);
- LOG.debug("PUT {} bytes to {}", len, putObjectRequest.getKey());
+ LOG.debug("PUT {} bytes to {}", len, putObjectRequest.key());
incrementPutStartStatistics(len);
try {
- PutObjectResult result = trackDurationOfSupplier(
- nonNullDurationTrackerFactory(durationTrackerFactory),
- OBJECT_PUT_REQUESTS.getSymbol(), () ->
- s3.putObject(putObjectRequest));
+ PutObjectResponse response =
+ trackDurationOfSupplier(nonNullDurationTrackerFactory(durationTrackerFactory),
+ OBJECT_PUT_REQUESTS.getSymbol(),
+ () -> isFile ?
+ s3Client.putObject(putObjectRequest, RequestBody.fromFile(uploadData.getFile())) :
+ s3Client.putObject(putObjectRequest,
+ RequestBody.fromInputStream(uploadData.getUploadStream(),
+ putObjectRequest.contentLength())));
incrementPutCompletedStatistics(true, len);
// apply any post-write actions.
- finishedWrite(putObjectRequest.getKey(), len,
- result.getETag(), result.getVersionId(),
+ finishedWrite(putObjectRequest.key(), len,
+ response.eTag(), response.versionId(),
putOptions);
- return result;
- } catch (SdkBaseException e) {
+ return response;
+ } catch (SdkException e) {
incrementPutCompletedStatistics(false, len);
throw e;
}
@@ -3009,12 +3185,8 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @throws IllegalArgumentException if the length is negative
*/
private long getPutRequestLength(PutObjectRequest putObjectRequest) {
- long len;
- if (putObjectRequest.getFile() != null) {
- len = putObjectRequest.getFile().length();
- } else {
- len = putObjectRequest.getMetadata().getContentLength();
- }
+ long len = putObjectRequest.contentLength();
+
Preconditions.checkState(len >= 0, "Cannot PUT object of unknown length");
return len;
}
@@ -3022,28 +3194,29 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
/**
* Upload part of a multi-partition file.
* Increments the write and put counters.
- * <i>Important: this call does not close any input stream in the request.</i>
+ * <i>Important: this call does not close any input stream in the body.</i>
*
* Retry Policy: none.
- * @param request request
* @param durationTrackerFactory duration tracker factory for operation
+ * @param request the upload part request.
+ * @param body the request body.
* @return the result of the operation.
- * @throws AmazonClientException on problems
+ * @throws AwsServiceException on problems
*/
@Retries.OnceRaw
- UploadPartResult uploadPart(UploadPartRequest request,
+ UploadPartResponse uploadPart(UploadPartRequest request, RequestBody body,
final DurationTrackerFactory durationTrackerFactory)
- throws AmazonClientException {
- long len = request.getPartSize();
+ throws AwsServiceException {
+ long len = request.contentLength();
incrementPutStartStatistics(len);
try {
- UploadPartResult uploadPartResult = trackDurationOfSupplier(
+ UploadPartResponse uploadPartResponse = trackDurationOfSupplier(
nonNullDurationTrackerFactory(durationTrackerFactory),
MULTIPART_UPLOAD_PART_PUT.getSymbol(), () ->
- s3.uploadPart(request));
+ s3Client.uploadPart(request, body));
incrementPutCompletedStatistics(true, len);
- return uploadPartResult;
- } catch (AmazonClientException e) {
+ return uploadPartResponse;
+ } catch (AwsServiceException e) {
incrementPutCompletedStatistics(false, len);
throw e;
}
@@ -3108,56 +3281,57 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* be deleted in a multiple object delete operation.
* The number of rejected objects will be added to the metric
* {@link Statistic#FILES_DELETE_REJECTED}.
- * @throws AmazonClientException other amazon-layer failure.
+ * @throws AwsServiceException other amazon-layer failure.
*/
@Retries.RetryRaw
private void removeKeysS3(
- List<DeleteObjectsRequest.KeyVersion> keysToDelete,
+ List<ObjectIdentifier> keysToDelete,
boolean deleteFakeDir)
- throws MultiObjectDeleteException, AmazonClientException,
- IOException {
+ throws MultiObjectDeleteException, AwsServiceException, IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("Initiating delete operation for {} objects",
keysToDelete.size());
- for (DeleteObjectsRequest.KeyVersion key : keysToDelete) {
- LOG.debug(" {} {}", key.getKey(),
- key.getVersion() != null ? key.getVersion() : "");
+ for (ObjectIdentifier objectIdentifier : keysToDelete) {
+ LOG.debug(" {} {}", objectIdentifier.key(),
+ objectIdentifier.versionId() != null ? objectIdentifier.versionId() : "");
}
}
if (keysToDelete.isEmpty()) {
// exit fast if there are no keys to delete
return;
}
- for (DeleteObjectsRequest.KeyVersion keyVersion : keysToDelete) {
- blockRootDelete(keyVersion.getKey());
+ for (ObjectIdentifier objectIdentifier : keysToDelete) {
+ blockRootDelete(objectIdentifier.key());
}
try {
if (enableMultiObjectsDelete) {
if (keysToDelete.size() <= pageSize) {
deleteObjects(getRequestFactory()
- .newBulkDeleteRequest(keysToDelete));
+ .newBulkDeleteRequestBuilder(keysToDelete)
+ .build());
} else {
// Multi object deletion of more than 1000 keys is not supported
// by s3. So we are paging the keys by page size.
LOG.debug("Partitioning the keys to delete as it is more than " +
"page size. Number of keys: {}, Page size: {}",
keysToDelete.size(), pageSize);
- for (List<DeleteObjectsRequest.KeyVersion> batchOfKeysToDelete :
+ for (List<ObjectIdentifier> batchOfKeysToDelete :
Lists.partition(keysToDelete, pageSize)) {
deleteObjects(getRequestFactory()
- .newBulkDeleteRequest(batchOfKeysToDelete));
+ .newBulkDeleteRequestBuilder(batchOfKeysToDelete)
+ .build());
}
}
} else {
- for (DeleteObjectsRequest.KeyVersion keyVersion : keysToDelete) {
- deleteObject(keyVersion.getKey());
+ for (ObjectIdentifier objectIdentifier : keysToDelete) {
+ deleteObject(objectIdentifier.key());
}
}
} catch (MultiObjectDeleteException ex) {
// partial delete.
// Update the stats with the count of the actual number of successful
// deletions.
- int rejected = ex.getErrors().size();
+ int rejected = ex.errors().size();
noteDeleted(keysToDelete.size() - rejected, deleteFakeDir);
incrementStatistic(FILES_DELETE_REJECTED, rejected);
throw ex;
@@ -3190,15 +3364,15 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* a mistaken attempt to delete the root directory.
* @throws MultiObjectDeleteException one or more of the keys could not
* be deleted in a multiple object delete operation.
- * @throws AmazonClientException amazon-layer failure.
+ * @throws AwsServiceException amazon-layer failure.
* @throws IOException other IO Exception.
*/
@VisibleForTesting
@Retries.RetryRaw
public void removeKeys(
- final List<DeleteObjectsRequest.KeyVersion> keysToDelete,
+ final List<ObjectIdentifier> keysToDelete,
final boolean deleteFakeDir)
- throws MultiObjectDeleteException, AmazonClientException,
+ throws MultiObjectDeleteException, AwsServiceException,
IOException {
try (DurationInfo ignored = new DurationInfo(LOG, false,
"Deleting %d keys", keysToDelete.size())) {
@@ -3268,7 +3442,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
LOG.debug("Couldn't delete {} - does not exist: {}", path, e.toString());
instrumentation.errorIgnored();
return false;
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw translateException("delete", path, e);
}
}
@@ -3282,7 +3456,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
*/
@Retries.RetryTranslated
private void createFakeDirectoryIfNecessary(Path f)
- throws IOException, AmazonClientException {
+ throws IOException, SdkException {
String key = pathToKey(f);
// we only make the LIST call; the codepaths to get here should not
// be reached if there is an empty dir marker -and if they do, it
@@ -3302,7 +3476,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
@Retries.RetryTranslated
@VisibleForTesting
protected void maybeCreateFakeParentDirectory(Path path)
- throws IOException, AmazonClientException {
+ throws IOException, SdkException {
Path parent = path.getParent();
if (parent != null && !parent.isRoot() && !isUnderMagicCommitPath(parent)) {
createFakeDirectoryIfNecessary(parent);
@@ -3356,11 +3530,11 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @return the statuses of the files/directories in the given patch
* @throws FileNotFoundException when the path does not exist;
* @throws IOException due to an IO problem.
- * @throws AmazonClientException on failures inside the AWS SDK
+ * @throws SdkException on failures inside the AWS SDK
*/
private RemoteIterator<S3AFileStatus> innerListStatus(Path f)
throws FileNotFoundException,
- IOException, AmazonClientException {
+ IOException, SdkException {
Path path = qualify(f);
LOG.debug("List status for path: {}", path);
@@ -3424,15 +3598,15 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
private S3ListRequest createListObjectsRequest(String key,
String delimiter, int limit) {
if (!useListV1) {
- ListObjectsV2Request request =
- getRequestFactory().newListObjectsV2Request(
+ ListObjectsV2Request.Builder requestBuilder =
+ getRequestFactory().newListObjectsV2RequestBuilder(
key, delimiter, limit);
- return S3ListRequest.v2(request);
+ return S3ListRequest.v2(requestBuilder.build());
} else {
- ListObjectsRequest request =
- getRequestFactory().newListObjectsV1Request(
+ ListObjectsRequest.Builder requestBuilder =
+ getRequestFactory().newListObjectsV1RequestBuilder(
key, delimiter, limit);
- return S3ListRequest.v1(request);
+ return S3ListRequest.v1(requestBuilder.build());
}
}
@@ -3720,31 +3894,31 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
&& probes.contains(StatusProbeEnum.Head)) {
try {
// look for the simple file
- ObjectMetadata meta = getObjectMetadata(key);
+ HeadObjectResponse meta = getObjectMetadata(key);
LOG.debug("Found exact file: normal file {}", key);
- long contentLength = meta.getContentLength();
+ long contentLength = meta.contentLength();
// check if CSE is enabled, then strip padded length.
- if (isCSEEnabled
- && meta.getUserMetaDataOf(Headers.CRYPTO_CEK_ALGORITHM) != null
+ if (isCSEEnabled &&
+ meta.metadata().get(AWSHeaders.CRYPTO_CEK_ALGORITHM) != null
&& contentLength >= CSE_PADDING_LENGTH) {
contentLength -= CSE_PADDING_LENGTH;
}
return new S3AFileStatus(contentLength,
- dateToLong(meta.getLastModified()),
+ meta.lastModified().toEpochMilli(),
path,
getDefaultBlockSize(path),
username,
- meta.getETag(),
- meta.getVersionId());
- } catch (AmazonServiceException e) {
+ meta.eTag(),
+ meta.versionId());
+ } catch (AwsServiceException e) {
// if the response is a 404 error, it just means that there is
// no file at that path...the remaining checks will be needed.
// But: an empty bucket is also a 404, so check for that
// and fail.
- if (e.getStatusCode() != SC_404 || isUnknownBucket(e)) {
+ if (e.statusCode() != SC_404_NOT_FOUND || isUnknownBucket(e)) {
throw translateException("getFileStatus", path, e);
}
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw translateException("getFileStatus", path, e);
}
}
@@ -3787,11 +3961,11 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
LOG.debug("Found root directory");
return new S3AFileStatus(Tristate.TRUE, path, username);
}
- } catch (AmazonServiceException e) {
- if (e.getStatusCode() != SC_404 || isUnknownBucket(e)) {
+ } catch (AwsServiceException e) {
+ if (e.statusCode() != SC_404_NOT_FOUND || isUnknownBucket(e)) {
throw translateException("getFileStatus", path, e);
}
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw translateException("getFileStatus", path, e);
}
}
@@ -3835,7 +4009,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @throws IOException IO problem
* @throws FileAlreadyExistsException the destination file exists and
* overwrite==false
- * @throws AmazonClientException failure in the AWS SDK
+ * @throws SdkException failure in the AWS SDK
*/
@Override
@AuditEntryPoint
@@ -3890,13 +4064,12 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
to,
() -> {
final String key = pathToKey(to);
- final ObjectMetadata om = newObjectMetadata(file.length());
Progressable progress = null;
- PutObjectRequest putObjectRequest = newPutObjectRequest(key, om, file);
- S3AFileSystem.this.invoker.retry(
- "putObject(" + "" + ")", to.toString(),
- true,
- () -> executePut(putObjectRequest, progress, putOptionsForPath(to)));
+ PutObjectRequest.Builder putObjectRequestBuilder =
+ newPutObjectRequestBuilder(key, file.length(), false);
+ S3AFileSystem.this.invoker.retry("putObject(" + "" + ")", to.toString(), true,
+ () -> executePut(putObjectRequestBuilder.build(), progress, putOptionsForPath(to),
+ file));
return null;
});
@@ -3921,40 +4094,35 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
/**
* Execute a PUT via the transfer manager, blocking for completion.
- * If the waiting for completion is interrupted, the upload will be
- * aborted before an {@code InterruptedIOException} is thrown.
* @param putObjectRequest request
* @param progress optional progress callback
* @param putOptions put object options
* @return the upload result
- * @throws InterruptedIOException if the blocking was interrupted.
+ * @throws IOException IO failure
*/
@Retries.OnceRaw("For PUT; post-PUT actions are RetrySwallowed")
- UploadResult executePut(
+ PutObjectResponse executePut(
final PutObjectRequest putObjectRequest,
final Progressable progress,
- final PutObjectOptions putOptions)
- throws InterruptedIOException {
- String key = putObjectRequest.getKey();
+ final PutObjectOptions putOptions,
+ final File file)
+ throws IOException {
+ String key = putObjectRequest.key();
long len = getPutRequestLength(putObjectRequest);
- UploadInfo info = putObject(putObjectRequest);
- Upload upload = info.getUpload();
- ProgressableProgressListener listener = new ProgressableProgressListener(
- this, key, upload, progress);
- upload.addProgressListener(listener);
- UploadResult result = waitForUploadCompletion(key, info);
- listener.uploadCompleted();
+ ProgressableProgressListener listener =
+ new ProgressableProgressListener(this, putObjectRequest.key(), progress);
+ UploadInfo info = putObject(putObjectRequest, file, listener);
+ PutObjectResponse result = waitForUploadCompletion(key, info).response();
+ listener.uploadCompleted(info.getFileUpload());
// post-write actions
finishedWrite(key, len,
- result.getETag(), result.getVersionId(), putOptions);
+ result.eTag(), result.versionId(), putOptions);
return result;
}
/**
* Wait for an upload to complete.
- * If the waiting for completion is interrupted, the upload will be
- * aborted before an {@code InterruptedIOException} is thrown.
* If the upload (or its result collection) failed, this is where
* the failure is raised as an AWS exception.
* Calls {@link #incrementPutCompletedStatistics(boolean, long)}
@@ -3962,24 +4130,20 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @param key destination key
* @param uploadInfo upload to wait for
* @return the upload result
- * @throws InterruptedIOException if the blocking was interrupted.
+ * @throws IOException IO failure
*/
@Retries.OnceRaw
- UploadResult waitForUploadCompletion(String key, UploadInfo uploadInfo)
- throws InterruptedIOException {
- Upload upload = uploadInfo.getUpload();
+ CompletedFileUpload waitForUploadCompletion(String key, UploadInfo uploadInfo)
+ throws IOException {
+ FileUpload upload = uploadInfo.getFileUpload();
try {
- UploadResult result = upload.waitForUploadResult();
+ CompletedFileUpload result = upload.completionFuture().join();
incrementPutCompletedStatistics(true, uploadInfo.getLength());
return result;
- } catch (InterruptedException e) {
+ } catch (CompletionException e) {
LOG.info("Interrupted: aborting upload");
incrementPutCompletedStatistics(false, uploadInfo.getLength());
- upload.abort();
- throw (InterruptedIOException)
- new InterruptedIOException("Interrupted in PUT to "
- + keyToQualifiedPath(key))
- .initCause(e);
+ throw extractException("upload", key, e);
}
}
@@ -4072,17 +4236,13 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* both the expected state of this FS and of failures while being stopped.
*/
protected synchronized void stopAllServices() {
- // shutting down the transfer manager also shuts
- // down the S3 client it is bonded to.
- if (transfers != null) {
- try {
- transfers.shutdownNow(true);
- } catch (RuntimeException e) {
- // catch and swallow for resilience.
- LOG.debug("When shutting down", e);
- }
- transfers = null;
- }
+ closeAutocloseables(LOG, transferManager,
+ s3Client,
+ getS3AsyncClient());
+ transferManager = null;
+ s3Client = null;
+ s3AsyncClient = null;
+
// At this point the S3A client is shut down,
// now the executor pools are closed
HadoopExecutors.shutdown(boundedThreadPool, LOG,
@@ -4237,21 +4397,11 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @throws IOException Other IO problems
*/
@Retries.RetryTranslated
- private CopyResult copyFile(String srcKey, String dstKey, long size,
+ private CopyObjectResponse copyFile(String srcKey, String dstKey, long size,
S3ObjectAttributes srcAttributes, S3AReadOpContext readContext)
- throws IOException, InterruptedIOException {
+ throws IOException {
LOG.debug("copyFile {} -> {} ", srcKey, dstKey);
- ProgressListener progressListener = progressEvent -> {
- switch (progressEvent.getEventType()) {
- case TRANSFER_PART_COMPLETED_EVENT:
- incrementWriteOperations();
- break;
- default:
- break;
- }
- };
-
ChangeTracker changeTracker = new ChangeTracker(
keyToQualifiedPath(srcKey).toString(),
changeDetectionPolicy,
@@ -4263,7 +4413,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
String action = "copyFile(" + srcKey + ", " + dstKey + ")";
Invoker readInvoker = readContext.getReadInvoker();
- ObjectMetadata srcom;
+ HeadObjectResponse srcom;
try {
srcom = once(action, srcKey,
() ->
@@ -4286,33 +4436,32 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
action, srcKey,
true,
() -> {
- CopyObjectRequest copyObjectRequest =
- getRequestFactory().newCopyObjectRequest(srcKey, dstKey, srcom);
- changeTracker.maybeApplyConstraint(copyObjectRequest);
+ CopyObjectRequest.Builder copyObjectRequestBuilder =
+ getRequestFactory().newCopyObjectRequestBuilder(srcKey, dstKey, srcom);
+ changeTracker.maybeApplyConstraint(copyObjectRequestBuilder);
incrementStatistic(OBJECT_COPY_REQUESTS);
- Copy copy = transfers.copy(copyObjectRequest,
- getAuditManager().createStateChangeListener());
- copy.addProgressListener(progressListener);
- CopyOutcome copyOutcome = CopyOutcome.waitForCopy(copy);
- InterruptedException interruptedException =
- copyOutcome.getInterruptedException();
- if (interruptedException != null) {
- // copy interrupted: convert to an IOException.
- throw (IOException)new InterruptedIOException(
- "Interrupted copying " + srcKey
- + " to " + dstKey + ", cancelling")
- .initCause(interruptedException);
- }
- SdkBaseException awsException = copyOutcome.getAwsException();
- if (awsException != null) {
- changeTracker.processException(awsException, "copy");
- throw awsException;
+
+ Copy copy = transferManager.copy(
+ CopyRequest.builder()
+ .copyObjectRequest(copyObjectRequestBuilder.build())
+ .build());
+
+ try {
+ CompletedCopy completedCopy = copy.completionFuture().join();
+ CopyObjectResponse result = completedCopy.response();
+ changeTracker.processResponse(result);
+ incrementWriteOperations();
+ instrumentation.filesCopied(1, size);
+ return result;
+ } catch (CompletionException e) {
+ Throwable cause = e.getCause();
+ if (cause instanceof SdkException) {
+ SdkException awsException = (SdkException)cause;
+ changeTracker.processException(awsException, "copy");
+ throw awsException;
+ }
+ throw extractException(action, srcKey, e);
}
- CopyResult result = copyOutcome.getCopyResult();
- changeTracker.processResponse(result);
- incrementWriteOperations();
- instrumentation.filesCopied(1, size);
- return result;
});
}
@@ -4321,16 +4470,16 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* Retry policy: none + untranslated.
* @param request request to initiate
* @return the result of the call
- * @throws AmazonClientException on failures inside the AWS SDK
+ * @throws SdkException on failures inside the AWS SDK
* @throws IOException Other IO problems
*/
@Retries.OnceRaw
- InitiateMultipartUploadResult initiateMultipartUpload(
- InitiateMultipartUploadRequest request) throws IOException {
- LOG.debug("Initiate multipart upload to {}", request.getKey());
+ CreateMultipartUploadResponse initiateMultipartUpload(
+ CreateMultipartUploadRequest request) throws IOException {
+ LOG.debug("Initiate multipart upload to {}", request.key());
return trackDurationOfSupplier(getDurationTrackerFactory(),
OBJECT_MULTIPART_UPLOAD_INITIATED.getSymbol(),
- () -> getAmazonS3Client().initiateMultipartUpload(request));
+ () -> s3Client.createMultipartUpload(request));
}
/**
@@ -4403,22 +4552,22 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
*/
@Retries.RetryExceptionsSwallowed
private void deleteUnnecessaryFakeDirectories(Path path) {
- List<DeleteObjectsRequest.KeyVersion> keysToRemove = new ArrayList<>();
+ List<ObjectIdentifier> keysToRemove = new ArrayList<>();
while (!path.isRoot()) {
String key = pathToKey(path);
key = (key.endsWith("/")) ? key : (key + "/");
LOG.trace("To delete unnecessary fake directory {} for {}", key, path);
- keysToRemove.add(new DeleteObjectsRequest.KeyVersion(key));
+ keysToRemove.add(ObjectIdentifier.builder().key(key).build());
path = path.getParent();
}
try {
removeKeys(keysToRemove, true);
- } catch(AmazonClientException | IOException e) {
+ } catch (AwsServiceException | IOException e) {
instrumentation.errorIgnored();
if (LOG.isDebugEnabled()) {
StringBuilder sb = new StringBuilder();
- for(DeleteObjectsRequest.KeyVersion kv : keysToRemove) {
- sb.append(kv.getKey()).append(",");
+ for (ObjectIdentifier objectIdentifier : keysToRemove) {
+ sb.append(objectIdentifier.key()).append(",");
}
LOG.debug("While deleting keys {} ", sb.toString(), e);
}
@@ -4451,11 +4600,18 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
@Retries.RetryTranslated
private void createEmptyObject(final String objectName, PutObjectOptions putOptions)
throws IOException {
- invoker.retry("PUT 0-byte object ", objectName,
- true, () ->
- putObjectDirect(getRequestFactory().newDirectoryMarkerRequest(objectName),
- putOptions,
- getDurationTrackerFactory()));
+ final InputStream im = new InputStream() {
+ @Override
+ public int read() throws IOException {
+ return -1;
+ }
+ };
+
+ S3ADataBlocks.BlockUploadData uploadData = new S3ADataBlocks.BlockUploadData(im);
+
+ invoker.retry("PUT 0-byte object ", objectName, true,
+ () -> putObjectDirect(getRequestFactory().newDirectoryMarkerRequest(objectName).build(),
+ putOptions, uploadData, false, getDurationTrackerFactory()));
incrementPutProgressStatistics(objectName, 0);
instrumentation.directoryCreated();
}
@@ -4712,10 +4868,10 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
ETAG_CHECKSUM_ENABLED_DEFAULT)) {
return trackDurationAndSpan(INVOCATION_GET_FILE_CHECKSUM, path, () -> {
LOG.debug("getFileChecksum({})", path);
- ObjectMetadata headers = getObjectMetadata(path, null,
+ HeadObjectResponse headers = getObjectMetadata(path, null,
invoker,
"getFileChecksum are");
- String eTag = headers.getETag();
+ String eTag = headers.eTag();
return eTag != null ? new EtagChecksum(eTag) : null;
});
} else {
@@ -4797,11 +4953,18 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
HeaderProcessing.HeaderProcessingCallbacks {
@Override
- public ObjectMetadata getObjectMetadata(final String key)
+ public HeadObjectResponse getObjectMetadata(final String key)
throws IOException {
return once("getObjectMetadata", key, () ->
S3AFileSystem.this.getObjectMetadata(key));
}
+
+ @Override
+ public HeadBucketResponse getBucketMetadata()
+ throws IOException {
+ return once("getBucketMetadata", bucket, () ->
+ S3AFileSystem.this.getBucketMetadata());
+ }
}
/**
* {@inheritDoc}.
@@ -4914,7 +5077,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
// If we have reached here, it means either there are files
// in this directory or it is empty.
return listFilesAssumingDir;
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw translateException("listFiles", path, e);
}
}
@@ -5012,8 +5175,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
// span is picked up retained in the listing.
return trackDurationAndSpan(MULTIPART_UPLOAD_LIST, prefix, null, () ->
MultipartUtils.listMultipartUploads(
- createStoreContext(),
- s3, prefix, maxKeys
+ createStoreContext(), s3Client, prefix, maxKeys
));
}
@@ -5024,7 +5186,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* Retry policy: retry, translated.
* @return a listing of multipart uploads.
* @param prefix prefix to scan for, "" for none
- * @throws IOException IO failure, including any uprated AmazonClientException
+ * @throws IOException IO failure, including any uprated SdkException
*/
@InterfaceAudience.Private
@Retries.RetryTranslated
@@ -5036,9 +5198,9 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
}
String p = prefix;
return invoker.retry("listMultipartUploads", p, true, () -> {
- ListMultipartUploadsRequest request = getRequestFactory()
- .newListMultipartUploadsRequest(p);
- return s3.listMultipartUploads(request).getMultipartUploads();
+ ListMultipartUploadsRequest.Builder requestBuilder = getRequestFactory()
+ .newListMultipartUploadsRequestBuilder(p);
+ return s3Client.listMultipartUploads(requestBuilder.build()).uploads();
});
}
@@ -5051,10 +5213,10 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
@Retries.OnceRaw
void abortMultipartUpload(String destKey, String uploadId) {
LOG.info("Aborting multipart upload {} to {}", uploadId, destKey);
- getAmazonS3Client().abortMultipartUpload(
- getRequestFactory().newAbortMultipartUploadRequest(
+ s3Client.abortMultipartUpload(
+ getRequestFactory().newAbortMultipartUploadRequestBuilder(
destKey,
- uploadId));
+ uploadId).build());
}
/**
@@ -5066,18 +5228,18 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
void abortMultipartUpload(MultipartUpload upload) {
String destKey;
String uploadId;
- destKey = upload.getKey();
- uploadId = upload.getUploadId();
+ destKey = upload.key();
+ uploadId = upload.uploadId();
if (LOG.isInfoEnabled()) {
DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
LOG.debug("Aborting multipart upload {} to {} initiated by {} on {}",
- uploadId, destKey, upload.getInitiator(),
- df.format(upload.getInitiated()));
+ uploadId, destKey, upload.initiator(),
+ df.format(Date.from(upload.initiated())));
}
- getAmazonS3Client().abortMultipartUpload(
- getRequestFactory().newAbortMultipartUploadRequest(
+ s3Client.abortMultipartUpload(
+ getRequestFactory().newAbortMultipartUploadRequestBuilder(
destKey,
- uploadId));
+ uploadId).build());
}
/**
@@ -5464,4 +5626,5 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
public boolean isMultipartUploadEnabled() {
return isMultipartUploadEnabled;
}
+
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
index 4b50ab2c04bd..2ed9083efcdd 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
@@ -22,6 +22,7 @@ import javax.annotation.Nullable;
import java.io.Closeable;
import java.io.EOFException;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InterruptedIOException;
import java.net.SocketTimeoutException;
import java.nio.ByteBuffer;
@@ -31,9 +32,9 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.IntFunction;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.S3ObjectInputStream;
+import software.amazon.awssdk.core.ResponseInputStream;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,7 +47,6 @@ import org.apache.hadoop.fs.CanUnbuffer;
import org.apache.hadoop.fs.FSExceptionMessages;
import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.fs.FileRange;
-import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.StreamCapabilities;
import org.apache.hadoop.fs.impl.CombinedFileRange;
import org.apache.hadoop.fs.VectoredReadUtils;
@@ -61,6 +61,7 @@ import org.apache.hadoop.fs.statistics.IOStatisticsSource;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.functional.CallableRaisingIOE;
+
import static java.util.Objects.requireNonNull;
import static org.apache.commons.lang3.StringUtils.isNotEmpty;
import static org.apache.hadoop.fs.VectoredReadUtils.isOrderedDisjoint;
@@ -125,14 +126,9 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
*/
private volatile boolean closed;
/**
- * wrappedStream is associated with an object (instance of S3Object). When
- * the object is garbage collected, the associated wrappedStream will be
- * closed. Keep a reference to this object to prevent the wrapperStream
- * still in use from being closed unexpectedly due to garbage collection.
- * See HADOOP-17338 for details.
+ * Input stream returned by a getObject call.
*/
- private S3Object object;
- private S3ObjectInputStream wrappedStream;
+ private ResponseInputStream<GetObjectResponse> wrappedStream;
private final S3AReadOpContext context;
private final InputStreamCallbacks client;
@@ -271,28 +267,22 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
uri, reason, targetPos, contentRangeFinish, length, pos, nextReadPos,
inputPolicy);
+ GetObjectRequest request = client.newGetRequestBuilder(key)
+ .range(S3AUtils.formatRange(targetPos, contentRangeFinish - 1))
+ .applyMutation(changeTracker::maybeApplyConstraint)
+ .build();
long opencount = streamStatistics.streamOpened();
- GetObjectRequest request = client.newGetRequest(key)
- .withRange(targetPos, contentRangeFinish - 1);
String operation = opencount == 0 ? OPERATION_OPEN : OPERATION_REOPEN;
String text = String.format("%s %s at %d",
operation, uri, targetPos);
- changeTracker.maybeApplyConstraint(request);
-
- object = onceTrackingDuration(text, uri,
+ wrappedStream = onceTrackingDuration(text, uri,
streamStatistics.initiateGetRequest(), () ->
client.getObject(request));
-
- changeTracker.processResponse(object, operation,
+ changeTracker.processResponse(wrappedStream.response(), operation,
targetPos);
- wrappedStream = object.getObjectContent();
- contentRangeStart = targetPos;
- if (wrappedStream == null) {
- throw new PathIOException(uri,
- "Null IO stream from " + operation + " of (" + reason + ") ");
- }
+ contentRangeStart = targetPos;
this.pos = targetPos;
}
@@ -505,14 +495,15 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
*/
@Retries.OnceTranslated
private void onReadFailure(IOException ioe, boolean forceAbort) {
+ GetObjectResponse objectResponse = wrappedStream == null ? null : wrappedStream.response();
if (LOG.isDebugEnabled()) {
LOG.debug("Got exception while trying to read from stream {}, " +
"client: {} object: {}, trying to recover: ",
- uri, client, object, ioe);
+ uri, client, objectResponse, ioe);
} else {
LOG.info("Got exception while trying to read from stream {}, " +
"client: {} object: {}, trying to recover: " + ioe,
- uri, client, object);
+ uri, client, objectResponse);
}
streamStatistics.readException();
closeStream("failure recovery", forceAbort, false);
@@ -672,7 +663,6 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
CompletableFuture<Boolean> operation;
SDKStreamDrainer drainer = new SDKStreamDrainer(
uri,
- object,
wrappedStream,
shouldAbort,
(int) remaining,
@@ -694,7 +684,6 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
// either the stream is closed in the blocking call or the async call is
// submitted with its own copy of the references
wrappedStream = null;
- object = null;
return operation;
}
@@ -910,23 +899,19 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
private void readCombinedRangeAndUpdateChildren(CombinedFileRange combinedFileRange,
IntFunction<ByteBuffer> allocate) {
LOG.debug("Start reading combined range {} from path {} ", combinedFileRange, pathStr);
- // This reference must be kept till all buffers are populated as this is a
- // finalizable object which closes the internal stream when gc triggers.
- S3Object objectRange = null;
- S3ObjectInputStream objectContent = null;
+ ResponseInputStream<GetObjectResponse> rangeContent = null;
try {
- objectRange = getS3ObjectAndValidateNotNull("readCombinedFileRange",
+ rangeContent = getS3ObjectInputStream("readCombinedFileRange",
combinedFileRange.getOffset(),
combinedFileRange.getLength());
- objectContent = objectRange.getObjectContent();
- populateChildBuffers(combinedFileRange, objectContent, allocate);
+ populateChildBuffers(combinedFileRange, rangeContent, allocate);
} catch (Exception ex) {
LOG.debug("Exception while reading a range {} from path {} ", combinedFileRange, pathStr, ex);
for(FileRange child : combinedFileRange.getUnderlying()) {
child.getData().completeExceptionally(ex);
}
} finally {
- IOUtils.cleanupWithLogger(LOG, objectRange, objectContent);
+ IOUtils.cleanupWithLogger(LOG, rangeContent);
}
LOG.debug("Finished reading range {} from path {} ", combinedFileRange, pathStr);
}
@@ -939,7 +924,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
* @throws IOException any IOE.
*/
private void populateChildBuffers(CombinedFileRange combinedFileRange,
- S3ObjectInputStream objectContent,
+ InputStream objectContent,
IntFunction<ByteBuffer> allocate) throws IOException {
// If the combined file range just contains a single child
// range, we only have to fill that one child buffer else
@@ -971,7 +956,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
* @param drainQuantity how many bytes to drain.
* @throws IOException any IOE.
*/
- private void drainUnnecessaryData(S3ObjectInputStream objectContent, long drainQuantity)
+ private void drainUnnecessaryData(InputStream objectContent, long drainQuantity)
throws IOException {
int drainBytes = 0;
int readCount;
@@ -1013,28 +998,24 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
*/
private void readSingleRange(FileRange range, ByteBuffer buffer) {
LOG.debug("Start reading range {} from path {} ", range, pathStr);
- // This reference must be kept till all buffers are populated as this is a
- // finalizable object which closes the internal stream when gc triggers.
- S3Object objectRange = null;
- S3ObjectInputStream objectContent = null;
+ ResponseInputStream<GetObjectResponse> objectRange = null;
try {
long position = range.getOffset();
int length = range.getLength();
- objectRange = getS3ObjectAndValidateNotNull("readSingleRange", position, length);
- objectContent = objectRange.getObjectContent();
- populateBuffer(length, buffer, objectContent);
+ objectRange = getS3ObjectInputStream("readSingleRange", position, length);
+ populateBuffer(length, buffer, objectRange);
range.getData().complete(buffer);
} catch (Exception ex) {
LOG.warn("Exception while reading a range {} from path {} ", range, pathStr, ex);
range.getData().completeExceptionally(ex);
} finally {
- IOUtils.cleanupWithLogger(LOG, objectRange, objectContent);
+ IOUtils.cleanupWithLogger(LOG, objectRange);
}
LOG.debug("Finished reading range {} from path {} ", range, pathStr);
}
/**
- * Get the s3 object for S3 server for a specified range.
+ * Get the s3 object input stream for S3 server for a specified range.
* Also checks if the vectored io operation has been stopped before and after
* the http get request such that we don't waste time populating the buffers.
* @param operationName name of the operation for which get object on S3 is called.
@@ -1043,15 +1024,11 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
* @return result s3 object.
* @throws IOException exception if any.
*/
- private S3Object getS3ObjectAndValidateNotNull(final String operationName,
- final long position,
- final int length) throws IOException {
+ private ResponseInputStream<GetObjectResponse> getS3ObjectInputStream(
+ final String operationName, final long position, final int length) throws IOException {
checkIfVectoredIOStopped();
- S3Object objectRange = getS3Object(operationName, position, length);
- if (objectRange.getObjectContent() == null) {
- throw new PathIOException(uri,
- "Null IO stream received during " + operationName);
- }
+ ResponseInputStream<GetObjectResponse> objectRange =
+ getS3Object(operationName, position, length);
checkIfVectoredIOStopped();
return objectRange;
}
@@ -1066,7 +1043,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
*/
private void populateBuffer(int length,
ByteBuffer buffer,
- S3ObjectInputStream objectContent) throws IOException {
+ InputStream objectContent) throws IOException {
if (buffer.isDirect()) {
VectoredReadUtils.readInDirectBuffer(length, buffer,
@@ -1091,7 +1068,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
* @param length number of bytes to fill in dest.
* @throws IOException any IOE.
*/
- private void readByteArray(S3ObjectInputStream objectContent,
+ private void readByteArray(InputStream objectContent,
byte[] dest,
int offset,
int length) throws IOException {
@@ -1118,13 +1095,16 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
* @return S3Object result s3 object.
* @throws IOException exception if any.
*/
- private S3Object getS3Object(String operationName, long position,
- int length) throws IOException {
- final GetObjectRequest request = client.newGetRequest(key)
- .withRange(position, position + length - 1);
- changeTracker.maybeApplyConstraint(request);
+ private ResponseInputStream<GetObjectResponse> getS3Object(String operationName,
+ long position,
+ int length)
+ throws IOException {
+ final GetObjectRequest request = client.newGetRequestBuilder(key)
+ .range(S3AUtils.formatRange(position, position + length - 1))
+ .applyMutation(changeTracker::maybeApplyConstraint)
+ .build();
DurationTracker tracker = streamStatistics.initiateGetRequest();
- S3Object objectRange;
+ ResponseInputStream<GetObjectResponse> objectRange;
Invoker invoker = context.getReadInvoker();
try {
objectRange = invoker.retry(operationName, pathStr, true,
@@ -1139,7 +1119,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
} finally {
tracker.close();
}
- changeTracker.processResponse(objectRange, operationName,
+ changeTracker.processResponse(objectRange.response(), operationName,
position);
return objectRange;
}
@@ -1293,11 +1273,11 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
public interface InputStreamCallbacks extends Closeable {
/**
- * Create a GET request.
+ * Create a GET request builder.
* @param key object key
- * @return the request
+ * @return the request builder
*/
- GetObjectRequest newGetRequest(String key);
+ GetObjectRequest.Builder newGetRequestBuilder(String key);
/**
* Execute the request.
@@ -1305,7 +1285,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
* @return the response
*/
@Retries.OnceRaw
- S3Object getObject(GetObjectRequest request);
+ ResponseInputStream<GetObjectResponse> getObject(GetObjectRequest request);
/**
* Submit some asynchronous work, for example, draining a stream.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java
new file mode 100644
index 000000000000..23c4d3501206
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.io.IOException;
+import java.nio.file.AccessDeniedException;
+
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.HeadBucketResponse;
+import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.store.audit.AuditEntryPoint;
+
+/**
+ * This is an unstable interface for access to S3A Internal state, S3 operations
+ * and the S3 client connector itself.
+ */
+@InterfaceStability.Unstable
+@InterfaceAudience.LimitedPrivate("testing/diagnostics")
+public interface S3AInternals {
+
+ /**
+ * Returns the S3 client used by this filesystem.
+ * Will log at debug.
+ * <p>
+ * <i>Warning</i>
+ * This bypasses core S3A operations, including auditing.
+ * It is intended for use in testing, diagnostics and for accessing
+ * operations not available through the S3A connector itself.
+ * <p>
+ * Unless audit spans are created through the S3AFileSystem, make
+ * sure that {@code fs.s3a.audit.reject.out.of.span.operations} is
+ * set to false.
+ * <p>
+ * Mocking note: this is the same S3Client as is used by the owning
+ * filesystem; changes to this client will be reflected by changes
+ * in the behavior of that filesystem.
+ * @param reason a justification for requesting access.
+ * @return S3Client
+ */
+ S3Client getAmazonS3Client(String reason);
+
+ /**
+ * Get the region of a bucket.
+ * Invoked from StoreContext; consider an entry point.
+ * @return the region in which a bucket is located
+ * @throws AccessDeniedException if the caller lacks permission.
+ * @throws IOException on any failure.
+ */
+ @Retries.RetryTranslated
+ @AuditEntryPoint
+ String getBucketLocation() throws IOException;
+
+ /**
+ * Get the region of a bucket; fixing up the region so it can be used
+ * in the builders of other AWS clients.
+ * Requires the caller to have the AWS role permission
+ * {@code s3:GetBucketLocation}.
+ * Retry policy: retrying, translated.
+ * @param bucketName the name of the bucket
+ * @return the region in which a bucket is located
+ * @throws AccessDeniedException if the caller lacks permission.
+ * @throws IOException on any failure.
+ */
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ String getBucketLocation(String bucketName) throws IOException;
+
+ /**
+ * Low-level call to get at the object metadata.
+ * Auditing: An audit entry point.
+ * @param path path to the object. This will be qualified.
+ * @return metadata
+ * @throws IOException IO and object access problems.
+ */
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ HeadObjectResponse getObjectMetadata(Path path) throws IOException;
+
+ /**
+ * Get a shared copy of the AWS credentials, with its reference
+ * counter updated.
+ * Caller is required to call {@code close()} on this after
+ * they have finished using it.
+ * @param purpose what is this for? This is for logging
+ * @return a reference to shared credentials.
+ */
+ AWSCredentialProviderList shareCredentials(String purpose);
+
+ /**
+ * Request bucket metadata.
+ * @return the metadata
+ * @throws UnknownStoreException the bucket is absent
+ * @throws IOException any other problem talking to S3
+ */
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ HeadBucketResponse getBucketMetadata() throws IOException;
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
index 528a99f5e096..fdb4591476c5 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
@@ -30,7 +30,8 @@ import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
-import com.amazonaws.AmazonClientException;
+import software.amazon.awssdk.core.exception.SdkException;
+import org.apache.hadoop.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,7 +43,7 @@ import org.apache.hadoop.fs.s3a.auth.NoAuthWithAWSException;
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.net.ConnectTimeoutException;
-import org.apache.hadoop.util.Preconditions;
+
import static org.apache.hadoop.io.retry.RetryPolicies.*;
@@ -68,9 +69,9 @@ import static org.apache.hadoop.fs.s3a.Constants.*;
*
* The retry policy is all built around that of the normal IO exceptions,
* particularly those extracted from
- * {@link S3AUtils#translateException(String, Path, AmazonClientException)}.
+ * {@link S3AUtils#translateException(String, Path, SdkException)}.
* Because the {@link #shouldRetry(Exception, int, int, boolean)} method
- * does this translation if an {@code AmazonClientException} is processed,
+ * does this translation if an {@code SdkException} is processed,
* the policy defined for the IOEs also applies to the original exceptions.
*
* Put differently: this retry policy aims to work for handlers of the
@@ -111,6 +112,11 @@ public class S3ARetryPolicy implements RetryPolicy {
*/
protected final RetryPolicy connectivityFailure;
+ /**
+ * Handling of AWSClientIOException and subclasses.
+ */
+ protected final RetryPolicy retryAwsClientExceptions;
+
/**
* Instantiate.
* @param conf configuration to read.
@@ -138,6 +144,11 @@ public class S3ARetryPolicy implements RetryPolicy {
retryIdempotentCalls = new FailNonIOEs(
new IdempotencyRetryFilter(baseExponentialRetry));
+ // retry on AWSClientIOException and possibly subclasses;
+ // See: HADOOP-18871. S3ARetryPolicy to use sdk exception retryable() if it is valid
+ // currently the normal retryIdempotentCalls policy is used.
+ retryAwsClientExceptions = retryIdempotentCalls;
+
// and a separate policy for throttle requests, which are considered
// repeatable, even for non-idempotent calls, as the service
// rejected the call entirely
@@ -223,9 +234,11 @@ public class S3ARetryPolicy implements RetryPolicy {
// server didn't respond.
policyMap.put(AWSNoResponseException.class, retryIdempotentCalls);
+ // use specific retry policy for aws client exceptions
+ policyMap.put(AWSClientIOException.class, retryAwsClientExceptions);
+ policyMap.put(AWSServiceIOException.class, retryAwsClientExceptions);
+
// other operations
- policyMap.put(AWSClientIOException.class, retryIdempotentCalls);
- policyMap.put(AWSServiceIOException.class, retryIdempotentCalls);
policyMap.put(AWSS3IOException.class, retryIdempotentCalls);
policyMap.put(SocketTimeoutException.class, retryIdempotentCalls);
@@ -242,11 +255,10 @@ public class S3ARetryPolicy implements RetryPolicy {
boolean idempotent) throws Exception {
Preconditions.checkArgument(exception != null, "Null exception");
Exception ex = exception;
- if (exception instanceof AmazonClientException) {
- // uprate the amazon client exception for the purpose of exception
+ if (exception instanceof SdkException) {
+ // update the sdk exception for the purpose of exception
// processing.
- ex = S3AUtils.translateException("", "",
- (AmazonClientException) exception);
+ ex = S3AUtils.translateException("", "", (SdkException) exception);
}
return retryPolicy.shouldRetry(ex, retries, failovers, idempotent);
}
@@ -315,4 +327,28 @@ public class S3ARetryPolicy implements RetryPolicy {
}
}
+ /**
+ * Policy where AWS SDK exceptions are retried if they state that they are retryable.
+ * See HADOOP-18871. S3ARetryPolicy to use sdk exception retryable() if it is valid.
+ */
+ private static final class RetryFromAWSClientExceptionPolicy implements RetryPolicy {
+
+ private final RetryPolicy next;
+
+ private RetryFromAWSClientExceptionPolicy(RetryPolicy next) {
+ this.next = next;
+ }
+
+ @Override
+ public RetryAction shouldRetry(Exception e,
+ int retries,
+ int failovers,
+ boolean isIdempotentOrAtMostOnce) throws Exception {
+ return
+ e instanceof AWSClientIOException ?
+ next.shouldRetry(e, retries, failovers, ((AWSClientIOException)e).retryable())
+ : RetryAction.FAIL;
+ }
+ }
+
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index dc0c211fcab5..62be975b656f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -18,20 +18,12 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.AbortedException;
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.AmazonServiceException;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.Protocol;
-import com.amazonaws.SdkBaseException;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
-import com.amazonaws.retry.RetryUtils;
-import com.amazonaws.services.s3.model.AmazonS3Exception;
-import com.amazonaws.services.s3.model.MultiObjectDeleteException;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
+import software.amazon.awssdk.core.exception.AbortedException;
+import software.amazon.awssdk.core.exception.SdkException;
+import software.amazon.awssdk.core.retry.RetryUtils;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.model.S3Object;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -44,16 +36,12 @@ import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.util.functional.RemoteIterators;
-import org.apache.hadoop.fs.s3a.audit.AuditFailureException;
-import org.apache.hadoop.fs.s3a.audit.AuditIntegration;
import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets;
-import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider;
-import org.apache.hadoop.fs.s3a.impl.NetworkBinding;
-import org.apache.hadoop.fs.s3a.impl.V2Migration;
+import org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteException;
import org.apache.hadoop.fs.s3native.S3xLoginHelper;
import org.apache.hadoop.net.ConnectTimeoutException;
import org.apache.hadoop.security.ProviderUtils;
-import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
import org.slf4j.Logger;
@@ -74,23 +62,24 @@ import java.net.SocketTimeoutException;
import java.net.URI;
import java.nio.file.AccessDeniedException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
import java.util.Date;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
+import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
import static org.apache.commons.lang3.StringUtils.isEmpty;
+import static org.apache.hadoop.fs.s3a.AWSCredentialProviderList.maybeTranslateCredentialException;
import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.audit.AuditIntegration.maybeTranslateAuditException;
import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isUnknownBucket;
-import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH;
-import static org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteSupport.translateDeleteException;
+import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.instantiationException;
+import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.isAbstract;
+import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.isNotInstanceOf;
+import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.unsupportedConstructor;
+import static org.apache.hadoop.fs.s3a.impl.InternalConstants.*;
import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteIterator;
@@ -102,13 +91,7 @@ import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteI
public final class S3AUtils {
private static final Logger LOG = LoggerFactory.getLogger(S3AUtils.class);
- static final String CONSTRUCTOR_EXCEPTION = "constructor exception";
- static final String INSTANTIATION_EXCEPTION
- = "instantiation exception";
- static final String NOT_AWS_PROVIDER =
- "does not implement AWSCredentialsProvider";
- static final String ABSTRACT_PROVIDER =
- "is abstract and therefore cannot be created";
+
static final String ENDPOINT_KEY = "Endpoint";
/** Filesystem is closed; kept here to keep the errors close. */
@@ -143,21 +126,13 @@ public final class S3AUtils {
private static final String BUCKET_PATTERN = FS_S3A_BUCKET_PREFIX + "%s.%s";
- /**
- * Error message when the AWS provider list built up contains a forbidden
- * entry.
- */
- @VisibleForTesting
- public static final String E_FORBIDDEN_AWS_PROVIDER
- = "AWS provider class cannot be used";
-
private S3AUtils() {
}
/**
* Translate an exception raised in an operation into an IOException.
* The specific type of IOException depends on the class of
- * {@link AmazonClientException} passed in, and any status codes included
+ * {@link SdkException} passed in, and any status codes included
* in the operation. That is: HTTP error codes are examined and can be
* used to build a more specific response.
*
@@ -170,14 +145,14 @@ public final class S3AUtils {
*/
public static IOException translateException(String operation,
Path path,
- AmazonClientException exception) {
+ SdkException exception) {
return translateException(operation, path.toString(), exception);
}
/**
* Translate an exception raised in an operation into an IOException.
* The specific type of IOException depends on the class of
- * {@link AmazonClientException} passed in, and any status codes included
+ * {@link SdkException} passed in, and any status codes included
* in the operation. That is: HTTP error codes are examined and can be
* used to build a more specific response.
* @param operation operation
@@ -188,12 +163,14 @@ public final class S3AUtils {
@SuppressWarnings("ThrowableInstanceNeverThrown")
public static IOException translateException(@Nullable String operation,
String path,
- SdkBaseException exception) {
+ SdkException exception) {
String message = String.format("%s%s: %s",
operation,
StringUtils.isNotEmpty(path)? (" on " + path) : "",
exception);
- if (!(exception instanceof AmazonServiceException)) {
+ if (!(exception instanceof AwsServiceException)) {
+ // exceptions raised client-side: connectivity, auth, network problems...
+
Exception innerCause = containsInterruptedException(exception);
if (innerCause != null) {
// interrupted IO, or a socket exception underneath that class
@@ -205,57 +182,60 @@ public final class S3AUtils {
}
// if the exception came from the auditor, hand off translation
// to it.
- if (exception instanceof AuditFailureException) {
- return AuditIntegration.translateAuditException(path, (AuditFailureException) exception);
+ IOException ioe = maybeTranslateAuditException(path, exception);
+ if (ioe != null) {
+ return ioe;
}
- if (exception instanceof CredentialInitializationException) {
- // the exception raised by AWSCredentialProvider list if the
- // credentials were not accepted,
- return (AccessDeniedException)new AccessDeniedException(path, null,
- exception.toString()).initCause(exception);
+ ioe = maybeTranslateCredentialException(path, exception);
+ if (ioe != null) {
+ return ioe;
+ } else {
+ // no custom handling.
+ return new AWSClientIOException(message, exception);
}
- return new AWSClientIOException(message, exception);
} else {
+ // "error response returned by an S3 or other service."
+ // These contain more details and should be translated based
+ // on the HTTP status code and other details.
IOException ioe;
- AmazonServiceException ase = (AmazonServiceException) exception;
+ AwsServiceException ase = (AwsServiceException) exception;
// this exception is non-null if the service exception is an s3 one
- AmazonS3Exception s3Exception = ase instanceof AmazonS3Exception
- ? (AmazonS3Exception) ase
+ S3Exception s3Exception = ase instanceof S3Exception
+ ? (S3Exception) ase
: null;
- int status = ase.getStatusCode();
- message = message + ":" + ase.getErrorCode();
+ int status = ase.statusCode();
+ if (ase.awsErrorDetails() != null) {
+ message = message + ":" + ase.awsErrorDetails().errorCode();
+ }
switch (status) {
- case 301:
- case 307:
+ case SC_301_MOVED_PERMANENTLY:
+ case SC_307_TEMPORARY_REDIRECT:
if (s3Exception != null) {
- if (s3Exception.getAdditionalDetails() != null &&
- s3Exception.getAdditionalDetails().containsKey(ENDPOINT_KEY)) {
- message = String.format("Received permanent redirect response to "
- + "endpoint %s. This likely indicates that the S3 endpoint "
- + "configured in %s does not match the AWS region containing "
- + "the bucket.",
- s3Exception.getAdditionalDetails().get(ENDPOINT_KEY), ENDPOINT);
- }
+ message = String.format("Received permanent redirect response to "
+ + "region %s. This likely indicates that the S3 region "
+ + "configured in %s does not match the AWS region containing " + "the bucket.",
+ s3Exception.awsErrorDetails().sdkHttpResponse().headers().get(BUCKET_REGION_HEADER),
+ AWS_REGION);
ioe = new AWSRedirectException(message, s3Exception);
} else {
ioe = new AWSRedirectException(message, ase);
}
break;
- case 400:
+ case SC_400_BAD_REQUEST:
ioe = new AWSBadRequestException(message, ase);
break;
// permissions
- case 401:
- case 403:
+ case SC_401_UNAUTHORIZED:
+ case SC_403_FORBIDDEN:
ioe = new AccessDeniedException(path, null, message);
ioe.initCause(ase);
break;
// the object isn't there
- case 404:
+ case SC_404_NOT_FOUND:
if (isUnknownBucket(ase)) {
// this is a missing bucket
ioe = new UnknownStoreException(path, message, ase);
@@ -268,20 +248,20 @@ public final class S3AUtils {
// this also surfaces sometimes and is considered to
// be ~ a not found exception.
- case 410:
+ case SC_410_GONE:
ioe = new FileNotFoundException(message);
ioe.initCause(ase);
break;
// method not allowed; seen on S3 Select.
// treated as a bad request
- case 405:
+ case SC_405_METHOD_NOT_ALLOWED:
ioe = new AWSBadRequestException(message, s3Exception);
break;
// out of range. This may happen if an object is overwritten with
// a shorter one while it is being read.
- case 416:
+ case SC_416_RANGE_NOT_SATISFIABLE:
ioe = new EOFException(message);
ioe.initCause(ase);
break;
@@ -289,26 +269,26 @@ public final class S3AUtils {
// this has surfaced as a "no response from server" message.
// so rare we haven't replicated it.
// Treating as an idempotent proxy error.
- case 443:
- case 444:
+ case SC_443_NO_RESPONSE:
+ case SC_444_NO_RESPONSE:
ioe = new AWSNoResponseException(message, ase);
break;
// throttling
- case 503:
+ case SC_503_SERVICE_UNAVAILABLE:
ioe = new AWSServiceThrottledException(message, ase);
break;
// internal error
- case 500:
+ case SC_500_INTERNAL_SERVER_ERROR:
ioe = new AWSStatus500Exception(message, ase);
break;
- case 200:
+ case SC_200_OK:
if (exception instanceof MultiObjectDeleteException) {
// failure during a bulk delete
- return translateDeleteException(message,
- (MultiObjectDeleteException) exception);
+ return ((MultiObjectDeleteException) exception)
+ .translateException(message);
}
// other 200: FALL THROUGH
@@ -334,10 +314,35 @@ public final class S3AUtils {
public static IOException extractException(String operation,
String path,
ExecutionException ee) {
+ return convertExceptionCause(operation, path, ee.getCause());
+ }
+
+ /**
+ * Extract an exception from a failed future, and convert to an IOE.
+ * @param operation operation which failed
+ * @param path path operated on (may be null)
+ * @param ce completion exception
+ * @return an IOE which can be thrown
+ */
+ public static IOException extractException(String operation,
+ String path,
+ CompletionException ce) {
+ return convertExceptionCause(operation, path, ce.getCause());
+ }
+
+ /**
+ * Convert the cause of a concurrent exception to an IOE.
+ * @param operation operation which failed
+ * @param path path operated on (may be null)
+ * @param cause cause of a concurrent exception
+ * @return an IOE which can be thrown
+ */
+ private static IOException convertExceptionCause(String operation,
+ String path,
+ Throwable cause) {
IOException ioe;
- Throwable cause = ee.getCause();
- if (cause instanceof AmazonClientException) {
- ioe = translateException(operation, path, (AmazonClientException) cause);
+ if (cause instanceof SdkException) {
+ ioe = translateException(operation, path, (SdkException) cause);
} else if (cause instanceof IOException) {
ioe = (IOException) cause;
} else {
@@ -375,7 +380,7 @@ public final class S3AUtils {
* @return an IOE which can be rethrown
*/
private static InterruptedIOException translateInterruptedException(
- SdkBaseException exception,
+ SdkException exception,
final Exception innerCause,
String message) {
InterruptedIOException ioe;
@@ -386,6 +391,7 @@ public final class S3AUtils {
if (name.endsWith(".ConnectTimeoutException")
|| name.endsWith(".ConnectionPoolTimeoutException")
|| name.endsWith("$ConnectTimeoutException")) {
+ // TODO: review in v2
// TCP connection http timeout from the shaded or unshaded filenames
// com.amazonaws.thirdparty.apache.http.conn.ConnectTimeoutException
ioe = new ConnectTimeoutException(message);
@@ -409,10 +415,10 @@ public final class S3AUtils {
*/
public static boolean isThrottleException(Exception ex) {
return ex instanceof AWSServiceThrottledException
- || (ex instanceof AmazonServiceException
- && 503 == ((AmazonServiceException)ex).getStatusCode())
- || (ex instanceof SdkBaseException
- && RetryUtils.isThrottlingException((SdkBaseException) ex));
+ || (ex instanceof AwsServiceException
+ && 503 == ((AwsServiceException)ex).statusCode())
+ || (ex instanceof SdkException
+ && RetryUtils.isThrottlingException((SdkException) ex));
}
/**
@@ -422,7 +428,8 @@ public final class S3AUtils {
* @param ex exception
* @return true if this is believed to be a sign the connection was broken.
*/
- public static boolean isMessageTranslatableToEOF(SdkBaseException ex) {
+ public static boolean isMessageTranslatableToEOF(SdkException ex) {
+ // TODO: review in v2
return ex.toString().contains(EOF_MESSAGE_IN_XML_PARSER) ||
ex.toString().contains(EOF_READ_DIFFERENT_LENGTH);
}
@@ -432,47 +439,26 @@ public final class S3AUtils {
* @param e exception
* @return string details
*/
- public static String stringify(AmazonServiceException e) {
+ public static String stringify(AwsServiceException e) {
StringBuilder builder = new StringBuilder(
- String.format("%s: %s error %d: %s; %s%s%n",
- e.getErrorType(),
- e.getServiceName(),
- e.getStatusCode(),
- e.getErrorCode(),
- e.getErrorMessage(),
- (e.isRetryable() ? " (retryable)": "")
+ String.format("%s error %d: %s; %s%s%n",
+ e.awsErrorDetails().serviceName(),
+ e.statusCode(),
+ e.awsErrorDetails().errorCode(),
+ e.awsErrorDetails().errorMessage(),
+ (e.retryable() ? " (retryable)": "")
));
- String rawResponseContent = e.getRawResponseContent();
+ String rawResponseContent = e.awsErrorDetails().rawResponse().asUtf8String();
if (rawResponseContent != null) {
builder.append(rawResponseContent);
}
return builder.toString();
}
- /**
- * Get low level details of an amazon exception for logging; multi-line.
- * @param e exception
- * @return string details
- */
- public static String stringify(AmazonS3Exception e) {
- // get the low level details of an exception,
- StringBuilder builder = new StringBuilder(
- stringify((AmazonServiceException) e));
- Map<String, String> details = e.getAdditionalDetails();
- if (details != null) {
- builder.append('\n');
- for (Map.Entry<String, String> d : details.entrySet()) {
- builder.append(d.getKey()).append('=')
- .append(d.getValue()).append('\n');
- }
- }
- return builder.toString();
- }
-
/**
* Create a files status instance from a listing.
* @param keyPath path to entry
- * @param summary summary from AWS
+ * @param s3Object s3Object entry
* @param blockSize block size to declare.
* @param owner owner of the file
* @param eTag S3 object eTag or null if unavailable
@@ -481,20 +467,20 @@ public final class S3AUtils {
* @return a status entry
*/
public static S3AFileStatus createFileStatus(Path keyPath,
- S3ObjectSummary summary,
+ S3Object s3Object,
long blockSize,
String owner,
String eTag,
String versionId,
boolean isCSEEnabled) {
- long size = summary.getSize();
+ long size = s3Object.size();
// check if cse is enabled; strip out constant padding length.
if (isCSEEnabled && size >= CSE_PADDING_LENGTH) {
size -= CSE_PADDING_LENGTH;
}
return createFileStatus(keyPath,
- objectRepresentsDirectory(summary.getKey()),
- size, summary.getLastModified(), blockSize, owner, eTag, versionId);
+ objectRepresentsDirectory(s3Object.key()),
+ size, Date.from(s3Object.lastModified()), blockSize, owner, eTag, versionId);
}
/**
@@ -556,114 +542,7 @@ public final class S3AUtils {
}
/**
- * The standard AWS provider list for AWS connections.
- */
- @SuppressWarnings("deprecation")
- public static final List<Class<?>>
- STANDARD_AWS_PROVIDERS = Collections.unmodifiableList(
- Arrays.asList(
- TemporaryAWSCredentialsProvider.class,
- SimpleAWSCredentialsProvider.class,
- EnvironmentVariableCredentialsProvider.class,
- IAMInstanceCredentialsProvider.class));
-
- /**
- * Create the AWS credentials from the providers, the URI and
- * the key {@link Constants#AWS_CREDENTIALS_PROVIDER} in the configuration.
- * @param binding Binding URI -may be null
- * @param conf filesystem configuration
- * @return a credentials provider list
- * @throws IOException Problems loading the providers (including reading
- * secrets from credential files).
- */
- public static AWSCredentialProviderList createAWSCredentialProviderSet(
- @Nullable URI binding,
- Configuration conf) throws IOException {
- // this will reject any user:secret entries in the URI
- S3xLoginHelper.rejectSecretsInURIs(binding);
- AWSCredentialProviderList credentials =
- buildAWSProviderList(binding,
- conf,
- AWS_CREDENTIALS_PROVIDER,
- STANDARD_AWS_PROVIDERS,
- new HashSet<>());
- // make sure the logging message strips out any auth details
- LOG.debug("For URI {}, using credentials {}",
- binding, credentials);
- return credentials;
- }
-
- /**
- * Load list of AWS credential provider/credential provider factory classes.
- * @param conf configuration
- * @param key key
- * @param defaultValue list of default values
- * @return the list of classes, possibly empty
- * @throws IOException on a failure to load the list.
- */
- public static List<Class<?>> loadAWSProviderClasses(Configuration conf,
- String key,
- Class<?>... defaultValue) throws IOException {
- try {
- return Arrays.asList(conf.getClasses(key, defaultValue));
- } catch (RuntimeException e) {
- Throwable c = e.getCause() != null ? e.getCause() : e;
- throw new IOException("From option " + key + ' ' + c, c);
- }
- }
-
- /**
- * Load list of AWS credential provider/credential provider factory classes;
- * support a forbidden list to prevent loops, mandate full secrets, etc.
- * @param binding Binding URI -may be null
- * @param conf configuration
- * @param key key
- * @param forbidden a possibly empty set of forbidden classes.
- * @param defaultValues list of default providers.
- * @return the list of classes, possibly empty
- * @throws IOException on a failure to load the list.
- */
- public static AWSCredentialProviderList buildAWSProviderList(
- @Nullable final URI binding,
- final Configuration conf,
- final String key,
- final List<Class<?>> defaultValues,
- final Set<Class<?>> forbidden) throws IOException {
-
- // build up the base provider
- List<Class<?>> awsClasses = loadAWSProviderClasses(conf,
- key,
- defaultValues.toArray(new Class[defaultValues.size()]));
- // and if the list is empty, switch back to the defaults.
- // this is to address the issue that configuration.getClasses()
- // doesn't return the default if the config value is just whitespace.
- if (awsClasses.isEmpty()) {
- awsClasses = defaultValues;
- }
- // iterate through, checking for blacklists and then instantiating
- // each provider
- AWSCredentialProviderList providers = new AWSCredentialProviderList();
- for (Class<?> aClass : awsClasses) {
-
- // List of V1 credential providers that will be migrated with V2 upgrade
- if (!Arrays.asList("EnvironmentVariableCredentialsProvider",
- "EC2ContainerCredentialsProviderWrapper", "InstanceProfileCredentialsProvider")
- .contains(aClass.getSimpleName()) && aClass.getName().contains(AWS_AUTH_CLASS_PREFIX)) {
- V2Migration.v1ProviderReferenced(aClass.getName());
- }
-
- if (forbidden.contains(aClass)) {
- throw new IOException(E_FORBIDDEN_AWS_PROVIDER
- + " in option " + key + ": " + aClass);
- }
- providers.add(createAWSCredentialProvider(conf,
- aClass, binding));
- }
- return providers;
- }
-
- /**
- * Create an AWS credential provider from its class by using reflection. The
+ * Creates an instance of a class using reflection. The
* class must implement one of the following means of construction, which are
* attempted in order:
*
@@ -672,92 +551,87 @@ public final class S3AUtils {
* org.apache.hadoop.conf.Configuration</li>
* <li>a public constructor accepting
* org.apache.hadoop.conf.Configuration</li>
- * <li>a public static method named getInstance that accepts no
+ * <li>a public static method named as per methodName, that accepts no
* arguments and returns an instance of
- * com.amazonaws.auth.AWSCredentialsProvider, or</li>
+ * specified type, or</li>
* <li>a public default constructor.</li>
* </ol>
*
+ * @param className name of class for which instance is to be created
* @param conf configuration
- * @param credClass credential class
* @param uri URI of the FS
- * @return the instantiated class
- * @throws IOException on any instantiation failure.
+ * @param interfaceImplemented interface that this class implements
+ * @param methodName name of factory method to be invoked
+ * @param configKey config key under which this class is specified
+ * @param <InstanceT> Instance of class
+ * @return instance of the specified class
+ * @throws IOException on any problem
*/
- private static AWSCredentialsProvider createAWSCredentialProvider(
+ @SuppressWarnings("unchecked")
+ public static <InstanceT> InstanceT getInstanceFromReflection(String className,
Configuration conf,
- Class<?> credClass,
- @Nullable URI uri) throws IOException {
- AWSCredentialsProvider credentials = null;
- String className = credClass.getName();
- if (!AWSCredentialsProvider.class.isAssignableFrom(credClass)) {
- throw new IOException("Class " + credClass + " " + NOT_AWS_PROVIDER);
- }
- if (Modifier.isAbstract(credClass.getModifiers())) {
- throw new IOException("Class " + credClass + " " + ABSTRACT_PROVIDER);
- }
- LOG.debug("Credential provider class is {}", className);
-
+ @Nullable URI uri,
+ Class<? extends InstanceT> interfaceImplemented,
+ String methodName,
+ String configKey) throws IOException {
try {
- // new X(uri, conf)
- Constructor cons = getConstructor(credClass, URI.class,
- Configuration.class);
- if (cons != null) {
- credentials = (AWSCredentialsProvider)cons.newInstance(uri, conf);
- return credentials;
+ Class<?> instanceClass = S3AUtils.class.getClassLoader().loadClass(className);
+ if (Modifier.isAbstract(instanceClass.getModifiers())) {
+ throw isAbstract(uri, className, configKey);
}
- // new X(conf)
- cons = getConstructor(credClass, Configuration.class);
- if (cons != null) {
- credentials = (AWSCredentialsProvider)cons.newInstance(conf);
- return credentials;
+ if (!interfaceImplemented.isAssignableFrom(instanceClass)) {
+ throw isNotInstanceOf(uri, className, interfaceImplemented.getName(), configKey);
+
}
+ Constructor cons;
+ if (conf != null) {
+ // new X(uri, conf)
+ cons = getConstructor(instanceClass, URI.class, Configuration.class);
- // X.getInstance()
- Method factory = getFactoryMethod(credClass, AWSCredentialsProvider.class,
- "getInstance");
+ if (cons != null) {
+ return (InstanceT) cons.newInstance(uri, conf);
+ }
+ // new X(conf)
+ cons = getConstructor(instanceClass, Configuration.class);
+ if (cons != null) {
+ return (InstanceT) cons.newInstance(conf);
+ }
+ }
+
+ // X.methodName()
+ Method factory = getFactoryMethod(instanceClass, interfaceImplemented, methodName);
if (factory != null) {
- credentials = (AWSCredentialsProvider)factory.invoke(null);
- return credentials;
+ return (InstanceT) factory.invoke(null);
}
// new X()
- cons = getConstructor(credClass);
+ cons = getConstructor(instanceClass);
if (cons != null) {
- credentials = (AWSCredentialsProvider)cons.newInstance();
- return credentials;
+ return (InstanceT) cons.newInstance();
}
// no supported constructor or factory method found
- throw new IOException(String.format("%s " + CONSTRUCTOR_EXCEPTION
- + ". A class specified in %s must provide a public constructor "
- + "of a supported signature, or a public factory method named "
- + "getInstance that accepts no arguments.",
- className, AWS_CREDENTIALS_PROVIDER));
+ throw unsupportedConstructor(uri, className, configKey);
} catch (InvocationTargetException e) {
Throwable targetException = e.getTargetException();
if (targetException == null) {
- targetException = e;
+ targetException = e;
}
if (targetException instanceof IOException) {
throw (IOException) targetException;
- } else if (targetException instanceof SdkBaseException) {
- throw translateException("Instantiate " + className, "",
- (SdkBaseException) targetException);
+ } else if (targetException instanceof SdkException) {
+ throw translateException("Instantiate " + className, "", (SdkException) targetException);
} else {
// supported constructor or factory method found, but the call failed
- throw new IOException(className + " " + INSTANTIATION_EXCEPTION
- + ": " + targetException,
- targetException);
+ throw instantiationException(uri, className, configKey, targetException);
}
} catch (ReflectiveOperationException | IllegalArgumentException e) {
// supported constructor or factory method found, but the call failed
- throw new IOException(className + " " + INSTANTIATION_EXCEPTION
- + ": " + e,
- e);
+ throw instantiationException(uri, className, configKey, e);
}
}
+
/**
* Set a key if the value is non-empty.
* @param config config to patch
@@ -941,13 +815,13 @@ public final class S3AUtils {
/**
* String information about a summary entry for debug messages.
- * @param summary summary object
+ * @param s3Object s3Object entry
* @return string value
*/
- public static String stringify(S3ObjectSummary summary) {
- StringBuilder builder = new StringBuilder(summary.getKey().length() + 100);
- builder.append(summary.getKey()).append(' ');
- builder.append("size=").append(summary.getSize());
+ public static String stringify(S3Object s3Object) {
+ StringBuilder builder = new StringBuilder(s3Object.key().length() + 100);
+ builder.append(s3Object.key()).append(' ');
+ builder.append("size=").append(s3Object.size());
return builder.toString();
}
@@ -1215,215 +1089,6 @@ public final class S3AUtils {
}
}
- /**
- * Create a new AWS {@code ClientConfiguration}.
- * All clients to AWS services <i>MUST</i> use this for consistent setup
- * of connectivity, UA, proxy settings.
- * @param conf The Hadoop configuration
- * @param bucket Optional bucket to use to look up per-bucket proxy secrets
- * @return new AWS client configuration
- * @throws IOException problem creating AWS client configuration
- *
- * @deprecated use {@link #createAwsConf(Configuration, String, String)}
- */
- @Deprecated
- public static ClientConfiguration createAwsConf(Configuration conf,
- String bucket)
- throws IOException {
- return createAwsConf(conf, bucket, null);
- }
-
- /**
- * Create a new AWS {@code ClientConfiguration}. All clients to AWS services
- * <i>MUST</i> use this or the equivalents for the specific service for
- * consistent setup of connectivity, UA, proxy settings.
- *
- * @param conf The Hadoop configuration
- * @param bucket Optional bucket to use to look up per-bucket proxy secrets
- * @param awsServiceIdentifier a string representing the AWS service (S3,
- * etc) for which the ClientConfiguration is being created.
- * @return new AWS client configuration
- * @throws IOException problem creating AWS client configuration
- */
- public static ClientConfiguration createAwsConf(Configuration conf,
- String bucket, String awsServiceIdentifier)
- throws IOException {
- final ClientConfiguration awsConf = new ClientConfiguration();
- initConnectionSettings(conf, awsConf);
- initProxySupport(conf, bucket, awsConf);
- initUserAgent(conf, awsConf);
- if (StringUtils.isNotEmpty(awsServiceIdentifier)) {
- String configKey = null;
- switch (awsServiceIdentifier) {
- case AWS_SERVICE_IDENTIFIER_S3:
- configKey = SIGNING_ALGORITHM_S3;
- break;
- case AWS_SERVICE_IDENTIFIER_STS:
- configKey = SIGNING_ALGORITHM_STS;
- break;
- default:
- // Nothing to do. The original signer override is already setup
- }
- if (configKey != null) {
- String signerOverride = conf.getTrimmed(configKey, "");
- if (!signerOverride.isEmpty()) {
- LOG.debug("Signer override for {}} = {}", awsServiceIdentifier,
- signerOverride);
- awsConf.setSignerOverride(signerOverride);
- }
- }
- }
- return awsConf;
- }
-
- /**
- * Initializes all AWS SDK settings related to connection management.
- *
- * @param conf Hadoop configuration
- * @param awsConf AWS SDK configuration
- *
- * @throws IOException if there was an error initializing the protocol
- * settings
- */
- public static void initConnectionSettings(Configuration conf,
- ClientConfiguration awsConf) throws IOException {
- awsConf.setMaxConnections(intOption(conf, MAXIMUM_CONNECTIONS,
- DEFAULT_MAXIMUM_CONNECTIONS, 1));
- awsConf.setConnectionTTL(longOption(conf, CONNECTION_TTL,
- DEFAULT_CONNECTION_TTL, -1));
- initProtocolSettings(conf, awsConf);
- awsConf.setMaxErrorRetry(intOption(conf, MAX_ERROR_RETRIES,
- DEFAULT_MAX_ERROR_RETRIES, 0));
- awsConf.setConnectionTimeout(intOption(conf, ESTABLISH_TIMEOUT,
- DEFAULT_ESTABLISH_TIMEOUT, 0));
- awsConf.setSocketTimeout(intOption(conf, SOCKET_TIMEOUT,
- DEFAULT_SOCKET_TIMEOUT, 0));
- int sockSendBuffer = intOption(conf, SOCKET_SEND_BUFFER,
- DEFAULT_SOCKET_SEND_BUFFER, 2048);
- int sockRecvBuffer = intOption(conf, SOCKET_RECV_BUFFER,
- DEFAULT_SOCKET_RECV_BUFFER, 2048);
- long requestTimeoutMillis = conf.getTimeDuration(REQUEST_TIMEOUT,
- DEFAULT_REQUEST_TIMEOUT, TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
-
- if (requestTimeoutMillis > Integer.MAX_VALUE) {
- LOG.debug("Request timeout is too high({} ms). Setting to {} ms instead",
- requestTimeoutMillis, Integer.MAX_VALUE);
- requestTimeoutMillis = Integer.MAX_VALUE;
- }
- awsConf.setRequestTimeout((int) requestTimeoutMillis);
- awsConf.setSocketBufferSizeHints(sockSendBuffer, sockRecvBuffer);
- String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM, "");
- if (!signerOverride.isEmpty()) {
- LOG.debug("Signer override = {}", signerOverride);
- awsConf.setSignerOverride(signerOverride);
- }
- }
-
- /**
- * Initializes the connection protocol settings when connecting to S3 (e.g.
- * either HTTP or HTTPS). If secure connections are enabled, this method
- * will load the configured SSL providers.
- *
- * @param conf Hadoop configuration
- * @param awsConf AWS SDK configuration
- *
- * @throws IOException if there is an error initializing the configured
- * {@link javax.net.ssl.SSLSocketFactory}
- */
- private static void initProtocolSettings(Configuration conf,
- ClientConfiguration awsConf) throws IOException {
- boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS,
- DEFAULT_SECURE_CONNECTIONS);
- awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);
- if (secureConnections) {
- NetworkBinding.bindSSLChannelMode(conf, awsConf);
- }
- }
-
- /**
- * Initializes AWS SDK proxy support in the AWS client configuration
- * if the S3A settings enable it.
- *
- * @param conf Hadoop configuration
- * @param bucket Optional bucket to use to look up per-bucket proxy secrets
- * @param awsConf AWS SDK configuration to update
- * @throws IllegalArgumentException if misconfigured
- * @throws IOException problem getting username/secret from password source.
- */
- public static void initProxySupport(Configuration conf,
- String bucket,
- ClientConfiguration awsConf) throws IllegalArgumentException,
- IOException {
- String proxyHost = conf.getTrimmed(PROXY_HOST, "");
- int proxyPort = conf.getInt(PROXY_PORT, -1);
- if (!proxyHost.isEmpty()) {
- awsConf.setProxyHost(proxyHost);
- if (proxyPort >= 0) {
- awsConf.setProxyPort(proxyPort);
- } else {
- if (conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS)) {
- LOG.warn("Proxy host set without port. Using HTTPS default 443");
- awsConf.setProxyPort(443);
- } else {
- LOG.warn("Proxy host set without port. Using HTTP default 80");
- awsConf.setProxyPort(80);
- }
- }
- final String proxyUsername = lookupPassword(bucket, conf, PROXY_USERNAME,
- null, null);
- final String proxyPassword = lookupPassword(bucket, conf, PROXY_PASSWORD,
- null, null);
- if ((proxyUsername == null) != (proxyPassword == null)) {
- String msg = "Proxy error: " + PROXY_USERNAME + " or " +
- PROXY_PASSWORD + " set without the other.";
- LOG.error(msg);
- throw new IllegalArgumentException(msg);
- }
- boolean isProxySecured = conf.getBoolean(PROXY_SECURED, false);
- awsConf.setProxyUsername(proxyUsername);
- awsConf.setProxyPassword(proxyPassword);
- awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN));
- awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION));
- awsConf.setProxyProtocol(isProxySecured ? Protocol.HTTPS : Protocol.HTTP);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Using proxy server {}://{}:{} as user {} with password {} "
- + "on domain {} as workstation {}",
- awsConf.getProxyProtocol(),
- awsConf.getProxyHost(),
- awsConf.getProxyPort(),
- String.valueOf(awsConf.getProxyUsername()),
- awsConf.getProxyPassword(), awsConf.getProxyDomain(),
- awsConf.getProxyWorkstation());
- }
- } else if (proxyPort >= 0) {
- String msg =
- "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST;
- LOG.error(msg);
- throw new IllegalArgumentException(msg);
- }
- }
-
- /**
- * Initializes the User-Agent header to send in HTTP requests to AWS
- * services. We always include the Hadoop version number. The user also
- * may set an optional custom prefix to put in front of the Hadoop version
- * number. The AWS SDK internally appends its own information, which seems
- * to include the AWS SDK version, OS and JVM version.
- *
- * @param conf Hadoop configuration
- * @param awsConf AWS SDK configuration to update
- */
- private static void initUserAgent(Configuration conf,
- ClientConfiguration awsConf) {
- String userAgent = "Hadoop " + VersionInfo.getVersion();
- String userAgentPrefix = conf.getTrimmed(USER_AGENT_PREFIX, "");
- if (!userAgentPrefix.isEmpty()) {
- userAgent = userAgentPrefix + ", " + userAgent;
- }
- LOG.debug("Using User-Agent: {}", userAgent);
- awsConf.setUserAgentPrefix(userAgent);
- }
-
/**
* Convert the data of an iterator of {@link S3AFileStatus} to
* an array.
@@ -1917,4 +1582,15 @@ public final class S3AUtils {
}
};
+ /**
+ * Format a byte range for a request header.
+ * See https://www.rfc-editor.org/rfc/rfc9110.html#section-14.1.2
+ *
+ * @param rangeStart the start byte offset
+ * @param rangeEnd the end byte offset (inclusive)
+ * @return a formatted byte range
+ */
+ public static String formatRange(long rangeStart, long rangeEnd) {
+ return String.format("bytes=%d-%d", rangeStart, rangeEnd);
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
index 9010f34dc259..d4504cd08d74 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
@@ -24,11 +24,14 @@ import java.net.URI;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.Executor;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.handlers.RequestHandler2;
-import com.amazonaws.monitoring.MonitoringListener;
-import com.amazonaws.services.s3.AmazonS3;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.transfer.s3.S3TransferManager;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -37,7 +40,7 @@ import org.apache.hadoop.fs.s3a.statistics.StatisticsFromAwsSdk;
import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_ENDPOINT;
/**
- * Factory for creation of {@link AmazonS3} client instances.
+ * Factory for creation of {@link S3Client} client instances.
* Important: HBase's HBoss module implements this interface in its
* tests.
* Take care when updating this interface to ensure that a client
@@ -49,20 +52,44 @@ import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_ENDPOINT;
*/
@InterfaceAudience.LimitedPrivate("HBoss")
@InterfaceStability.Evolving
-@Deprecated
public interface S3ClientFactory {
/**
- * Creates a new {@link AmazonS3} client.
+ * Creates a new {@link S3Client}.
+ * The client returned supports synchronous operations. For
+ * asynchronous operations, use
+ * {@link #createS3AsyncClient(URI, S3ClientCreationParameters)}.
*
* @param uri S3A file system URI
* @param parameters parameter object
* @return S3 client
- * @throws IOException IO problem
+ * @throws IOException on any IO problem
*/
- AmazonS3 createS3Client(URI uri,
+ S3Client createS3Client(URI uri,
S3ClientCreationParameters parameters) throws IOException;
+ /**
+ * Creates a new {@link S3AsyncClient}.
+ * The client returned supports asynchronous operations. For
+ * synchronous operations, use
+ * {@link #createS3Client(URI, S3ClientCreationParameters)}.
+ *
+ * @param uri S3A file system URI
+ * @param parameters parameter object
+ * @return Async S3 client
+ * @throws IOException on any IO problem
+ */
+ S3AsyncClient createS3AsyncClient(URI uri,
+ S3ClientCreationParameters parameters) throws IOException;
+
+ /**
+ * Creates a new {@link S3TransferManager}.
+ *
+ * @param s3AsyncClient the async client to be used by the TM.
+ * @return S3 transfer manager
+ */
+ S3TransferManager createS3TransferManager(S3AsyncClient s3AsyncClient);
+
/**
* Settings for the S3 Client.
* Implemented as a class to pass in so that adding
@@ -74,7 +101,7 @@ public interface S3ClientFactory {
/**
* Credentials.
*/
- private AWSCredentialsProvider credentialSet;
+ private AwsCredentialsProvider credentialSet;
/**
* Endpoint.
@@ -86,11 +113,6 @@ public interface S3ClientFactory {
*/
private final Map<String, String> headers = new HashMap<>();
- /**
- * Monitoring listener.
- */
- private MonitoringListener monitoringListener;
-
/**
* RequestMetricCollector metrics...if not-null will be wrapped
* with an {@code AwsStatisticsCollector} and passed to
@@ -109,9 +131,9 @@ public interface S3ClientFactory {
private boolean requesterPays;
/**
- * Request handlers; used for auditing, X-Ray etc.
- */
- private List<RequestHandler2> requestHandlers;
+ * Execution interceptors; used for auditing, X-Ray etc.
+ * */
+ private List<ExecutionInterceptor> executionInterceptors;
/**
* Suffix to UA.
@@ -125,37 +147,43 @@ public interface S3ClientFactory {
private URI pathUri;
/**
- * List of request handlers to include in the chain
- * of request execution in the SDK.
- * @return the handler list
+ * Minimum part size for transfer parts.
*/
- public List<RequestHandler2> getRequestHandlers() {
- return requestHandlers;
- }
+ private long minimumPartSize;
/**
- * List of request handlers.
- * @param handlers handler list.
- * @return this object
+ * Threshold for multipart operations.
*/
- public S3ClientCreationParameters withRequestHandlers(
- @Nullable final List<RequestHandler2> handlers) {
- requestHandlers = handlers;
- return this;
- }
+ private long multiPartThreshold;
- public MonitoringListener getMonitoringListener() {
- return monitoringListener;
+ /**
+ * Executor that the transfer manager will use to execute background tasks.
+ */
+ private Executor transferManagerExecutor;
+
+ /**
+ * Region of the S3 bucket.
+ */
+ private Region region;
+
+
+ /**
+ * List of execution interceptors to include in the chain
+ * of interceptors in the SDK.
+ * @return the interceptors list
+ */
+ public List<ExecutionInterceptor> getExecutionInterceptors() {
+ return executionInterceptors;
}
/**
- * listener for AWS monitoring events.
- * @param listener listener
+ * List of execution interceptors.
+ * @param interceptors interceptors list.
* @return this object
*/
- public S3ClientCreationParameters withMonitoringListener(
- @Nullable final MonitoringListener listener) {
- monitoringListener = listener;
+ public S3ClientCreationParameters withExecutionInterceptors(
+ @Nullable final List<ExecutionInterceptor> interceptors) {
+ executionInterceptors = interceptors;
return this;
}
@@ -191,7 +219,7 @@ public interface S3ClientFactory {
return requesterPays;
}
- public AWSCredentialsProvider getCredentialSet() {
+ public AwsCredentialsProvider getCredentialSet() {
return credentialSet;
}
@@ -202,7 +230,7 @@ public interface S3ClientFactory {
*/
public S3ClientCreationParameters withCredentialSet(
- final AWSCredentialsProvider value) {
+ final AwsCredentialsProvider value) {
credentialSet = value;
return this;
}
@@ -294,5 +322,82 @@ public interface S3ClientFactory {
pathUri = value;
return this;
}
+
+ /**
+ * Get the minimum part size for transfer parts.
+ * @return part size
+ */
+ public long getMinimumPartSize() {
+ return minimumPartSize;
+ }
+
+ /**
+ * Set the minimum part size for transfer parts.
+ * @param value new value
+ * @return the builder
+ */
+ public S3ClientCreationParameters withMinimumPartSize(
+ final long value) {
+ minimumPartSize = value;
+ return this;
+ }
+
+ /**
+ * Get the threshold for multipart operations.
+ * @return multipart threshold
+ */
+ public long getMultiPartThreshold() {
+ return multiPartThreshold;
+ }
+
+ /**
+ * Set the threshold for multipart operations.
+ * @param value new value
+ * @return the builder
+ */
+ public S3ClientCreationParameters withMultipartThreshold(
+ final long value) {
+ multiPartThreshold = value;
+ return this;
+ }
+
+ /**
+ * Get the executor that the transfer manager will use to execute background tasks.
+ * @return part size
+ */
+ public Executor getTransferManagerExecutor() {
+ return transferManagerExecutor;
+ }
+
+ /**
+ * Set the executor that the transfer manager will use to execute background tasks.
+ * @param value new value
+ * @return the builder
+ */
+ public S3ClientCreationParameters withTransferManagerExecutor(
+ final Executor value) {
+ transferManagerExecutor = value;
+ return this;
+ }
+
+ /**
+ * Set region.
+ *
+ * @param value new value
+ * @return the builder
+ */
+ public S3ClientCreationParameters withRegion(
+ final Region value) {
+ region = value;
+ return this;
+ }
+
+ /**
+ * Get the region.
+ * @return invoker
+ */
+ public Region getRegion() {
+ return region;
+ }
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java
index d51211516f25..c729f3de15f0 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java
@@ -18,8 +18,8 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
/**
* API version-independent container for S3 List requests.
@@ -78,14 +78,14 @@ public final class S3ListRequest {
public String toString() {
if (isV1()) {
return String.format(DESCRIPTION,
- v1Request.getBucketName(), v1Request.getPrefix(),
- v1Request.getDelimiter(), v1Request.getMaxKeys(),
- v1Request.isRequesterPays());
+ v1Request.bucket(), v1Request.prefix(),
+ v1Request.delimiter(), v1Request.maxKeys(),
+ v1Request.requestPayerAsString());
} else {
return String.format(DESCRIPTION,
- v2Request.getBucketName(), v2Request.getPrefix(),
- v2Request.getDelimiter(), v2Request.getMaxKeys(),
- v2Request.isRequesterPays());
+ v2Request.bucket(), v2Request.prefix(),
+ v2Request.delimiter(), v2Request.maxKeys(),
+ v2Request.requestPayerAsString());
}
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java
index 69c42bfe1471..c77311211abc 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java
@@ -22,19 +22,21 @@ import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;
-import com.amazonaws.services.s3.model.ListObjectsV2Result;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
+import software.amazon.awssdk.services.s3.model.CommonPrefix;
+import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
import org.slf4j.Logger;
/**
* API version-independent container for S3 List responses.
*/
public class S3ListResult {
- private ObjectListing v1Result;
- private ListObjectsV2Result v2Result;
+ private ListObjectsResponse v1Result;
+ private ListObjectsV2Response v2Result;
- protected S3ListResult(ObjectListing v1, ListObjectsV2Result v2) {
+ protected S3ListResult(ListObjectsResponse v1, ListObjectsV2Response v2) {
v1Result = v1;
v2Result = v2;
}
@@ -44,7 +46,7 @@ public class S3ListResult {
* @param result v1 result
* @return new list result container
*/
- public static S3ListResult v1(ObjectListing result) {
+ public static S3ListResult v1(ListObjectsResponse result) {
return new S3ListResult(result, null);
}
@@ -53,7 +55,7 @@ public class S3ListResult {
* @param result v2 result
* @return new list result container
*/
- public static S3ListResult v2(ListObjectsV2Result result) {
+ public static S3ListResult v2(ListObjectsV2Response result) {
return new S3ListResult(null, result);
}
@@ -65,19 +67,19 @@ public class S3ListResult {
return v1Result != null;
}
- public ObjectListing getV1() {
+ public ListObjectsResponse getV1() {
return v1Result;
}
- public ListObjectsV2Result getV2() {
+ public ListObjectsV2Response getV2() {
return v2Result;
}
- public List<S3ObjectSummary> getObjectSummaries() {
+ public List<S3Object> getS3Objects() {
if (isV1()) {
- return v1Result.getObjectSummaries();
+ return v1Result.contents();
} else {
- return v2Result.getObjectSummaries();
+ return v2Result.contents();
}
}
@@ -89,21 +91,21 @@ public class S3ListResult {
}
}
- public List<String> getCommonPrefixes() {
+ public List<CommonPrefix> getCommonPrefixes() {
if (isV1()) {
- return v1Result.getCommonPrefixes();
+ return v1Result.commonPrefixes();
} else {
- return v2Result.getCommonPrefixes();
+ return v2Result.commonPrefixes();
}
}
/**
- * Get the list of keys in the object summary.
+ * Get the list of keys in the list result.
* @return a possibly empty list
*/
- private List<String> objectSummaryKeys() {
- return getObjectSummaries().stream()
- .map(S3ObjectSummary::getKey)
+ private List<String> objectKeys() {
+ return getS3Objects().stream()
+ .map(S3Object::key)
.collect(Collectors.toList());
}
@@ -112,9 +114,8 @@ public class S3ListResult {
* @return true if the result is non-empty
*/
public boolean hasPrefixesOrObjects() {
-
return !(getCommonPrefixes()).isEmpty()
- || !getObjectSummaries().isEmpty();
+ || !getS3Objects().isEmpty();
}
/**
@@ -128,7 +129,7 @@ public class S3ListResult {
// no children.
// So the listing must contain the marker entry only as an object,
// and prefixes is null
- List<String> keys = objectSummaryKeys();
+ List<String> keys = objectKeys();
return keys.size() == 1 && keys.contains(dirKey)
&& getCommonPrefixes().isEmpty();
}
@@ -138,15 +139,15 @@ public class S3ListResult {
* @param log log to use
*/
public void logAtDebug(Logger log) {
- Collection<String> prefixes = getCommonPrefixes();
- Collection<S3ObjectSummary> summaries = getObjectSummaries();
+ Collection<CommonPrefix> prefixes = getCommonPrefixes();
+ Collection<S3Object> s3Objects = getS3Objects();
log.debug("Prefix count = {}; object count={}",
- prefixes.size(), summaries.size());
- for (S3ObjectSummary summary : summaries) {
- log.debug("Summary: {} {}", summary.getKey(), summary.getSize());
+ prefixes.size(), s3Objects.size());
+ for (S3Object s3Object : s3Objects) {
+ log.debug("Summary: {} {}", s3Object.key(), s3Object.size());
}
- for (String prefix : prefixes) {
- log.debug("Prefix: {}", prefix);
+ for (CommonPrefix prefix : prefixes) {
+ log.debug("Prefix: {}", prefix.prefix());
}
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java
index 0a0454854b2a..4fc5b8658b60 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.services.s3.transfer.model.CopyResult;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.Path;
@@ -74,31 +72,6 @@ public class S3ObjectAttributes {
this.len = len;
}
- /**
- * Construct from the result of a copy and those parameters
- * which aren't included in an AWS SDK response.
- * @param path path
- * @param copyResult copy result.
- * @param serverSideEncryptionAlgorithm current encryption algorithm
- * @param serverSideEncryptionKey any server side encryption key?
- * @param len object length
- */
- public S3ObjectAttributes(
- final Path path,
- final CopyResult copyResult,
- final S3AEncryptionMethods serverSideEncryptionAlgorithm,
- final String serverSideEncryptionKey,
- final long len) {
- this.bucket = copyResult.getDestinationBucketName();
- this.key = copyResult.getDestinationKey();
- this.path = path;
- this.serverSideEncryptionAlgorithm = serverSideEncryptionAlgorithm;
- this.serverSideEncryptionKey = serverSideEncryptionKey;
- this.eTag = copyResult.getETag();
- this.versionId = copyResult.getVersionId();
- this.len = len;
- }
-
public String getBucket() {
return bucket;
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java
index 6579a2bc3e7d..b7de937f5137 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java
@@ -39,6 +39,5 @@ import org.apache.hadoop.fs.s3a.auth.NoAwsCredentialsException;
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
-@SuppressWarnings("deprecation")
public final class SharedInstanceCredentialProvider extends IAMInstanceCredentialsProvider {
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
index 657bd437b5be..13c951e83b9c 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
@@ -18,9 +18,11 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
+
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang3.StringUtils;
@@ -42,13 +44,10 @@ import static org.apache.hadoop.fs.s3a.S3AUtils.getAWSAccessKeys;
* property fs.s3a.aws.credentials.provider. Therefore, changing the class name
* would be a backward-incompatible change.
*
- * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
- * as part of upgrading S3A to SDK V2. See HADOOP-18073.
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
-@Deprecated
-public class SimpleAWSCredentialsProvider implements AWSCredentialsProvider {
+public class SimpleAWSCredentialsProvider implements AwsCredentialsProvider {
public static final String NAME
= "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider";
@@ -63,7 +62,7 @@ public class SimpleAWSCredentialsProvider implements AWSCredentialsProvider {
*/
public SimpleAWSCredentialsProvider(final URI uri, final Configuration conf)
throws IOException {
- this(getAWSAccessKeys(uri, conf));
+ this(getAWSAccessKeys(uri, conf));
}
/**
@@ -75,25 +74,25 @@ public class SimpleAWSCredentialsProvider implements AWSCredentialsProvider {
@VisibleForTesting
SimpleAWSCredentialsProvider(final S3xLoginHelper.Login login)
throws IOException {
- this.accessKey = login.getUser();
- this.secretKey = login.getPassword();
+ this.accessKey = login.getUser();
+ this.secretKey = login.getPassword();
}
@Override
- public AWSCredentials getCredentials() {
+ public AwsCredentials resolveCredentials() {
if (!StringUtils.isEmpty(accessKey) && !StringUtils.isEmpty(secretKey)) {
- return new BasicAWSCredentials(accessKey, secretKey);
+ return AwsBasicCredentials.create(accessKey, secretKey);
}
throw new NoAwsCredentialsException("SimpleAWSCredentialsProvider",
"No AWS credentials in the Hadoop configuration");
}
- @Override
- public void refresh() {}
-
@Override
public String toString() {
- return getClass().getSimpleName();
+ return "SimpleAWSCredentialsProvider{" +
+ "accessKey.empty=" + accessKey.isEmpty() +
+ ", secretKey.empty'" + secretKey.isEmpty() +
+ '}';
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
index f9694871d730..b5f7fcbcd016 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
@@ -548,6 +548,11 @@ public enum Statistic {
StoreStatisticNames.STORE_IO_THROTTLE_RATE,
"Rate of S3 request throttling",
TYPE_QUANTILE),
+ STORE_REGION_PROBE(
+ StoreStatisticNames.STORE_REGION_PROBE,
+ "Store Region Probe",
+ TYPE_DURATION
+ ),
/*
* Delegation Token Operations.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
index db3d0bb13297..5587b11a36f8 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.fs.s3a;
import javax.annotation.Nullable;
import java.io.IOException;
-import com.amazonaws.auth.AWSCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
import java.net.URI;
@@ -44,12 +44,9 @@ import org.apache.hadoop.fs.s3a.auth.NoAwsCredentialsException;
* This credential provider must not fail in creation because that will
* break a chain of credential providers.
*
- * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
- * as part of upgrading S3A to SDK V2. See HADOOP-18073.
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
-@Deprecated
public class TemporaryAWSCredentialsProvider extends AbstractSessionCredentialsProvider {
public static final String NAME
@@ -92,7 +89,7 @@ public class TemporaryAWSCredentialsProvider extends AbstractSessionCredentialsP
* @throws NoAwsCredentialsException the credentials are actually empty.
*/
@Override
- protected AWSCredentials createCredentials(Configuration config)
+ protected AwsCredentials createCredentials(Configuration config)
throws IOException {
MarshalledCredentials creds = MarshalledCredentialBinding.fromFileSystem(
getUri(), config);
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Tristate.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Tristate.java
index 0462ccfd7cbb..d7123787735a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Tristate.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Tristate.java
@@ -18,15 +18,58 @@
package org.apache.hadoop.fs.s3a;
+import java.util.Optional;
+
+import static java.util.Optional.empty;
+import static java.util.Optional.of;
+
/**
* Simple enum to express {true, false, don't know}.
*/
public enum Tristate {
+
// Do not add additional values here. Logic will assume there are exactly
// three possibilities.
- TRUE, FALSE, UNKNOWN;
+ TRUE(of(Boolean.TRUE)),
+ FALSE(of(Boolean.FALSE)),
+ UNKNOWN(empty());
+
+ /**
+ * Mapping to an optional boolean.
+ */
+ @SuppressWarnings("NonSerializableFieldInSerializableClass")
+ private final Optional<Boolean> mapping;
+
+ Tristate(final Optional<Boolean> t) {
+ mapping = t;
+ }
+
+ /**
+ * Get the boolean mapping, if present.
+ * @return the boolean value, if present.
+ */
+ public Optional<Boolean> getMapping() {
+ return mapping;
+ }
+
+ /**
+ * Does this value map to a boolean.
+ * @return true if the state is one of true or false.
+ */
+ public boolean isBoolean() {
+ return mapping.isPresent();
+ }
public static Tristate fromBool(boolean v) {
return v ? TRUE : FALSE;
}
+
+ /**
+ * Build a tristate from a boolean.
+ * @param b source optional
+ * @return a tristate derived from the argument.
+ */
+ public static Tristate from(Optional<Boolean> b) {
+ return b.map(Tristate::fromBool).orElse(UNKNOWN);
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/UploadInfo.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/UploadInfo.java
index 238cd97ed8f4..85ed69e14bc7 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/UploadInfo.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/UploadInfo.java
@@ -18,22 +18,22 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.services.s3.transfer.Upload;
+import software.amazon.awssdk.transfer.s3.model.FileUpload;
/**
* Simple struct that contains information about a S3 upload.
*/
public class UploadInfo {
- private final Upload upload;
+ private final FileUpload fileUpload;
private final long length;
- public UploadInfo(Upload upload, long length) {
- this.upload = upload;
+ public UploadInfo(FileUpload upload, long length) {
+ this.fileUpload = upload;
this.length = length;
}
- public Upload getUpload() {
- return upload;
+ public FileUpload getFileUpload() {
+ return fileUpload;
}
public long getLength() {
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java
index 0bbb8a35f51c..1eacc4cbe07b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java
@@ -19,29 +19,28 @@
package org.apache.hadoop.fs.s3a;
import javax.annotation.Nullable;
-import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.InputStream;
import java.util.List;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.AtomicInteger;
-import com.amazonaws.services.s3.model.AmazonS3Exception;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
-import com.amazonaws.services.s3.model.MultipartUpload;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PartETag;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.PutObjectResult;
-import com.amazonaws.services.s3.model.SelectObjectContentRequest;
-import com.amazonaws.services.s3.model.SelectObjectContentResult;
-import com.amazonaws.services.s3.model.UploadPartRequest;
-import com.amazonaws.services.s3.model.UploadPartResult;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse;
+import software.amazon.awssdk.services.s3.model.CompletedPart;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.MultipartUpload;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
+import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest;
+import software.amazon.awssdk.services.s3.model.SelectObjectContentResponseHandler;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -50,6 +49,8 @@ import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.s3a.api.RequestFactory;
import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
import org.apache.hadoop.fs.s3a.impl.StoreContext;
+import org.apache.hadoop.fs.s3a.select.SelectEventStreamPublisher;
+import org.apache.hadoop.fs.s3a.select.SelectObjectContentHelper;
import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext;
import org.apache.hadoop.fs.s3a.select.SelectBinding;
import org.apache.hadoop.fs.statistics.DurationTrackerFactory;
@@ -239,49 +240,25 @@ public class WriteOperationHelper implements WriteOperations {
/**
* Create a {@link PutObjectRequest} request against the specific key.
* @param destKey destination key
- * @param inputStream source data.
* @param length size, if known. Use -1 for not known
* @param options options for the request
+ * @param isFile is data to be uploaded a file
* @return the request
*/
@Retries.OnceRaw
public PutObjectRequest createPutObjectRequest(String destKey,
- InputStream inputStream,
long length,
- final PutObjectOptions options) {
- activateAuditSpan();
- ObjectMetadata objectMetadata = newObjectMetadata(length);
- return getRequestFactory().newPutObjectRequest(
- destKey,
- objectMetadata,
- options,
- inputStream);
- }
+ final PutObjectOptions options,
+ boolean isFile) {
- /**
- * Create a {@link PutObjectRequest} request to upload a file.
- * @param dest key to PUT to.
- * @param sourceFile source file
- * @param options options for the request
- * @return the request
- */
- @Retries.OnceRaw
- public PutObjectRequest createPutObjectRequest(
- String dest,
- File sourceFile,
- final PutObjectOptions options) {
activateAuditSpan();
- final ObjectMetadata objectMetadata =
- newObjectMetadata((int) sourceFile.length());
-
- PutObjectRequest putObjectRequest = getRequestFactory().
- newPutObjectRequest(dest,
- objectMetadata,
- options,
- sourceFile);
- return putObjectRequest;
+
+ return getRequestFactory()
+ .newPutObjectRequestBuilder(destKey, options, length, false)
+ .build();
}
+
/**
* Callback on a successful write.
* @param length length of the write
@@ -297,17 +274,6 @@ public class WriteOperationHelper implements WriteOperations {
LOG.debug("Write to {} failed", this, ex);
}
- /**
- * Create a new object metadata instance.
- * Any standard metadata headers are added here, for example:
- * encryption.
- * @param length size, if known. Use -1 for not known
- * @return a new metadata instance
- */
- public ObjectMetadata newObjectMetadata(long length) {
- return getRequestFactory().newObjectMetadata(length);
- }
-
/**
* {@inheritDoc}
*/
@@ -320,11 +286,11 @@ public class WriteOperationHelper implements WriteOperations {
try (AuditSpan span = activateAuditSpan()) {
return retry("initiate MultiPartUpload", destKey, true,
() -> {
- final InitiateMultipartUploadRequest initiateMPURequest =
- getRequestFactory().newMultipartUploadRequest(
+ final CreateMultipartUploadRequest.Builder initiateMPURequestBuilder =
+ getRequestFactory().newMultipartUploadRequestBuilder(
destKey, options);
- return owner.initiateMultipartUpload(initiateMPURequest)
- .getUploadId();
+ return owner.initiateMultipartUpload(initiateMPURequestBuilder.build())
+ .uploadId();
});
}
}
@@ -345,10 +311,10 @@ public class WriteOperationHelper implements WriteOperations {
* @throws IOException on problems.
*/
@Retries.RetryTranslated
- private CompleteMultipartUploadResult finalizeMultipartUpload(
+ private CompleteMultipartUploadResponse finalizeMultipartUpload(
String destKey,
String uploadId,
- List<PartETag> partETags,
+ List<CompletedPart> partETags,
long length,
PutObjectOptions putOptions,
Retried retrying) throws IOException {
@@ -357,18 +323,18 @@ public class WriteOperationHelper implements WriteOperations {
"No upload parts in multipart upload");
}
try (AuditSpan span = activateAuditSpan()) {
- CompleteMultipartUploadResult uploadResult;
+ CompleteMultipartUploadResponse uploadResult;
uploadResult = invoker.retry("Completing multipart upload", destKey,
true,
retrying,
() -> {
- final CompleteMultipartUploadRequest request =
- getRequestFactory().newCompleteMultipartUploadRequest(
+ final CompleteMultipartUploadRequest.Builder requestBuilder =
+ getRequestFactory().newCompleteMultipartUploadRequestBuilder(
destKey, uploadId, partETags);
- return writeOperationHelperCallbacks.completeMultipartUpload(request);
+ return writeOperationHelperCallbacks.completeMultipartUpload(requestBuilder.build());
});
- owner.finishedWrite(destKey, length, uploadResult.getETag(),
- uploadResult.getVersionId(),
+ owner.finishedWrite(destKey, length, uploadResult.eTag(),
+ uploadResult.versionId(),
putOptions);
return uploadResult;
}
@@ -391,10 +357,10 @@ public class WriteOperationHelper implements WriteOperations {
* the retry count was exceeded
*/
@Retries.RetryTranslated
- public CompleteMultipartUploadResult completeMPUwithRetries(
+ public CompleteMultipartUploadResponse completeMPUwithRetries(
String destKey,
String uploadId,
- List<PartETag> partETags,
+ List<CompletedPart> partETags,
long length,
AtomicInteger errorCount,
PutObjectOptions putOptions)
@@ -452,7 +418,7 @@ public class WriteOperationHelper implements WriteOperations {
@Retries.RetryTranslated
public void abortMultipartUpload(MultipartUpload upload)
throws IOException {
- invoker.retry("Aborting multipart commit", upload.getKey(), true,
+ invoker.retry("Aborting multipart commit", upload.key(), true,
withinAuditSpan(getAuditSpan(),
() -> owner.abortMultipartUpload(upload)));
}
@@ -477,7 +443,7 @@ public class WriteOperationHelper implements WriteOperations {
abortMultipartUpload(upload);
count++;
} catch (FileNotFoundException e) {
- LOG.debug("Already aborted: {}", upload.getKey(), e);
+ LOG.debug("Already aborted: {}", upload.key(), e);
}
}
return count;
@@ -506,45 +472,31 @@ public class WriteOperationHelper implements WriteOperations {
}
/**
- * Create and initialize a part request of a multipart upload.
- * Exactly one of: {@code uploadStream} or {@code sourceFile}
- * must be specified.
- * A subset of the file may be posted, by providing the starting point
- * in {@code offset} and a length of block in {@code size} equal to
- * or less than the remaining bytes.
+ * Create and initialize a part request builder of a multipart upload.
* The part number must be less than 10000.
* Retry policy is once-translated; to much effort
* @param destKey destination key of ongoing operation
* @param uploadId ID of ongoing upload
* @param partNumber current part number of the upload
* @param size amount of data
- * @param uploadStream source of data to upload
- * @param sourceFile optional source file.
- * @param offset offset in file to start reading.
- * @return the request.
+ * @return the request builder.
* @throws IllegalArgumentException if the parameters are invalid.
* @throws PathIOException if the part number is out of range.
*/
@Override
@Retries.OnceTranslated
- public UploadPartRequest newUploadPartRequest(
+ public UploadPartRequest.Builder newUploadPartRequestBuilder(
String destKey,
String uploadId,
int partNumber,
- long size,
- InputStream uploadStream,
- File sourceFile,
- Long offset) throws IOException {
+ long size) throws IOException {
return once("upload part request", destKey,
withinAuditSpan(getAuditSpan(), () ->
- getRequestFactory().newUploadPartRequest(
+ getRequestFactory().newUploadPartRequestBuilder(
destKey,
uploadId,
partNumber,
- size,
- uploadStream,
- sourceFile,
- offset)));
+ size)));
}
/**
@@ -566,18 +518,20 @@ public class WriteOperationHelper implements WriteOperations {
* @param putObjectRequest the request
* @param putOptions put object options
* @param durationTrackerFactory factory for duration tracking
+ * @param uploadData data to be uploaded
+ * @param isFile is data to be uploaded a file
+ *
* @return the upload initiated
* @throws IOException on problems
*/
@Retries.RetryTranslated
- public PutObjectResult putObject(PutObjectRequest putObjectRequest,
- PutObjectOptions putOptions,
+ public PutObjectResponse putObject(PutObjectRequest putObjectRequest,
+ PutObjectOptions putOptions, S3ADataBlocks.BlockUploadData uploadData, boolean isFile,
DurationTrackerFactory durationTrackerFactory)
throws IOException {
- return retry("Writing Object",
- putObjectRequest.getKey(), true,
- withinAuditSpan(getAuditSpan(), () ->
- owner.putObjectDirect(putObjectRequest, putOptions, durationTrackerFactory)));
+ return retry("Writing Object", putObjectRequest.key(), true, withinAuditSpan(getAuditSpan(),
+ () -> owner.putObjectDirect(putObjectRequest, putOptions, uploadData, isFile,
+ durationTrackerFactory)));
}
/**
@@ -613,10 +567,10 @@ public class WriteOperationHelper implements WriteOperations {
* the retry count was exceeded
*/
@Retries.RetryTranslated
- public CompleteMultipartUploadResult commitUpload(
+ public CompleteMultipartUploadResponse commitUpload(
String destKey,
String uploadId,
- List<PartETag> partETags,
+ List<CompletedPart> partETags,
long length)
throws IOException {
checkNotNull(uploadId);
@@ -635,19 +589,21 @@ public class WriteOperationHelper implements WriteOperations {
* Upload part of a multi-partition file.
* @param request request
* @param durationTrackerFactory duration tracker factory for operation
+ * @param request the upload part request.
+ * @param body the request body.
* @return the result of the operation.
* @throws IOException on problems
*/
@Retries.RetryTranslated
- public UploadPartResult uploadPart(UploadPartRequest request,
+ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody body,
final DurationTrackerFactory durationTrackerFactory)
throws IOException {
- return retry("upload part #" + request.getPartNumber()
- + " upload ID " + request.getUploadId(),
- request.getKey(),
+ return retry("upload part #" + request.partNumber()
+ + " upload ID " + request.uploadId(),
+ request.key(),
true,
withinAuditSpan(getAuditSpan(),
- () -> owner.uploadPart(request, durationTrackerFactory)));
+ () -> owner.uploadPart(request, body, durationTrackerFactory)));
}
/**
@@ -659,15 +615,9 @@ public class WriteOperationHelper implements WriteOperations {
return conf;
}
- /**
- * Create a S3 Select request for the destination path.
- * This does not build the query.
- * @param path pre-qualified path for query
- * @return the request
- */
- public SelectObjectContentRequest newSelectRequest(Path path) {
+ public SelectObjectContentRequest.Builder newSelectRequestBuilder(Path path) {
try (AuditSpan span = getAuditSpan()) {
- return getRequestFactory().newSelectRequest(
+ return getRequestFactory().newSelectRequestBuilder(
storeContext.pathToKey(path));
}
}
@@ -676,26 +626,27 @@ public class WriteOperationHelper implements WriteOperations {
* Execute an S3 Select operation.
* On a failure, the request is only logged at debug to avoid the
* select exception being printed.
- * @param source source for selection
+ *
+ * @param source source for selection
* @param request Select request to issue.
- * @param action the action for use in exception creation
+ * @param action the action for use in exception creation
* @return response
* @throws IOException failure
*/
@Retries.RetryTranslated
- public SelectObjectContentResult select(
+ public SelectEventStreamPublisher select(
final Path source,
final SelectObjectContentRequest request,
final String action)
throws IOException {
// no setting of span here as the select binding is (statically) created
// without any span.
- String bucketName = request.getBucketName();
+ String bucketName = request.bucket();
Preconditions.checkArgument(bucket.equals(bucketName),
"wrong bucket: %s", bucketName);
if (LOG.isDebugEnabled()) {
LOG.debug("Initiating select call {} {}",
- source, request.getExpression());
+ source, request.expression());
LOG.debug(SelectBinding.toString(request));
}
return invoker.retry(
@@ -706,8 +657,9 @@ public class WriteOperationHelper implements WriteOperations {
try (DurationInfo ignored =
new DurationInfo(LOG, "S3 Select operation")) {
try {
- return writeOperationHelperCallbacks.selectObjectContent(request);
- } catch (AmazonS3Exception e) {
+ return SelectObjectContentHelper.select(
+ writeOperationHelperCallbacks, source, request, action);
+ } catch (Throwable e) {
LOG.error("Failure of S3 Select request against {}",
source);
LOG.debug("S3 Select request against {}:\n{}",
@@ -756,16 +708,18 @@ public class WriteOperationHelper implements WriteOperations {
/**
* Initiates a select request.
* @param request selectObjectContent request
+ * @param t selectObjectContent request handler
* @return selectObjectContentResult
*/
- SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request);
+ CompletableFuture<Void> selectObjectContent(SelectObjectContentRequest request,
+ SelectObjectContentResponseHandler t);
/**
* Initiates a complete multi-part upload request.
* @param request Complete multi-part upload request
* @return completeMultipartUploadResult
*/
- CompleteMultipartUploadResult completeMultipartUpload(CompleteMultipartUploadRequest request);
+ CompleteMultipartUploadResponse completeMultipartUpload(CompleteMultipartUploadRequest request);
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java
index 1c3d36885758..0fda4921a30d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java
@@ -20,29 +20,27 @@ package org.apache.hadoop.fs.s3a;
import javax.annotation.Nullable;
import java.io.Closeable;
-import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.InputStream;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
-import com.amazonaws.services.s3.model.MultipartUpload;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PartETag;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.PutObjectResult;
-import com.amazonaws.services.s3.model.SelectObjectContentRequest;
-import com.amazonaws.services.s3.model.SelectObjectContentResult;
-import com.amazonaws.services.s3.model.UploadPartRequest;
-import com.amazonaws.services.s3.model.UploadPartResult;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse;
+import software.amazon.awssdk.services.s3.model.CompletedPart;
+import software.amazon.awssdk.services.s3.model.MultipartUpload;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
+import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartResponse;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
import org.apache.hadoop.fs.statistics.DurationTrackerFactory;
+import org.apache.hadoop.fs.s3a.select.SelectEventStreamPublisher;
import org.apache.hadoop.fs.store.audit.AuditSpanSource;
import org.apache.hadoop.util.functional.CallableRaisingIOE;
@@ -77,27 +75,15 @@ public interface WriteOperations extends AuditSpanSource, Closeable {
/**
* Create a {@link PutObjectRequest} request against the specific key.
* @param destKey destination key
- * @param inputStream source data.
* @param length size, if known. Use -1 for not known
* @param options options for the request
+ * @param isFile is data to be uploaded a file
* @return the request
*/
PutObjectRequest createPutObjectRequest(String destKey,
- InputStream inputStream,
long length,
- @Nullable PutObjectOptions options);
-
- /**
- * Create a {@link PutObjectRequest} request to upload a file.
- * @param dest key to PUT to.
- * @param sourceFile source file
- * @param options options for the request
- * @return the request
- */
- PutObjectRequest createPutObjectRequest(
- String dest,
- File sourceFile,
- @Nullable PutObjectOptions options);
+ @Nullable PutObjectOptions options,
+ boolean isFile);
/**
* Callback on a successful write.
@@ -111,15 +97,6 @@ public interface WriteOperations extends AuditSpanSource, Closeable {
*/
void writeFailed(Exception ex);
- /**
- * Create a new object metadata instance.
- * Any standard metadata headers are added here, for example:
- * encryption.
- * @param length size, if known. Use -1 for not known
- * @return a new metadata instance
- */
- ObjectMetadata newObjectMetadata(long length);
-
/**
* Start the multipart upload process.
* Retry policy: retrying, translated.
@@ -148,10 +125,10 @@ public interface WriteOperations extends AuditSpanSource, Closeable {
* the retry count was exceeded
*/
@Retries.RetryTranslated
- CompleteMultipartUploadResult completeMPUwithRetries(
+ CompleteMultipartUploadResponse completeMPUwithRetries(
String destKey,
String uploadId,
- List<PartETag> partETags,
+ List<CompletedPart> partETags,
long length,
AtomicInteger errorCount,
PutObjectOptions putOptions)
@@ -213,31 +190,20 @@ public interface WriteOperations extends AuditSpanSource, Closeable {
throws IOException;
/**
- * Create and initialize a part request of a multipart upload.
- * Exactly one of: {@code uploadStream} or {@code sourceFile}
- * must be specified.
- * A subset of the file may be posted, by providing the starting point
- * in {@code offset} and a length of block in {@code size} equal to
- * or less than the remaining bytes.
+ * Create and initialize a part request builder of a multipart upload.
* @param destKey destination key of ongoing operation
* @param uploadId ID of ongoing upload
* @param partNumber current part number of the upload
* @param size amount of data
- * @param uploadStream source of data to upload
- * @param sourceFile optional source file.
- * @param offset offset in file to start reading.
- * @return the request.
+ * @return the request builder.
* @throws IllegalArgumentException if the parameters are invalid
* @throws PathIOException if the part number is out of range.
*/
- UploadPartRequest newUploadPartRequest(
+ UploadPartRequest.Builder newUploadPartRequestBuilder(
String destKey,
String uploadId,
int partNumber,
- long size,
- InputStream uploadStream,
- File sourceFile,
- Long offset) throws IOException;
+ long size) throws IOException;
/**
* PUT an object directly (i.e. not via the transfer manager).
@@ -246,12 +212,14 @@ public interface WriteOperations extends AuditSpanSource, Closeable {
* @param putObjectRequest the request
* @param putOptions put object options
* @param durationTrackerFactory factory for duration tracking
+ * @param uploadData data to be uploaded
+ * @param isFile is data to be uploaded a file
* @return the upload initiated
* @throws IOException on problems
*/
@Retries.RetryTranslated
- PutObjectResult putObject(PutObjectRequest putObjectRequest,
- PutObjectOptions putOptions,
+ PutObjectResponse putObject(PutObjectRequest putObjectRequest,
+ PutObjectOptions putOptions, S3ADataBlocks.BlockUploadData uploadData, boolean isFile,
DurationTrackerFactory durationTrackerFactory)
throws IOException;
@@ -279,22 +247,23 @@ public interface WriteOperations extends AuditSpanSource, Closeable {
* the retry count was exceeded
*/
@Retries.RetryTranslated
- CompleteMultipartUploadResult commitUpload(
+ CompleteMultipartUploadResponse commitUpload(
String destKey,
String uploadId,
- List<PartETag> partETags,
+ List<CompletedPart> partETags,
long length)
throws IOException;
/**
* Upload part of a multi-partition file.
- * @param request request
+ * @param request the upload part request.
+ * @param body the request body.
* @param durationTrackerFactory factory for duration tracking
* @return the result of the operation.
* @throws IOException on problems
*/
@Retries.RetryTranslated
- UploadPartResult uploadPart(UploadPartRequest request,
+ UploadPartResponse uploadPart(UploadPartRequest request, RequestBody body,
DurationTrackerFactory durationTrackerFactory)
throws IOException;
@@ -306,25 +275,26 @@ public interface WriteOperations extends AuditSpanSource, Closeable {
Configuration getConf();
/**
- * Create a S3 Select request for the destination path.
+ * Create a S3 Select request builder for the destination path.
* This does not build the query.
* @param path pre-qualified path for query
- * @return the request
+ * @return the request builder
*/
- SelectObjectContentRequest newSelectRequest(Path path);
+ SelectObjectContentRequest.Builder newSelectRequestBuilder(Path path);
/**
* Execute an S3 Select operation.
* On a failure, the request is only logged at debug to avoid the
* select exception being printed.
- * @param source source for selection
+ *
+ * @param source source for selection
* @param request Select request to issue.
- * @param action the action for use in exception creation
+ * @param action the action for use in exception creation
* @return response
* @throws IOException failure
*/
@Retries.RetryTranslated
- SelectObjectContentResult select(
+ SelectEventStreamPublisher select(
Path source,
SelectObjectContentRequest request,
String action)
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/AwsV1BindingSupport.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/AwsV1BindingSupport.java
new file mode 100644
index 000000000000..177952cb9d22
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/AwsV1BindingSupport.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.adapter;
+
+import java.io.IOException;
+import java.net.URI;
+import javax.annotation.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.s3a.impl.InstantiationIOException;
+
+import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.unavailable;
+
+/**
+ * Binding support; the sole way which the rest of the code should instantiate v1 SDK libraries.
+ * Uses this class's Classloader for its analysis/loading.
+ */
+@SuppressWarnings("StaticNonFinalField")
+public final class AwsV1BindingSupport {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ AwsV1BindingSupport.class);
+
+ /**
+ * V1 credential provider classname: {@code}.
+ */
+ public static final String CREDENTIAL_PROVIDER_CLASSNAME =
+ "com.amazonaws.auth.AWSCredentialsProvider";
+
+ /**
+ * SDK availability.
+ */
+ private static final boolean SDK_V1_FOUND = checkForAwsV1Sdk();
+
+ private AwsV1BindingSupport() {
+ }
+
+ /**
+ * Probe for the AWS v1 SDK being available by looking for
+ * the class {@link #CREDENTIAL_PROVIDER_CLASSNAME}.
+ * @return true if it was found in the classloader
+ */
+ private static boolean checkForAwsV1Sdk() {
+
+ try {
+ ClassLoader cl = AwsV1BindingSupport.class.getClassLoader();
+ cl.loadClass(CREDENTIAL_PROVIDER_CLASSNAME);
+ LOG.debug("v1 SDK class {} found", CREDENTIAL_PROVIDER_CLASSNAME);
+ return true;
+ } catch (Exception e) {
+ LOG.debug("v1 SDK class {} not found", CREDENTIAL_PROVIDER_CLASSNAME, e);
+ return false;
+ }
+ }
+
+ /**
+ * Is the AWS v1 SDK available?
+ * @return true if it was found in the classloader
+ */
+ public static synchronized boolean isAwsV1SdkAvailable() {
+ return SDK_V1_FOUND;
+ }
+
+
+ /**
+ * Create an AWS credential provider from its class by using reflection. The
+ * class must implement one of the following means of construction, which are
+ * attempted in order:
+ *
+ * <ol>
+ * <li>a public constructor accepting java.net.URI and
+ * org.apache.hadoop.conf.Configuration</li>
+ * <li>a public constructor accepting
+ * org.apache.hadoop.conf.Configuration</li>
+ * <li>a public static method named getInstance that accepts no
+ * arguments and returns an instance of
+ * com.amazonaws.auth.AWSCredentialsProvider, or</li>
+ * <li>a public default constructor.</li>
+ * </ol>
+ * @param conf configuration
+ * @param className credential classname
+ * @param uri URI of the FS
+ * @param key configuration key to use
+ * @return the instantiated class
+ * @throws InstantiationIOException on any instantiation failure, including v1 SDK not found
+ * @throws IOException anything else.
+ */
+ public static AwsCredentialsProvider createAWSV1CredentialProvider(
+ Configuration conf,
+ String className,
+ @Nullable URI uri,
+ final String key) throws IOException {
+ if (!isAwsV1SdkAvailable()) {
+ throw unavailable(uri, className, key, "No AWS v1 SDK available");
+ }
+ return V1ToV2AwsCredentialProviderAdapter.create(conf, className, uri);
+ }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/V1ToV2AwsCredentialProviderAdapter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/V1ToV2AwsCredentialProviderAdapter.java
new file mode 100644
index 000000000000..2cbd9836d3a4
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/V1ToV2AwsCredentialProviderAdapter.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.adapter;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.URI;
+import javax.annotation.Nullable;
+
+import com.amazonaws.SdkClientException;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.AWSSessionCredentials;
+import com.amazonaws.auth.AnonymousAWSCredentials;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.s3a.CredentialInitializationException;
+import org.apache.hadoop.fs.s3a.S3AUtils;
+import org.apache.hadoop.fs.s3a.impl.InstantiationIOException;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.s3a.Constants.AWS_CREDENTIALS_PROVIDER;
+
+/**
+ * Adapts a V1 {@link AWSCredentialsProvider} to the V2 {@link AwsCredentialsProvider} interface.
+ */
+public final class V1ToV2AwsCredentialProviderAdapter
+ implements AwsCredentialsProvider, Closeable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ V1ToV2AwsCredentialProviderAdapter.class);
+
+ /**
+ * The V1 credential provider constructed.
+ */
+ private final AWSCredentialsProvider v1CredentialsProvider;
+
+
+ private V1ToV2AwsCredentialProviderAdapter(AWSCredentialsProvider v1CredentialsProvider) {
+ this.v1CredentialsProvider = requireNonNull(v1CredentialsProvider);
+ }
+
+
+ /**
+ * Collect v1 credentials and convert to v2.
+ * @return v2 credentials
+ * @throws CredentialInitializationException if the inner retrieval raised an exception
+ */
+ @Override
+ public AwsCredentials resolveCredentials() {
+ try {
+ // get the wrapped credentials
+ AWSCredentials toAdapt = v1CredentialsProvider.getCredentials();
+ return convertToV2Credentials(toAdapt);
+ } catch (SdkClientException e) {
+ // wrap with a v2 exception so that code which adds a try/catch for v2 sdk exceptions
+ // gets a compatible exception.
+ throw new CredentialInitializationException(e.toString(), e);
+ }
+ }
+
+ /**
+ * Close the wrapped provider if it implements Closeable/AutoCloseable.
+ * @throws IOException failure
+ */
+ @Override
+ public void close() throws IOException {
+ if (v1CredentialsProvider instanceof Closeable) {
+ ((Closeable) v1CredentialsProvider).close();
+ } else if (v1CredentialsProvider instanceof AutoCloseable) {
+ S3AUtils.closeAutocloseables(LOG, (AutoCloseable) v1CredentialsProvider);
+ }
+ }
+
+ /**
+ * Convert v1 credentials to v2, including support for session and anonymous
+ * credentials.
+ * @param toAdapt credentials to adapt.
+ * @return v2 credentials.
+ */
+ static AwsCredentials convertToV2Credentials(final AWSCredentials toAdapt) {
+ if (toAdapt instanceof AWSSessionCredentials) {
+ return AwsSessionCredentials.create(toAdapt.getAWSAccessKeyId(),
+ toAdapt.getAWSSecretKey(),
+ ((AWSSessionCredentials) toAdapt).getSessionToken());
+ } else if (toAdapt instanceof AnonymousAWSCredentials) {
+ return AnonymousCredentialsProvider.create().resolveCredentials();
+ } else {
+ return AwsBasicCredentials.create(toAdapt.getAWSAccessKeyId(), toAdapt.getAWSSecretKey());
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "V1ToV2AwsCredentialProviderAdapter{" +
+ "v1CredentialsProvider=" + v1CredentialsProvider +
+ '}';
+ }
+
+ /**
+ * @param v1CredentialsProvider V1 credential provider to adapt.
+ * @return A new instance of the credentials provider adapter.
+ */
+ static AwsCredentialsProvider create(AWSCredentialsProvider v1CredentialsProvider) {
+ return new V1ToV2AwsCredentialProviderAdapter(v1CredentialsProvider);
+ }
+
+ /**
+ * Create an AWS credential provider from its class by using reflection. The
+ * class must implement one of the following means of construction, which are
+ * attempted in order:
+ *
+ * <ol>
+ * <li>a public constructor accepting java.net.URI and
+ * org.apache.hadoop.conf.Configuration</li>
+ * <li>a public constructor accepting
+ * org.apache.hadoop.conf.Configuration</li>
+ * <li>a public static method named getInstance that accepts no
+ * arguments and returns an instance of
+ * com.amazonaws.auth.AWSCredentialsProvider, or</li>
+ * <li>a public default constructor.</li>
+ * </ol>
+ * @param conf configuration
+ * @param className classname
+ * @param uri URI of the FS
+ * @return the instantiated class
+ * @throws InstantiationIOException on construction and instantiation failures,
+ * including v1 SDK exceptions.
+ * @throws IOException if raised by a constructor/factory method.
+ */
+ static AwsCredentialsProvider create(
+ Configuration conf,
+ String className,
+ @Nullable URI uri) throws InstantiationIOException, IOException {
+
+ final AWSCredentialsProvider instance =
+ S3AUtils.getInstanceFromReflection(className, conf, uri, AWSCredentialsProvider.class,
+ "getInstance", AWS_CREDENTIALS_PROVIDER);
+ return create(instance);
+ }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/UploadInfo.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/package-info.java
similarity index 59%
copy from hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/UploadInfo.java
copy to hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/package-info.java
index 238cd97ed8f4..124534188bcc 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/UploadInfo.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/package-info.java
@@ -16,28 +16,16 @@
* limitations under the License.
*/
-package org.apache.hadoop.fs.s3a;
-
-import com.amazonaws.services.s3.transfer.Upload;
-
/**
- * Simple struct that contains information about a S3 upload.
+ * Adapter classes for allowing V1 credential providers to be used with SDKV2.
+ * This is the only package where use of aws v1 classes are permitted;
+ * all instantiations of objects here must use reflection to probe for
+ * availability or be prepared to catch exceptions which may be raised
+ * if the v1 SDK isn't found on the classpath
*/
-public class UploadInfo {
- private final Upload upload;
- private final long length;
-
- public UploadInfo(Upload upload, long length) {
- this.upload = upload;
- this.length = length;
- }
-
- public Upload getUpload() {
- return upload;
- }
-
- public long getLength() {
- return length;
- }
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.fs.s3a.adapter;
-}
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java
index 2a4771925f08..99a898f72816 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java
@@ -19,41 +19,37 @@
package org.apache.hadoop.fs.s3a.api;
import javax.annotation.Nullable;
-import java.io.File;
-import java.io.InputStream;
import java.util.List;
-import java.util.Optional;
-
-import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CannedAccessControlList;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CopyObjectRequest;
-import com.amazonaws.services.s3.model.DeleteObjectRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
-import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
-import com.amazonaws.services.s3.model.ListNextBatchOfObjectsRequest;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ListObjectsV2Request;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PartETag;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams;
-import com.amazonaws.services.s3.model.SSECustomerKey;
-import com.amazonaws.services.s3.model.SelectObjectContentRequest;
-import com.amazonaws.services.s3.model.StorageClass;
-import com.amazonaws.services.s3.model.UploadPartRequest;
+import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CompletedPart;
+import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
+import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest;
+import software.amazon.awssdk.services.s3.model.StorageClass;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.s3a.S3AEncryptionMethods;
import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets;
import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
/**
- * Factory for S3 objects.
+ * Factory for S3 request objects.
*
* This is where the owner FS's {@code prepareRequest()}
* callback is invoked to mark up a request for this span.
@@ -67,6 +63,8 @@ import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
* as there are no guarantees how they are processed.
* That is: no guarantees of retry or translation.
*/
+@InterfaceStability.Unstable
+@InterfaceAudience.LimitedPrivate("testing/diagnostics")
public interface RequestFactory {
/**
@@ -79,22 +77,7 @@ public interface RequestFactory {
* Get the canned ACL of this FS.
* @return an ACL, if any
*/
- CannedAccessControlList getCannedACL();
-
- /**
- * Create the AWS SDK structure used to configure SSE,
- * if the encryption secrets contain the information/settings for this.
- * @return an optional set of KMS Key settings
- */
- Optional<SSEAwsKeyManagementParams> generateSSEAwsKeyParams();
-
- /**
- * Create the SSE-C structure for the AWS SDK, if the encryption secrets
- * contain the information/settings for this.
- * This will contain a secret extracted from the bucket/configuration.
- * @return an optional customer key.
- */
- Optional<SSECustomerKey> generateSSECustomerKey();
+ String getCannedACL();
/**
* Get the encryption algorithm of this endpoint.
@@ -115,79 +98,58 @@ public interface RequestFactory {
StorageClass getStorageClass();
/**
- * Create a new object metadata instance.
- * Any standard metadata headers are added here, for example:
- * encryption.
- *
- * @param length length of data to set in header; Ignored if negative
- * @return a new metadata instance
- */
- ObjectMetadata newObjectMetadata(long length);
-
- /**
- * Create a copy request.
+ * Create a copy request builder.
* This includes the work of copying the relevant parts
* of the metadata from the source
* @param srcKey source
* @param dstKey destination
* @param srcom source object metadata.
- * @return the request
+ * @return the request builder
*/
- CopyObjectRequest newCopyObjectRequest(String srcKey,
+ CopyObjectRequest.Builder newCopyObjectRequestBuilder(String srcKey,
String dstKey,
- ObjectMetadata srcom);
+ HeadObjectResponse srcom);
- /**
- * Create a putObject request.
- * Adds the ACL and metadata
- * @param key key of object
- * @param metadata metadata header
- * @param options options for the request
- * @param srcfile source file
- * @return the request
- */
- PutObjectRequest newPutObjectRequest(String key,
- ObjectMetadata metadata, PutObjectOptions options, File srcfile);
/**
- * Create a {@link PutObjectRequest} request.
+ * Create a {@link PutObjectRequest} request builder.
* The metadata is assumed to have been configured with the size of the
* operation.
* @param key key of object
- * @param metadata metadata header
* @param options options for the request
- * @param inputStream source data.
- * @return the request
+ * @param length length of object to be uploaded
+ * @param isDirectoryMarker true if object to be uploaded is a directory marker
+ * @return the request builder
*/
- PutObjectRequest newPutObjectRequest(String key,
- ObjectMetadata metadata,
+ PutObjectRequest.Builder newPutObjectRequestBuilder(String key,
PutObjectOptions options,
- InputStream inputStream);
+ long length,
+ boolean isDirectoryMarker);
/**
* Create a {@link PutObjectRequest} request for creating
* an empty directory.
*
* @param directory destination directory.
- * @return request for a zero byte upload.
+ * @return request builder for a zero byte upload.
*/
- PutObjectRequest newDirectoryMarkerRequest(String directory);
+ PutObjectRequest.Builder newDirectoryMarkerRequest(String directory);
/**
* List all multipart uploads under a prefix.
* @param prefix prefix to list under
- * @return the request.
+ * @return the request builder.
*/
- ListMultipartUploadsRequest newListMultipartUploadsRequest(
+ ListMultipartUploadsRequest.Builder newListMultipartUploadsRequestBuilder(
@Nullable String prefix);
/**
* Abort a multipart upload.
* @param destKey destination object key
* @param uploadId ID of initiated upload
- * @return the request.
+ * @return the request builder.
*/
- AbortMultipartUploadRequest newAbortMultipartUploadRequest(
+ AbortMultipartUploadRequest.Builder newAbortMultipartUploadRequestBuilder(
String destKey,
String uploadId);
@@ -195,10 +157,10 @@ public interface RequestFactory {
* Start a multipart upload.
* @param destKey destination object key
* @param options options for the request
- * @return the request.
+ * @return the request builder.
* @throws PathIOException if multipart uploads are disabled
*/
- InitiateMultipartUploadRequest newMultipartUploadRequest(
+ CreateMultipartUploadRequest.Builder newMultipartUploadRequestBuilder(
String destKey,
@Nullable PutObjectOptions options) throws PathIOException;
@@ -207,107 +169,95 @@ public interface RequestFactory {
* @param destKey destination object key
* @param uploadId ID of initiated upload
* @param partETags ordered list of etags
- * @return the request.
+ * @return the request builder.
*/
- CompleteMultipartUploadRequest newCompleteMultipartUploadRequest(
+ CompleteMultipartUploadRequest.Builder newCompleteMultipartUploadRequestBuilder(
String destKey,
String uploadId,
- List<PartETag> partETags);
+ List<CompletedPart> partETags);
/**
- * Create a HEAD request.
+ * Create a HEAD object request builder.
* @param key key, may have trailing /
- * @return the request.
+ * @return the request builder.
+ */
+ HeadObjectRequest.Builder newHeadObjectRequestBuilder(String key);
+
+ /**
+ * Create a HEAD bucket request builder.
+ * @param bucket bucket to get metadata for
+ * @return the request builder.
*/
- GetObjectMetadataRequest newGetObjectMetadataRequest(String key);
+ HeadBucketRequest.Builder newHeadBucketRequestBuilder(String bucket);
+
/**
- * Create a GET request.
+ * Create a GET request builder.
* @param key object key
- * @return the request.
+ * @return the request builder.
*/
- GetObjectRequest newGetObjectRequest(String key);
+ GetObjectRequest.Builder newGetObjectRequestBuilder(String key);
/**
- * Create and initialize a part request of a multipart upload.
- * Exactly one of: {@code uploadStream} or {@code sourceFile}
- * must be specified.
- * A subset of the file may be posted, by providing the starting point
- * in {@code offset} and a length of block in {@code size} equal to
- * or less than the remaining bytes.
- * @param destKey destination key of ongoing operation
- * @param uploadId ID of ongoing upload
- * @param partNumber current part number of the upload
- * @param size amount of data
- * @param uploadStream source of data to upload
- * @param sourceFile optional source file.
- * @param offset offset in file to start reading.
- * @return the request.
+ * Create and initialize a part request builder of a multipart upload.
+ *
+ * @param destKey destination key of ongoing operation
+ * @param uploadId ID of ongoing upload
+ * @param partNumber current part number of the upload
+ * @param size amount of data
+ * @return the request builder.
* @throws PathIOException if the part number is out of range.
*/
- UploadPartRequest newUploadPartRequest(
+ UploadPartRequest.Builder newUploadPartRequestBuilder(
String destKey,
String uploadId,
int partNumber,
- long size,
- InputStream uploadStream,
- File sourceFile,
- long offset) throws PathIOException;
+ long size) throws PathIOException;
/**
- * Create a S3 Select request for the destination object.
+ * Create a S3 Select request builder for the destination object.
* This does not build the query.
* @param key object key
- * @return the request
+ * @return the request builder
*/
- SelectObjectContentRequest newSelectRequest(String key);
+ SelectObjectContentRequest.Builder newSelectRequestBuilder(String key);
/**
- * Create the (legacy) V1 list request.
+ * Create the (legacy) V1 list request builder.
* @param key key to list under
* @param delimiter delimiter for keys
* @param maxKeys maximum number in a list page.
- * @return the request
+ * @return the request builder.
*/
- ListObjectsRequest newListObjectsV1Request(String key,
+ ListObjectsRequest.Builder newListObjectsV1RequestBuilder(String key,
String delimiter,
int maxKeys);
/**
- * Create the next V1 page list request, following
- * on from the previous response.
- * @param prev previous response
- * @return the request
- */
-
- ListNextBatchOfObjectsRequest newListNextBatchOfObjectsRequest(
- ObjectListing prev);
-
- /**
- * Create a V2 list request.
+ * Create a V2 list request builder.
* This will be recycled for any subsequent requests.
* @param key key to list under
* @param delimiter delimiter for keys
* @param maxKeys maximum number in a list page.
- * @return the request
+ * @return the request builder.
*/
- ListObjectsV2Request newListObjectsV2Request(String key,
+ ListObjectsV2Request.Builder newListObjectsV2RequestBuilder(String key,
String delimiter,
int maxKeys);
/**
- * Create a request to delete a single object.
+ * Create a request builder to delete a single object.
* @param key object to delete
- * @return the request
+ * @return the request builder.
*/
- DeleteObjectRequest newDeleteObjectRequest(String key);
+ DeleteObjectRequest.Builder newDeleteObjectRequestBuilder(String key);
/**
- * Bulk delete request.
+ * Create a request builder to delete objects in bulk.
* @param keysToDelete list of keys to delete.
- * @return the request
+ * @return the request builder.
*/
- DeleteObjectsRequest newBulkDeleteRequest(
- List<DeleteObjectsRequest.KeyVersion> keysToDelete);
+ DeleteObjectsRequest.Builder newBulkDeleteRequestBuilder(
+ List<ObjectIdentifier> keysToDelete);
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSAuditEventCallbacks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSAuditEventCallbacks.java
index 8134d5cea942..712246c21961 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSAuditEventCallbacks.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSAuditEventCallbacks.java
@@ -18,15 +18,9 @@
package org.apache.hadoop.fs.s3a.audit;
-import com.amazonaws.AmazonWebServiceRequest;
-import com.amazonaws.Request;
-import com.amazonaws.Response;
-import com.amazonaws.SdkBaseException;
-import com.amazonaws.handlers.HandlerAfterAttemptContext;
-import com.amazonaws.handlers.HandlerBeforeAttemptContext;
-import com.amazonaws.http.HttpResponse;
+import software.amazon.awssdk.core.SdkRequest;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
-import org.apache.hadoop.fs.s3a.Retries;
/**
* Callbacks for audit spans. This is implemented
@@ -37,10 +31,10 @@ import org.apache.hadoop.fs.s3a.Retries;
* detect this and raise an exception.
*
* Look at the documentation for
- * {@code com.amazonaws.handlers.IRequestHandler2} for details
+ * {@code ExecutionInterceptor} for details
* on the callbacks.
*/
-public interface AWSAuditEventCallbacks {
+public interface AWSAuditEventCallbacks extends ExecutionInterceptor {
/**
* Return a span ID which must be unique for all spans within
@@ -66,95 +60,8 @@ public interface AWSAuditEventCallbacks {
* It is not invoked on any AWS requests created in the SDK.
* Avoid raising exceptions or talking to any remote service;
* this callback is for annotation rather than validation.
- * @param request request request.
- * @param <T> type of request
- * @return the request, possibly modified.
+ * @param builder the request builder.
*/
- default <T extends AmazonWebServiceRequest> T requestCreated(T request) {
- return request;
- }
+ default void requestCreated(SdkRequest.Builder builder) {}
- /**
- * Preflight preparation of AWS request.
- * @param request request
- * @param <T> type of request
- * @return an updated request.
- * @throws AuditFailureException for generic audit failures
- * @throws SdkBaseException for other reasons.
- */
- @Retries.OnceRaw
- default <T extends AmazonWebServiceRequest> T beforeExecution(T request)
- throws AuditFailureException, SdkBaseException {
- return request;
- }
-
- /**
- * Callback after S3 responded to a request.
- * @param request request
- * @param response response.
- * @throws AuditFailureException for generic audit failures
- * @throws SdkBaseException for other reasons.
- */
- default void afterResponse(Request<?> request,
- Response<?> response)
- throws AuditFailureException, SdkBaseException {
- }
-
- /**
- * Callback after a request resulted in an error.
- * @param request request
- * @param response response.
- * @param exception exception raised.
- * @throws AuditFailureException for generic audit failures
- * @throws SdkBaseException for other reasons.
- */
- default void afterError(Request<?> request,
- Response<?> response,
- Exception exception)
- throws AuditFailureException, SdkBaseException {
- }
-
- /**
- * Request before marshalling.
- * @param request request
- * @return possibly modified request.
- */
- default AmazonWebServiceRequest beforeMarshalling(
- AmazonWebServiceRequest request) {
- return request;
- }
-
- /**
- * Request before marshalling.
- * @param request request
- */
- default void beforeRequest(Request<?> request) {
- }
-
- /**
- * Before any attempt is made.
- * @param context full context, including the request.
- */
- default void beforeAttempt(HandlerBeforeAttemptContext context) {
- }
-
- /**
- * After any attempt is made.
- * @param context full context, including the request.
- */
- default void afterAttempt(
- HandlerAfterAttemptContext context) {
- }
-
- /**
- * Before unmarshalling the response.
- * @param request request made.
- * @param httpResponse response received
- * @return updated response.
- */
- default HttpResponse beforeUnmarshalling(
- final Request<?> request,
- final HttpResponse httpResponse) {
- return httpResponse;
- }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java
index b4be341c912e..8a24a4e14db1 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java
@@ -20,24 +20,23 @@ package org.apache.hadoop.fs.s3a.audit;
import java.util.List;
-import com.amazonaws.AmazonWebServiceRequest;
-import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CopyPartRequest;
-import com.amazonaws.services.s3.model.DeleteObjectRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.GetBucketLocationRequest;
-import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
-import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
-import com.amazonaws.services.s3.model.ListNextBatchOfObjectsRequest;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ListObjectsV2Request;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.SelectObjectContentRequest;
-import com.amazonaws.services.s3.model.UploadPartRequest;
+import software.amazon.awssdk.core.SdkRequest;
+import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.GetBucketLocationRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartCopyRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
import static org.apache.hadoop.fs.statistics.StoreStatisticNames.ACTION_HTTP_GET_REQUEST;
import static org.apache.hadoop.fs.statistics.StoreStatisticNames.ACTION_HTTP_HEAD_REQUEST;
@@ -64,102 +63,85 @@ public class AWSRequestAnalyzer {
* read/write and path.
* @param request request.
* @return information about the request.
- * @param <T> type of request.
*/
- public <T extends AmazonWebServiceRequest> RequestInfo analyze(T request) {
+ public RequestInfo analyze(SdkRequest request) {
// this is where Scala's case statement would massively
// simplify life.
// Please Keep in Alphabetical Order.
if (request instanceof AbortMultipartUploadRequest) {
return writing(MULTIPART_UPLOAD_ABORTED,
- ((AbortMultipartUploadRequest) request).getKey(),
+ ((AbortMultipartUploadRequest) request).key(),
0);
} else if (request instanceof CompleteMultipartUploadRequest) {
CompleteMultipartUploadRequest r
= (CompleteMultipartUploadRequest) request;
return writing(MULTIPART_UPLOAD_COMPLETED,
- r.getKey(),
- r.getPartETags().size());
+ r.key(),
+ r.multipartUpload().parts().size());
+ } else if (request instanceof CreateMultipartUploadRequest) {
+ return writing(MULTIPART_UPLOAD_STARTED,
+ ((CreateMultipartUploadRequest) request).key(),
+ 0);
} else if (request instanceof DeleteObjectRequest) {
// DeleteObject: single object
return writing(OBJECT_DELETE_REQUEST,
- ((DeleteObjectRequest) request).getKey(),
+ ((DeleteObjectRequest) request).key(),
1);
} else if (request instanceof DeleteObjectsRequest) {
// DeleteObjects: bulk delete
// use first key as the path
DeleteObjectsRequest r = (DeleteObjectsRequest) request;
- List<DeleteObjectsRequest.KeyVersion> keys
- = r.getKeys();
+ List<ObjectIdentifier> objectIdentifiers
+ = r.delete().objects();
return writing(OBJECT_BULK_DELETE_REQUEST,
- keys.isEmpty() ? null : keys.get(0).getKey(),
- keys.size());
+ objectIdentifiers.isEmpty() ? null : objectIdentifiers.get(0).key(),
+ objectIdentifiers.size());
} else if (request instanceof GetBucketLocationRequest) {
GetBucketLocationRequest r = (GetBucketLocationRequest) request;
return reading(STORE_EXISTS_PROBE,
- r.getBucketName(),
+ r.bucket(),
0);
- } else if (request instanceof GetObjectMetadataRequest) {
- return reading(ACTION_HTTP_HEAD_REQUEST,
- ((GetObjectMetadataRequest) request).getKey(), 0);
} else if (request instanceof GetObjectRequest) {
GetObjectRequest r = (GetObjectRequest) request;
- long[] range = r.getRange();
- long size = range == null
- ? -1
- : range[1] - range[0];
return reading(ACTION_HTTP_GET_REQUEST,
- r.getKey(),
- size);
- } else if (request instanceof InitiateMultipartUploadRequest) {
- return writing(MULTIPART_UPLOAD_STARTED,
- ((InitiateMultipartUploadRequest) request).getKey(),
- 0);
+ r.key(),
+ sizeFromRangeHeader(r.range()));
+ } else if (request instanceof HeadObjectRequest) {
+ return reading(ACTION_HTTP_HEAD_REQUEST,
+ ((HeadObjectRequest) request).key(), 0);
} else if (request instanceof ListMultipartUploadsRequest) {
ListMultipartUploadsRequest r
= (ListMultipartUploadsRequest) request;
return reading(MULTIPART_UPLOAD_LIST,
- r.getPrefix(),
- r.getMaxUploads());
+ r.prefix(),
+ r.maxUploads());
} else if (request instanceof ListObjectsRequest) {
ListObjectsRequest r = (ListObjectsRequest) request;
return reading(OBJECT_LIST_REQUEST,
- r.getPrefix(),
- r.getMaxKeys());
- } else if (request instanceof ListNextBatchOfObjectsRequest) {
- ListNextBatchOfObjectsRequest r = (ListNextBatchOfObjectsRequest) request;
- ObjectListing l = r.getPreviousObjectListing();
- String prefix = "";
- int size = 0;
- if (l != null) {
- prefix = l.getPrefix();
- size = l.getMaxKeys();
- }
- return reading(OBJECT_LIST_REQUEST,
- prefix,
- size);
+ r.prefix(),
+ r.maxKeys());
} else if (request instanceof ListObjectsV2Request) {
ListObjectsV2Request r = (ListObjectsV2Request) request;
return reading(OBJECT_LIST_REQUEST,
- r.getPrefix(),
- r.getMaxKeys());
+ r.prefix(),
+ r.maxKeys());
} else if (request instanceof PutObjectRequest) {
PutObjectRequest r = (PutObjectRequest) request;
return writing(OBJECT_PUT_REQUEST,
- r.getKey(),
+ r.key(),
0);
} else if (request instanceof SelectObjectContentRequest) {
SelectObjectContentRequest r =
(SelectObjectContentRequest) request;
return reading(OBJECT_SELECT_REQUESTS,
- r.getKey(),
+ r.key(),
1);
} else if (request instanceof UploadPartRequest) {
UploadPartRequest r = (UploadPartRequest) request;
return writing(MULTIPART_UPLOAD_PART_PUT,
- r.getKey(),
- r.getPartSize());
+ r.key(),
+ r.contentLength());
}
// no explicit support, return classname
return writing(request.getClass().getName(), null, 0);
@@ -212,7 +194,7 @@ public class AWSRequestAnalyzer {
*/
public static boolean
isRequestNotAlwaysInSpan(final Object request) {
- return request instanceof CopyPartRequest
+ return request instanceof UploadPartCopyRequest
|| request instanceof CompleteMultipartUploadRequest
|| request instanceof GetBucketLocationRequest;
}
@@ -225,9 +207,9 @@ public class AWSRequestAnalyzer {
* @return true if the transfer manager creates them.
*/
public static boolean isRequestMultipartIO(final Object request) {
- return request instanceof CopyPartRequest
+ return request instanceof UploadPartCopyRequest
|| request instanceof CompleteMultipartUploadRequest
- || request instanceof InitiateMultipartUploadRequest
+ || request instanceof CreateMultipartUploadRequest
|| request instanceof UploadPartRequest;
}
@@ -307,4 +289,23 @@ public class AWSRequestAnalyzer {
private static long toSafeLong(final Number size) {
return size != null ? size.longValue() : 0;
}
+
+ private static final String BYTES_PREFIX = "bytes=";
+
+ private static Number sizeFromRangeHeader(String rangeHeader) {
+ if (rangeHeader != null && rangeHeader.startsWith(BYTES_PREFIX)) {
+ String[] values = rangeHeader
+ .substring(BYTES_PREFIX.length())
+ .split("-");
+ if (values.length == 2) {
+ try {
+ long start = Long.parseUnsignedLong(values[0]);
+ long end = Long.parseUnsignedLong(values[0]);
+ return end - start;
+ } catch(NumberFormatException e) {
+ }
+ }
+ }
+ return -1;
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditIntegration.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditIntegration.java
index c66f45eb309f..dfbe8e1a1350 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditIntegration.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditIntegration.java
@@ -23,7 +23,7 @@ import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.nio.file.AccessDeniedException;
-import com.amazonaws.HandlerContextAware;
+import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,10 +34,11 @@ import org.apache.hadoop.fs.s3a.audit.impl.LoggingAuditor;
import org.apache.hadoop.fs.s3a.audit.impl.NoopAuditManagerS3A;
import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+
import static java.util.Objects.requireNonNull;
import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_ENABLED;
import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_ENABLED_DEFAULT;
-import static org.apache.hadoop.fs.s3a.audit.impl.S3AInternalAuditConstants.AUDIT_SPAN_HANDLER_CONTEXT;
+import static org.apache.hadoop.fs.s3a.audit.impl.S3AInternalAuditConstants.AUDIT_SPAN_EXECUTION_ATTRIBUTE;
/**
* Support for integrating auditing within the S3A code.
@@ -123,25 +124,24 @@ public final class AuditIntegration {
}
/**
- * Get the span from a handler context.
- * @param request request
- * @param <T> type of request.
+ * Get the span from the execution attributes.
+ * @param executionAttributes the execution attributes
* @return the span callbacks or null
*/
- public static <T extends HandlerContextAware> AWSAuditEventCallbacks
- retrieveAttachedSpan(final T request) {
- return request.getHandlerContext(AUDIT_SPAN_HANDLER_CONTEXT);
+ public static AuditSpanS3A
+ retrieveAttachedSpan(final ExecutionAttributes executionAttributes) {
+ return executionAttributes.getAttribute(AUDIT_SPAN_EXECUTION_ATTRIBUTE);
}
/**
- * Attach a span to a handler context.
- * @param request request
+ * Attach a span to the execution attributes.
+ * @param executionAttributes the execution attributes
* @param span span to attach
- * @param <T> type of request.
*/
- public static <T extends HandlerContextAware> void attachSpanToRequest(
- final T request, final AWSAuditEventCallbacks span) {
- request.addHandlerContext(AUDIT_SPAN_HANDLER_CONTEXT, span);
+ public static void attachSpanToRequest(
+ final ExecutionAttributes executionAttributes,
+ final AuditSpanS3A span) {
+ executionAttributes.putAttribute(AUDIT_SPAN_EXECUTION_ATTRIBUTE, span);
}
/**
@@ -160,4 +160,30 @@ public final class AuditIntegration {
return (AccessDeniedException)new AccessDeniedException(path, null,
exception.toString()).initCause(exception);
}
+
+ /**
+ * Translate an exception if it or its inner exception is an
+ * {@link AuditFailureException}.
+ * If this condition is not met, null is returned.
+ * @param path path of operation.
+ * @param exception exception
+ * @return a translated exception or null.
+ */
+ public static IOException maybeTranslateAuditException(String path,
+ Exception exception) {
+ if (exception instanceof AuditFailureException) {
+ return translateAuditException(path, (AuditFailureException) exception);
+ } else if (exception.getCause() instanceof AuditFailureException) {
+ return translateAuditException(path,
+ (AuditFailureException) exception.getCause());
+ } else {
+ return null;
+ }
+ }
+
+ public static boolean containsAuditException(Exception exception) {
+ return exception instanceof AuditFailureException
+ || exception.getCause() instanceof AuditFailureException;
+ }
+
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditManagerS3A.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditManagerS3A.java
index c1302d57454b..bb7f94cfc206 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditManagerS3A.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditManagerS3A.java
@@ -21,8 +21,8 @@ package org.apache.hadoop.fs.s3a.audit;
import java.io.IOException;
import java.util.List;
-import com.amazonaws.handlers.RequestHandler2;
-import com.amazonaws.services.s3.transfer.internal.TransferStateChangeListener;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
+import software.amazon.awssdk.transfer.s3.progress.TransferListener;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.Path;
@@ -32,6 +32,7 @@ import org.apache.hadoop.fs.store.audit.ActiveThreadSpanSource;
import org.apache.hadoop.fs.store.audit.AuditSpanSource;
import org.apache.hadoop.service.Service;
+
/**
* Interface for Audit Managers auditing operations through the
* AWS libraries.
@@ -56,24 +57,24 @@ public interface AuditManagerS3A extends Service,
OperationAuditor getAuditor();
/**
- * Create the request handler(s) for this audit service.
- * The list returned is mutable; new handlers may be added.
- * @return list of handlers for the SDK.
+ * Create the execution interceptor(s) for this audit service.
+ * The list returned is mutable; new interceptors may be added.
+ * @return list of interceptors for the SDK.
* @throws IOException failure.
*/
- List<RequestHandler2> createRequestHandlers() throws IOException;
+ List<ExecutionInterceptor> createExecutionInterceptors() throws IOException;
/**
- * Return a transfer state change callback which
+ * Return a transfer callback which
* fixes the active span context to be that in which
- * the state change listener was created.
+ * the transfer listener was created.
* This can be used to audit the creation of the multipart
* upload initiation request which the transfer manager
* makes when a file to be copied is split up.
* This must be invoked/used within the active span.
- * @return a state change listener.
+ * @return a transfer listener.
*/
- TransferStateChangeListener createStateChangeListener();
+ TransferListener createTransferListener();
/**
* Check for permission to access a path.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3AAuditConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3AAuditConstants.java
index 1d76833f8cea..55deb0a1989d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3AAuditConstants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3AAuditConstants.java
@@ -66,13 +66,20 @@ public final class S3AAuditConstants {
"org.apache.hadoop.fs.s3a.audit.impl.NoopAuditor";
/**
- * List of extra AWS SDK request handlers: {@value}.
- * These are added to the SDK request chain <i>after</i>
- * any audit service.
+ * Deprecated list of extra AWS SDK request handlers: {@value}.
+ * Use {@link #AUDIT_EXECUTION_INTERCEPTORS} instead.
*/
public static final String AUDIT_REQUEST_HANDLERS =
"fs.s3a.audit.request.handlers";
+ /**
+ * List of extra AWS SDK execution interceptors: {@value}.
+ * These are added to the SDK request chain <i>after</i>
+ * any audit service.
+ */
+ public static final String AUDIT_EXECUTION_INTERCEPTORS =
+ "fs.s3a.audit.execution.interceptors";
+
/**
* Should operations outside spans be rejected?
* This is for testing coverage of the span code; if used
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/ActiveAuditManagerS3A.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/ActiveAuditManagerS3A.java
index ffd08d37e4aa..1da203dc8024 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/ActiveAuditManagerS3A.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/ActiveAuditManagerS3A.java
@@ -25,16 +25,16 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
-import com.amazonaws.AmazonWebServiceRequest;
-import com.amazonaws.HandlerContextAware;
-import com.amazonaws.Request;
-import com.amazonaws.Response;
-import com.amazonaws.SdkBaseException;
-import com.amazonaws.handlers.HandlerAfterAttemptContext;
-import com.amazonaws.handlers.HandlerBeforeAttemptContext;
-import com.amazonaws.handlers.RequestHandler2;
-import com.amazonaws.http.HttpResponse;
-import com.amazonaws.services.s3.transfer.internal.TransferStateChangeListener;
+import software.amazon.awssdk.core.SdkRequest;
+import software.amazon.awssdk.core.SdkResponse;
+import software.amazon.awssdk.core.interceptor.Context;
+import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
+import software.amazon.awssdk.http.SdkHttpRequest;
+import software.amazon.awssdk.http.SdkHttpResponse;
+import software.amazon.awssdk.transfer.s3.progress.TransferListener;
+
+import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,6 +56,7 @@ import org.apache.hadoop.fs.s3a.audit.AuditSpanS3A;
import org.apache.hadoop.fs.s3a.audit.OperationAuditor;
import org.apache.hadoop.fs.s3a.audit.OperationAuditorOptions;
import org.apache.hadoop.fs.s3a.audit.S3AAuditConstants;
+import org.apache.hadoop.fs.s3a.impl.V2Migration;
import org.apache.hadoop.fs.store.LogExactlyOnce;
import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
import org.apache.hadoop.service.CompositeService;
@@ -66,6 +67,7 @@ import static org.apache.hadoop.fs.s3a.Statistic.AUDIT_FAILURE;
import static org.apache.hadoop.fs.s3a.Statistic.AUDIT_REQUEST_EXECUTION;
import static org.apache.hadoop.fs.s3a.audit.AuditIntegration.attachSpanToRequest;
import static org.apache.hadoop.fs.s3a.audit.AuditIntegration.retrieveAttachedSpan;
+import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_EXECUTION_INTERCEPTORS;
import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_REQUEST_HANDLERS;
/**
@@ -82,10 +84,11 @@ import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_REQUEST_HAN
* will deactivate the wrapped span and then
* switch the active span to the unbounded span.
*
- * The inner class {@link AWSAuditEventCallbacks} is returned
- * as a request handler in {@link #createRequestHandlers()};
- * this forwards all requests to the outer {@code ActiveAuditManagerS3A},
- * which then locates the active span and forwards the request.
+ * This class also implements {@link ExecutionInterceptor} and
+ * returns itself in {@link #createExecutionInterceptors()};
+ * once registered with the S3 client, the implemented methods
+ * will be called during different parts of an SDK request lifecycle,
+ * which then locate the active span and forward the request.
* If any such invocation raises an {@link AuditFailureException}
* then the IOStatistics counter for {@code AUDIT_FAILURE}
* is incremented.
@@ -390,25 +393,39 @@ public final class ActiveAuditManagerS3A
}
/**
- * Return a request handler for the AWS SDK which
+ * Return a list of execution interceptors for the AWS SDK which
* relays to this class.
- * @return a request handler.
+ * @return a list of execution interceptors.
*/
@Override
- public List<RequestHandler2> createRequestHandlers()
+ public List<ExecutionInterceptor> createExecutionInterceptors()
throws IOException {
// wire up the AWS SDK To call back into this class when
// preparing to make S3 calls.
- List<RequestHandler2> requestHandlers = new ArrayList<>();
- requestHandlers.add(new SdkRequestHandler());
- // now look for any more handlers
- final Class<?>[] handlers = getConfig().getClasses(AUDIT_REQUEST_HANDLERS);
- if (handlers != null) {
- for (Class<?> handler : handlers) {
+ List<ExecutionInterceptor> executionInterceptors = new ArrayList<>();
+ executionInterceptors.add(this);
+
+ final String handlers = getConfig().getTrimmed(AUDIT_REQUEST_HANDLERS, "");
+ if (!handlers.isEmpty()) {
+ // warn and ignore v1 handlers.
+ V2Migration.v1RequestHandlersUsed(handlers);
+ }
+
+ // V2 SDK supports global/service interceptors, but they need to be configured on the
+ // classpath and don't get the filesystem/job configuration passed down.
+ final Class<?>[] interceptors = getConfig().getClasses(AUDIT_EXECUTION_INTERCEPTORS);
+ if (interceptors != null) {
+ for (Class<?> handler : interceptors) {
try {
+ LOG.debug("Adding intercept of class {}", handler);
Constructor<?> ctor = handler.getConstructor();
- requestHandlers.add((RequestHandler2)ctor.newInstance());
+ final ExecutionInterceptor interceptor = (ExecutionInterceptor) ctor.newInstance();
+ if (interceptor instanceof Configurable) {
+ // pass in the configuration.
+ ((Configurable) interceptor).setConf(getConfig());
+ }
+ executionInterceptors.add(interceptor);
} catch (ExceptionInInitializerError e) {
throw FutureIO.unwrapInnerException(e);
} catch (Exception e) {
@@ -416,13 +433,18 @@ public final class ActiveAuditManagerS3A
}
}
}
- return requestHandlers;
+ return executionInterceptors;
}
@Override
- public TransferStateChangeListener createStateChangeListener() {
+ public TransferListener createTransferListener() {
final WrappingAuditSpan span = activeSpan();
- return (transfer, state) -> switchToActiveSpan(span);
+ return new TransferListener() {
+ @Override
+ public void transferInitiated(Context.TransferInitiated context) {
+ switchToActiveSpan(span);
+ }
+ };
}
@Override
@@ -434,20 +456,18 @@ public final class ActiveAuditManagerS3A
}
/**
- * Attach a reference to the active thread span, then
- * invoke the same callback on that active thread.
+ * Audit the creation of a request and retrieve
+ * a reference to the active thread span.
*/
@Override
- public <T extends AmazonWebServiceRequest> T requestCreated(
- final T request) {
+ public void requestCreated(final SdkRequest.Builder builder) {
AuditSpanS3A span = getActiveAuditSpan();
if (LOG.isTraceEnabled()) {
LOG.trace("Created Request {} in span {}",
- analyzer.analyze(request), span);
+ analyzer.analyze(builder.build()), span);
}
- attachSpanToRequest(request, span);
try {
- return span.requestCreated(request);
+ span.requestCreated(builder);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -463,14 +483,13 @@ public final class ActiveAuditManagerS3A
* {@inheritDoc}
*/
@Override
- public <T extends AmazonWebServiceRequest> T beforeExecution(
- final T request) {
+ public void beforeExecution(Context.BeforeExecution context,
+ ExecutionAttributes executionAttributes) {
ioStatisticsStore.incrementCounter(AUDIT_REQUEST_EXECUTION.getSymbol());
-
- // identify the span and invoke the callback
+ AuditSpanS3A span = getActiveAuditSpan();
+ attachSpanToRequest(executionAttributes, span);
try {
- return extractAndActivateSpanFromRequest(request)
- .beforeExecution(request);
+ span.beforeExecution(context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -479,16 +498,14 @@ public final class ActiveAuditManagerS3A
/**
* Forward to active span.
- * @param request request
- * @param response response.
+ * {@inheritDoc}
*/
@Override
- public void afterResponse(final Request<?> request,
- final Response<?> response)
- throws AuditFailureException, SdkBaseException {
+ public void afterExecution(Context.AfterExecution context,
+ ExecutionAttributes executionAttributes) {
try {
- extractAndActivateSpanFromRequest(request)
- .afterResponse(request, response);
+ extractAndActivateSpanFromRequest(context.request(), executionAttributes)
+ .afterExecution(context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -496,18 +513,19 @@ public final class ActiveAuditManagerS3A
}
/**
- * Get the active span from the handler context,
+ * Get the active span from the execution attributes,
* falling back to the active thread span if there
- * is nothing in the context.
- * Provided the span is a wrapped span, the
+ * is nothing in the attributes.
+ * Provided the span is a wrapped span, the span is
+ * activated.
* @param request request
- * @param <T> type of request.
- * @return the callbacks
+ * @param executionAttributes the execution attributes
+ * @return the active span
*/
- private <T extends HandlerContextAware> AWSAuditEventCallbacks
- extractAndActivateSpanFromRequest(final T request) {
- AWSAuditEventCallbacks span;
- span = retrieveAttachedSpan(request);
+ private AuditSpanS3A extractAndActivateSpanFromRequest(
+ final SdkRequest request,
+ final ExecutionAttributes executionAttributes) {
+ AuditSpanS3A span = retrieveAttachedSpan(executionAttributes);
if (span == null) {
// no span is attached. Not unusual for the copy operations,
// or for calls to GetBucketLocation made by the AWS client
@@ -530,18 +548,16 @@ public final class ActiveAuditManagerS3A
/**
* Forward to active span.
- * @param request request
- * @param response response.
- * @param exception exception raised.
+ * @param context execution context
+ * @param executionAttributes the execution attributes
+ * {@inheritDoc}
*/
@Override
- public void afterError(final Request<?> request,
- final Response<?> response,
- final Exception exception)
- throws AuditFailureException, SdkBaseException {
+ public void onExecutionFailure(Context.FailedExecution context,
+ ExecutionAttributes executionAttributes) {
try {
- extractAndActivateSpanFromRequest(request)
- .afterError(request, response, exception);
+ extractAndActivateSpanFromRequest(context.request(), executionAttributes).onExecutionFailure(
+ context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -549,11 +565,12 @@ public final class ActiveAuditManagerS3A
}
@Override
- public AmazonWebServiceRequest beforeMarshalling(
- final AmazonWebServiceRequest request) {
+ public SdkRequest modifyRequest(Context.ModifyRequest context,
+ ExecutionAttributes executionAttributes) {
try {
- return extractAndActivateSpanFromRequest(request)
- .beforeMarshalling(request);
+ return extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .modifyRequest(context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -561,10 +578,12 @@ public final class ActiveAuditManagerS3A
}
@Override
- public void beforeRequest(final Request<?> request) {
+ public void beforeMarshalling(Context.BeforeMarshalling context,
+ ExecutionAttributes executionAttributes) {
try {
- extractAndActivateSpanFromRequest(request)
- .beforeRequest(request);
+ extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .beforeMarshalling(context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -572,10 +591,12 @@ public final class ActiveAuditManagerS3A
}
@Override
- public void beforeAttempt(final HandlerBeforeAttemptContext context) {
+ public void afterMarshalling(Context.AfterMarshalling context,
+ ExecutionAttributes executionAttributes) {
try {
- extractAndActivateSpanFromRequest(context.getRequest())
- .beforeAttempt(context);
+ extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .afterMarshalling(context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -583,10 +604,12 @@ public final class ActiveAuditManagerS3A
}
@Override
- public void afterAttempt(final HandlerAfterAttemptContext context) {
+ public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context,
+ ExecutionAttributes executionAttributes) {
try {
- extractAndActivateSpanFromRequest(context.getRequest())
- .afterAttempt(context);
+ return extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .modifyHttpRequest(context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -594,73 +617,80 @@ public final class ActiveAuditManagerS3A
}
@Override
- public HttpResponse beforeUnmarshalling(final Request<?> request,
- final HttpResponse httpResponse) {
+ public void beforeTransmission(Context.BeforeTransmission context,
+ ExecutionAttributes executionAttributes) {
try {
- extractAndActivateSpanFromRequest(request.getOriginalRequest())
- .beforeUnmarshalling(request, httpResponse);
+ extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .beforeTransmission(context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
}
- return httpResponse;
}
- /**
- * Callbacks from the AWS SDK; all forward to the ActiveAuditManagerS3A.
- * We need a separate class because the SDK requires the handler list
- * to be list of {@code RequestHandler2} instances.
- */
- private class SdkRequestHandler extends RequestHandler2 {
-
- @Override
- public AmazonWebServiceRequest beforeExecution(
- final AmazonWebServiceRequest request) {
- return ActiveAuditManagerS3A.this.beforeExecution(request);
- }
-
- @Override
- public void afterResponse(final Request<?> request,
- final Response<?> response) {
- ActiveAuditManagerS3A.this.afterResponse(request, response);
- }
-
- @Override
- public void afterError(final Request<?> request,
- final Response<?> response,
- final Exception e) {
- ActiveAuditManagerS3A.this.afterError(request, response, e);
- }
-
- @Override
- public AmazonWebServiceRequest beforeMarshalling(
- final AmazonWebServiceRequest request) {
- return ActiveAuditManagerS3A.this.beforeMarshalling(request);
+ @Override
+ public void afterTransmission(Context.AfterTransmission context,
+ ExecutionAttributes executionAttributes) {
+ try {
+ extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .afterTransmission(context, executionAttributes);
+ } catch (AuditFailureException e) {
+ ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
+ throw e;
}
+ }
- @Override
- public void beforeRequest(final Request<?> request) {
- ActiveAuditManagerS3A.this.beforeRequest(request);
+ @Override
+ public SdkHttpResponse modifyHttpResponse(Context.ModifyHttpResponse context,
+ ExecutionAttributes executionAttributes) {
+ try {
+ return extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .modifyHttpResponse(context, executionAttributes);
+ } catch (AuditFailureException e) {
+ ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
+ throw e;
}
+ }
- @Override
- public void beforeAttempt(
- final HandlerBeforeAttemptContext context) {
- ActiveAuditManagerS3A.this.beforeAttempt(context);
+ @Override
+ public void beforeUnmarshalling(Context.BeforeUnmarshalling context,
+ ExecutionAttributes executionAttributes) {
+ try {
+ extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .beforeUnmarshalling(context, executionAttributes);
+ } catch (AuditFailureException e) {
+ ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
+ throw e;
}
+ }
- @Override
- public HttpResponse beforeUnmarshalling(
- final Request<?> request,
- final HttpResponse httpResponse) {
- return ActiveAuditManagerS3A.this.beforeUnmarshalling(request,
- httpResponse);
+ @Override
+ public void afterUnmarshalling(Context.AfterUnmarshalling context,
+ ExecutionAttributes executionAttributes) {
+ try {
+ extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .afterUnmarshalling(context, executionAttributes);
+ } catch (AuditFailureException e) {
+ ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
+ throw e;
}
+ }
- @Override
- public void afterAttempt(
- final HandlerAfterAttemptContext context) {
- ActiveAuditManagerS3A.this.afterAttempt(context);
+ @Override
+ public SdkResponse modifyResponse(Context.ModifyResponse context,
+ ExecutionAttributes executionAttributes) {
+ try {
+ return extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .modifyResponse(context, executionAttributes);
+ } catch (AuditFailureException e) {
+ ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
+ throw e;
}
}
@@ -748,9 +778,8 @@ public final class ActiveAuditManagerS3A
* {@inheritDoc}
*/
@Override
- public <T extends AmazonWebServiceRequest> T requestCreated(
- final T request) {
- return span.requestCreated(request);
+ public void requestCreated(final SdkRequest.Builder builder) {
+ span.requestCreated(builder);
}
/**
@@ -774,79 +803,132 @@ public final class ActiveAuditManagerS3A
/**
* Forward to the inner span.
- * @param request request
- * @param <T> type of request
- * @return an updated request.
+ * {@inheritDoc}
*/
@Override
- public <T extends AmazonWebServiceRequest> T beforeExecution(
- final T request) {
- return span.beforeExecution(request);
+ public void beforeExecution(Context.BeforeExecution context,
+ ExecutionAttributes executionAttributes) {
+ span.beforeExecution(context, executionAttributes);
}
/**
* Forward to the inner span.
- * @param request request
- * @param response response.
+ * {@inheritDoc}
*/
@Override
- public void afterResponse(final Request<?> request,
- final Response<?> response) {
- span.afterResponse(request, response);
+ public void afterExecution(Context.AfterExecution context,
+ ExecutionAttributes executionAttributes) {
+ span.afterExecution(context, executionAttributes);
}
/**
* Forward to the inner span.
- * @param request request
- * @param response response.
- * @param exception exception raised.
+ * {@inheritDoc}
*/
@Override
- public void afterError(final Request<?> request,
- final Response<?> response,
- final Exception exception) {
- span.afterError(request, response, exception);
+ public void onExecutionFailure(Context.FailedExecution context,
+ ExecutionAttributes executionAttributes) {
+ span.onExecutionFailure(context, executionAttributes);
}
/**
* Forward to the inner span.
- * @param request request
- * @return request to marshall
+ * {@inheritDoc}
*/
@Override
- public AmazonWebServiceRequest beforeMarshalling(
- final AmazonWebServiceRequest request) {
- return span.beforeMarshalling(request);
+ public void beforeMarshalling(Context.BeforeMarshalling context,
+ ExecutionAttributes executionAttributes) {
+ span.beforeMarshalling(context, executionAttributes);
}
/**
* Forward to the inner span.
- * @param request request
+ * {@inheritDoc}
*/
@Override
- public void beforeRequest(final Request<?> request) {
- span.beforeRequest(request);
+ public SdkRequest modifyRequest(Context.ModifyRequest context,
+ ExecutionAttributes executionAttributes) {
+ return span.modifyRequest(context, executionAttributes);
}
/**
* Forward to the inner span.
- * @param context full context, including the request.
+ * {@inheritDoc}
*/
@Override
- public void beforeAttempt(
- final HandlerBeforeAttemptContext context) {
- span.beforeAttempt(context);
+ public void afterMarshalling(Context.AfterMarshalling context,
+ ExecutionAttributes executionAttributes) {
+ span.afterMarshalling(context, executionAttributes);
}
/**
* Forward to the inner span.
- *
- * @param context full context, including the request.
+ * {@inheritDoc}
*/
@Override
- public void afterAttempt(
- final HandlerAfterAttemptContext context) {
- span.afterAttempt(context);
+ public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context,
+ ExecutionAttributes executionAttributes) {
+ return span.modifyHttpRequest(context, executionAttributes);
+ }
+
+ /**
+ * Forward to the inner span.
+ * {@inheritDoc}
+ */
+ @Override
+ public void beforeTransmission(Context.BeforeTransmission context,
+ ExecutionAttributes executionAttributes) {
+ span.beforeTransmission(context, executionAttributes);
+ }
+
+ /**
+ * Forward to the inner span.
+ * {@inheritDoc}
+ */
+ @Override
+ public void afterTransmission(Context.AfterTransmission context,
+ ExecutionAttributes executionAttributes) {
+ span.afterTransmission(context, executionAttributes);
+ }
+
+ /**
+ * Forward to the inner span.
+ * {@inheritDoc}
+ */
+ @Override
+ public SdkHttpResponse modifyHttpResponse(Context.ModifyHttpResponse context,
+ ExecutionAttributes executionAttributes) {
+ return span.modifyHttpResponse(context, executionAttributes);
+ }
+
+ /**
+ * Forward to the inner span.
+ * {@inheritDoc}
+ */
+ @Override
+ public void beforeUnmarshalling(Context.BeforeUnmarshalling context,
+ ExecutionAttributes executionAttributes) {
+ span.beforeUnmarshalling(context, executionAttributes);
+ }
+
+ /**
+ * Forward to the inner span.
+ * {@inheritDoc}
+ */
+ @Override
+ public void afterUnmarshalling(Context.AfterUnmarshalling context,
+ ExecutionAttributes executionAttributes) {
+ span.afterUnmarshalling(context, executionAttributes);
+ }
+
+ /**
+ * Forward to the inner span.
+ * {@inheritDoc}
+ */
+ @Override
+ public SdkResponse modifyResponse(Context.ModifyResponse context,
+ ExecutionAttributes executionAttributes) {
+ return span.modifyResponse(context, executionAttributes);
}
@Override
@@ -859,5 +941,4 @@ public final class ActiveAuditManagerS3A
return sb.toString();
}
}
-
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java
index fcf2015487c4..3a2d9d7f823e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java
@@ -24,10 +24,14 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
-import com.amazonaws.AmazonWebServiceRequest;
-import com.amazonaws.services.s3.model.DeleteObjectRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.awscore.AwsExecutionAttribute;
+import software.amazon.awssdk.core.SdkRequest;
+import software.amazon.awssdk.core.interceptor.Context;
+import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
+import software.amazon.awssdk.http.SdkHttpRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -258,21 +262,22 @@ public class LoggingAuditor
/**
* Attach Range of data for GetObject Request.
- * @param request given get object request
+ * @param request the sdk request to be modified
+ * @param executionAttributes execution attributes for this request
*/
- private void attachRangeFromRequest(AmazonWebServiceRequest request) {
- if (request instanceof GetObjectRequest) {
- long[] rangeValue = ((GetObjectRequest) request).getRange();
- if (rangeValue == null || rangeValue.length == 0) {
- return;
- }
- if (rangeValue.length != 2) {
- WARN_INCORRECT_RANGE.warn("Expected range to contain 0 or 2 elements."
- + " Got {} elements. Ignoring.", rangeValue.length);
- return;
+ private void attachRangeFromRequest(SdkHttpRequest request,
+ ExecutionAttributes executionAttributes) {
+
+ String operationName = executionAttributes.getAttribute(AwsExecutionAttribute.OPERATION_NAME);
+
+ if (operationName != null && operationName.equals("GetObject")) {
+ if (request.headers() != null && request.headers().get("Range") != null) {
+ String[] rangeHeader = request.headers().get("Range").get(0).split("=");
+ // only set header if range unit is bytes
+ if (rangeHeader[0].equals("bytes")) {
+ referrer.set(AuditConstants.PARAM_RANGE, rangeHeader[1]);
+ }
}
- String combinedRangeValue = String.format("%d-%d", rangeValue[0], rangeValue[1]);
- referrer.set(AuditConstants.PARAM_RANGE, combinedRangeValue);
}
}
@@ -346,64 +351,78 @@ public class LoggingAuditor
referrer.set(key, value);
}
+
+
/**
- * Before execution, the logging auditor always builds
- * the referrer header, saves to the outer class
- * (where {@link #getLastHeader()} can retrieve it,
+ * Before transmitting a request, the logging auditor
+ * always builds the referrer header, saves to the outer
+ * class (where {@link #getLastHeader()} can retrieve it,
* and logs at debug.
* If configured to add the header to the S3 logs, it will
* be set as the HTTP referrer.
- * @param request request
- * @param <T> type of request.
- * @return the request with any extra headers.
+ * @param context The current state of the execution,
+ * including the SDK and current HTTP request.
+ * @param executionAttributes A mutable set of attributes scoped
+ * to one specific request/response
+ * cycle that can be used to give data
+ * to future lifecycle methods.
+ * @return The potentially-modified HTTP request that should be
+ * sent to the service. Must not be null.
*/
@Override
- public <T extends AmazonWebServiceRequest> T beforeExecution(
- final T request) {
+ public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context,
+ ExecutionAttributes executionAttributes) {
+ SdkHttpRequest httpRequest = context.httpRequest();
+ SdkRequest sdkRequest = context.request();
+
// attach range for GetObject requests
- attachRangeFromRequest(request);
+ attachRangeFromRequest(httpRequest, executionAttributes);
+
// for delete op, attach the number of files to delete
- attachDeleteKeySizeAttribute(request);
+ attachDeleteKeySizeAttribute(sdkRequest);
+
// build the referrer header
final String header = referrer.buildHttpReferrer();
// update the outer class's field.
setLastHeader(header);
if (headerEnabled) {
// add the referrer header
- request.putCustomRequestHeader(HEADER_REFERRER,
- header);
+ httpRequest = httpRequest.toBuilder()
+ .appendHeader(HEADER_REFERRER, header)
+ .build();
}
if (LOG.isDebugEnabled()) {
LOG.debug("[{}] {} Executing {} with {}; {}",
currentThreadID(),
getSpanId(),
getOperationName(),
- analyzer.analyze(request),
+ analyzer.analyze(context.request()),
header);
}
+
// now see if the request is actually a blocked multipart request
- if (!isMultipartUploadEnabled && isRequestMultipartIO(request)) {
+ if (!isMultipartUploadEnabled && isRequestMultipartIO(sdkRequest)) {
throw new AuditOperationRejectedException("Multipart IO request "
- + request + " rejected " + header);
+ + sdkRequest + " rejected " + header);
}
- return request;
+ return httpRequest;
}
/**
* For delete requests, attach delete key size as a referrer attribute.
*
* @param request the request object.
- * @param <T> type of the request.
*/
- private <T extends AmazonWebServiceRequest> void attachDeleteKeySizeAttribute(T request) {
+ private void attachDeleteKeySizeAttribute(SdkRequest request) {
+
if (request instanceof DeleteObjectsRequest) {
- int keySize = ((DeleteObjectsRequest) request).getKeys().size();
- this.set(DELETE_KEYS_SIZE, String.valueOf(keySize));
+ int keySize = ((DeleteObjectsRequest) request).delete().objects().size();
+ referrer.set(DELETE_KEYS_SIZE, String.valueOf(keySize));
} else if (request instanceof DeleteObjectRequest) {
- String key = ((DeleteObjectRequest) request).getKey();
+ String key = ((DeleteObjectRequest) request).key();
if (key != null && key.length() > 0) {
- this.set(DELETE_KEYS_SIZE, "1");
+ referrer.set(DELETE_KEYS_SIZE, "1");
}
}
}
@@ -460,15 +479,13 @@ public class LoggingAuditor
}
@Override
- public <T extends AmazonWebServiceRequest> T requestCreated(
- final T request) {
+ public void requestCreated(final SdkRequest.Builder builder) {
String error = "Creating a request outside an audit span "
- + analyzer.analyze(request);
+ + analyzer.analyze(builder.build());
LOG.info(error);
if (LOG.isDebugEnabled()) {
LOG.debug(error, new AuditFailureException("unaudited"));
}
- return request;
}
/**
@@ -476,20 +493,22 @@ public class LoggingAuditor
* increment the failure count.
* Some requests (e.g. copy part) are not expected in spans due
* to how they are executed; these do not trigger failures.
- * @param request request
- * @param <T> type of request
- * @return an updated request.
- * @throws AuditFailureException if failure is enabled.
+ * @param context The current state of the execution, including
+ * the unmodified SDK request from the service
+ * client call.
+ * @param executionAttributes A mutable set of attributes scoped
+ * to one specific request/response
+ * cycle that can be used to give data
+ * to future lifecycle methods.
*/
@Override
- public <T extends AmazonWebServiceRequest> T beforeExecution(
- final T request) {
-
+ public void beforeExecution(Context.BeforeExecution context,
+ ExecutionAttributes executionAttributes) {
String error = "executing a request outside an audit span "
- + analyzer.analyze(request);
+ + analyzer.analyze(context.request());
final String unaudited = getSpanId() + " "
+ UNAUDITED_OPERATION + " " + error;
- if (isRequestNotAlwaysInSpan(request)) {
+ if (isRequestNotAlwaysInSpan(context.request())) {
// can get by auditing during a copy, so don't overreact
LOG.debug(unaudited);
} else {
@@ -500,7 +519,7 @@ public class LoggingAuditor
}
}
// now hand off to the superclass for its normal preparation
- return super.beforeExecution(request);
+ super.beforeExecution(context, executionAttributes);
}
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditManagerS3A.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditManagerS3A.java
index d1ebd922e073..e58c906460da 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditManagerS3A.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditManagerS3A.java
@@ -24,9 +24,8 @@ import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
-import com.amazonaws.handlers.RequestHandler2;
-import com.amazonaws.services.s3.transfer.Transfer;
-import com.amazonaws.services.s3.transfer.internal.TransferStateChangeListener;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
+import software.amazon.awssdk.transfer.s3.progress.TransferListener;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
@@ -39,6 +38,7 @@ import org.apache.hadoop.fs.s3a.audit.OperationAuditor;
import org.apache.hadoop.fs.s3a.audit.OperationAuditorOptions;
import org.apache.hadoop.service.CompositeService;
+
import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.iostatisticsStore;
/**
@@ -121,17 +121,13 @@ public class NoopAuditManagerS3A extends CompositeService
}
@Override
- public List<RequestHandler2> createRequestHandlers() throws IOException {
+ public List<ExecutionInterceptor> createExecutionInterceptors() throws IOException {
return new ArrayList<>();
}
@Override
- public TransferStateChangeListener createStateChangeListener() {
- return new TransferStateChangeListener() {
- public void transferStateChanged(final Transfer transfer,
- final Transfer.TransferState state) {
- }
- };
+ public TransferListener createTransferListener() {
+ return new TransferListener() {};
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/S3AInternalAuditConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/S3AInternalAuditConstants.java
index f82e3d7f1e5e..c170a2be6611 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/S3AInternalAuditConstants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/S3AInternalAuditConstants.java
@@ -18,10 +18,10 @@
package org.apache.hadoop.fs.s3a.audit.impl;
-import com.amazonaws.handlers.HandlerContextKey;
+import software.amazon.awssdk.core.interceptor.ExecutionAttribute;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.fs.s3a.audit.AWSAuditEventCallbacks;
+import org.apache.hadoop.fs.s3a.audit.AuditSpanS3A;
/**
* Internal constants; not intended for public use, or
@@ -34,11 +34,11 @@ public final class S3AInternalAuditConstants {
}
/**
- * Handler key for audit span callbacks.
- * This is used to bind the handler in the AWS code.
+ * Exceution attribute for audit span callbacks.
+ * This is used to retrieve the span in the AWS code.
*/
- public static final HandlerContextKey<AWSAuditEventCallbacks>
- AUDIT_SPAN_HANDLER_CONTEXT =
- new HandlerContextKey<>(
- "org.apache.hadoop.fs.s3a.audit.AWSAuditEventCallbacks");
+ public static final ExecutionAttribute<AuditSpanS3A>
+ AUDIT_SPAN_EXECUTION_ATTRIBUTE =
+ new ExecutionAttribute<>(
+ "org.apache.hadoop.fs.s3a.audit.AuditSpanS3A");
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java
index 1815285738b0..4754427a4b11 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.fs.s3a.auth;
import javax.annotation.Nullable;
import java.net.URI;
... 15606 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 16/17: HADOOP-18996. S3A to provide full support for S3 Express One Zone (#6308)
Posted by ah...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ahmar pushed a commit to branch feature-HADOOP-18073-sdk-v2-upgrade-3.6-aws
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit b330e9e34c076b87193c8833f1dd921278fe92dd
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Fri Dec 1 14:16:33 2023 +0000
HADOOP-18996. S3A to provide full support for S3 Express One Zone (#6308)
This adds borad support for Amazon S3 Express One Zone to the S3A connector,
particularly resilience of other parts of the codebase to LIST operations returning
paths under which only in-progress uploads are taking place.
hadoop-common and hadoop-mapreduce treewalking routines all cope with this;
distcp is left alone.
There are still some outstanding followup issues, and we expect more to surface
with extended use.
Contains HADOOP-18955. AWS SDK v2: add path capability probe "fs.s3a.capability.aws.v2
* lets us probe for AWS SDK version
* bucket-info reports it
Contains HADOOP-18961 S3A: add s3guard command "bucket"
hadoop s3guard bucket -create -region us-west-2 -zone usw2-az2 \
s3a://stevel--usw2-az2--x-s3/
* requires -zone if bucket is zonal
* rejects it if not
* rejects zonal bucket suffixes if endpoint is not aws (safety feature)
* imperfect, but a functional starting point.
New path capability "fs.s3a.capability.zonal.storage"
* Used in tests to determine whether pending uploads manifest paths
* cli tests can probe for this
* bucket-info reports it
* some tests disable/change assertions as appropriate
----
Shell commands fail on S3Express buckets if pending uploads.
New path capability in hadoop-common
"fs.capability.directory.listing.inconsistent"
1. S3AFS returns true on a S3 Express bucket
2. FileUtil.maybeIgnoreMissingDirectory(fs, path, fnfe)
decides whether to swallow the exception or not.
3. This is used in: Shell, FileInputFormat, LocatedFileStatusFetcher
Fixes with tests
* fs -ls -R
* fs -du
* fs -df
* fs -find
* S3AFS.getContentSummary() (maybe...should discuss)
* mapred LocatedFileStatusFetcher
* Globber, HADOOP-15478 already fixed that when dealing with
S3 inconsistencies
* FileInputFormat
S3Express CreateSession request is permitted outside audit spans.
S3 Bulk Delete calls request the store to return the list of deleted objects
if RequestFactoryImpl is set to trace.
log4j.logger.org.apache.hadoop.fs.s3a.impl.RequestFactoryImpl=TRACE
Test Changes
* ITestS3AMiscOperations removes all tests which require unencrypted
buckets. AWS S3 defaults to SSE-S3 everywhere.
* ITestBucketTool to test new tool without actually creating new
buckets.
* S3ATestUtils add methods to skip test suites/cases if store is/is not
S3Express
* Cutting down on "is this a S3Express bucket" logic to trailing --x-s3 string
and not worrying about AZ naming logic. commented out relevant tests.
* ITestTreewalkProblems validated against standard and S3Express stores
Outstanding
* Distcp: tests show it fails. Proposed: release notes.
---
x-amz-checksum header not found when signing S3Express messages
This modifies the custom signer in ITestCustomSigner to be a subclass
of AwsS3V4Signer with a goal of preventing signing problems with
S3 Express stores.
----
RemoteFileChanged renaming multipart file
Maps 412 status code to RemoteFileChangedException
Modifies huge file tests
-Adds a check on etag match for stat vs list
-ITestS3AHugeFilesByteBufferBlocks renames parent dirs, rather than
files, to replicate distcp better.
----
S3Express custom Signing cannot handle bulk delete
Copy custom signer into production JAR, so enable downstream testing
Extend ITestCustomSigner to cover more filesystem operations
- PUT
- POST
- COPY
- LIST
- Bulk delete through delete() and rename()
- list + abort multipart uploads
Suite is parameterized on bulk delete enabled/disabled.
To use the new signer for a full test run:
<property>
<name>fs.s3a.custom.signers</name>
<value>CustomSdkSigner:org.apache.hadoop.fs.s3a.auth.CustomSdkSigner</value>
</property>
<property>
<name>fs.s3a.s3.signing-algorithm</name>
<value>CustomSdkSigner</value>
</property>
---
.../apache/hadoop/fs/CommonPathCapabilities.java | 11 +
.../main/java/org/apache/hadoop/fs/FileUtil.java | 29 ++
.../java/org/apache/hadoop/fs/shell/Command.java | 17 +-
.../org/apache/hadoop/fs/FileContextURIBase.java | 32 +-
.../AbstractContractRootDirectoryTest.java | 27 +-
.../hadoop/fs/contract/ContractTestUtils.java | 66 ++-
.../hadoop/mapred/LocatedFileStatusFetcher.java | 34 +-
.../mapreduce/lib/input/FileInputFormat.java | 30 +-
.../java/org/apache/hadoop/fs/s3a/Constants.java | 18 +-
.../java/org/apache/hadoop/fs/s3a/Listing.java | 2 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 112 ++++-
.../org/apache/hadoop/fs/s3a/S3AInternals.java | 10 +
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 16 +-
.../hadoop/fs/s3a/audit/AWSRequestAnalyzer.java | 4 +-
.../apache/hadoop/fs/s3a/auth/CustomSdkSigner.java | 178 ++++++++
.../apache/hadoop/fs/s3a/impl/DeleteOperation.java | 20 +-
.../hadoop/fs/s3a/impl/InternalConstants.java | 47 ++-
.../apache/hadoop/fs/s3a/impl/NetworkBinding.java | 11 +
.../hadoop/fs/s3a/impl/RequestFactoryImpl.java | 2 +-
.../hadoop/fs/s3a/impl/S3ExpressStorage.java | 76 ++++
.../apache/hadoop/fs/s3a/s3guard/S3GuardTool.java | 52 ++-
.../org/apache/hadoop/fs/s3a/tools/BucketTool.java | 304 ++++++++++++++
.../org/apache/hadoop/fs/s3a/tools/MarkerTool.java | 8 +-
.../markdown/tools/hadoop-aws/aws_sdk_upgrade.md | 10 +
.../tools/hadoop-aws/third_party_stores.md | 11 +-
.../hadoop/fs/s3a/AbstractTestS3AEncryption.java | 1 +
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 2 +-
.../hadoop/fs/s3a/ITestS3AEncryptionSSEC.java | 2 +-
.../hadoop/fs/s3a/ITestS3AMiscOperations.java | 77 ----
.../apache/hadoop/fs/s3a/MultipartTestUtils.java | 55 ++-
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 76 ++++
.../hadoop/fs/s3a/auth/ITestCustomSigner.java | 124 ++++--
.../fs/s3a/commit/ITestCommitOperations.java | 6 +-
.../hadoop/fs/s3a/impl/ITestTreewalkProblems.java | 463 +++++++++++++++++++++
.../ITestUploadPurgeOnDirectoryOperations.java | 22 +-
.../hadoop/fs/s3a/impl/TestS3ExpressStorage.java | 63 +++
.../performance/ITestDirectoryMarkerListing.java | 23 +-
.../s3a/s3guard/AbstractS3GuardToolTestBase.java | 6 +-
.../fs/s3a/scale/AbstractSTestS3AHugeFiles.java | 72 +++-
.../scale/ITestS3AHugeFilesByteBufferBlocks.java | 29 ++
.../hadoop/fs/s3a/tools/ITestBucketTool.java | 183 ++++++++
.../hadoop-aws/src/test/resources/core-site.xml | 2 +-
.../hadoop-aws/src/test/resources/log4j.properties | 4 +
.../apache/hadoop/tools/util/DistCpTestUtils.java | 10 +-
44 files changed, 2032 insertions(+), 315 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java
index b898fde43cc8..9ec07cbe966e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java
@@ -170,4 +170,15 @@ public final class CommonPathCapabilities {
*/
public static final String LEASE_RECOVERABLE = "fs.capability.lease.recoverable";
+ /**
+ * Is this a store where parent directory listings are potentially inconsistent with
+ * direct list/getFileStatus calls?
+ * This can happen with Amazon S3 Express One Zone Storage when there are pending
+ * uploads under a path.
+ * Application code can use this flag to decide whether or not to treat
+ * FileNotFoundExceptions on treewalk as errors or something to downgrade.
+ * Value: {@value}.
+ */
+ public static final String DIRECTORY_LISTING_INCONSISTENT =
+ "fs.capability.directory.listing.inconsistent";
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index 933f56927741..79041e0d7b94 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -76,6 +76,7 @@ import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.fs.CommonPathCapabilities.DIRECTORY_LISTING_INCONSISTENT;
import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_LENGTH;
import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE;
@@ -2058,4 +2059,32 @@ public class FileUtil {
final Options.Rename... options) throws IOException {
srcFs.rename(src, dst, options);
}
+
+ /**
+ * Method to call after a FNFE has been raised on a treewalk, so as to
+ * decide whether to throw the exception (default), or, if the FS
+ * supports inconsistent directory listings, to log and ignore it.
+ * If this returns then the caller should ignore the failure and continue.
+ * @param fs filesystem
+ * @param path path
+ * @param e exception caught
+ * @throws FileNotFoundException the exception passed in, if rethrown.
+ */
+ public static void maybeIgnoreMissingDirectory(FileSystem fs,
+ Path path,
+ FileNotFoundException e) throws FileNotFoundException {
+ final boolean b;
+ try {
+ b = !fs.hasPathCapability(path, DIRECTORY_LISTING_INCONSISTENT);
+ } catch (IOException ex) {
+ // something went wrong; rethrow the existing exception
+ e.addSuppressed(ex);
+ throw e;
+ }
+ if (b) {
+ throw e;
+ }
+ LOG.info("Ignoring missing directory {}", path);
+ LOG.debug("Directory missing", e);
+ }
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
index 7858238ee71f..c4ce115def68 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.fs.FileUtil.maybeIgnoreMissingDirectory;
import static org.apache.hadoop.util.functional.RemoteIterators.cleanupRemoteIterator;
/**
@@ -448,12 +449,16 @@ abstract public class Command extends Configured {
protected void recursePath(PathData item) throws IOException {
try {
depth++;
- if (isSorted()) {
- // use the non-iterative method for listing because explicit sorting is
- // required. Iterators not guaranteed to return sorted elements
- processPaths(item, item.getDirectoryContents());
- } else {
- processPaths(item, item.getDirectoryContentsIterator());
+ try {
+ if (isSorted()) {
+ // use the non-iterative method for listing because explicit sorting is
+ // required. Iterators not guaranteed to return sorted elements
+ processPaths(item, item.getDirectoryContents());
+ } else {
+ processPaths(item, item.getDirectoryContentsIterator());
+ }
+ } catch (FileNotFoundException e) {
+ maybeIgnoreMissingDirectory(item.fs, item.path, e);
}
} finally {
depth--;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextURIBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextURIBase.java
index abb6d4f90159..fad1e3774ad2 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextURIBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextURIBase.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.test.LambdaTestUtils;
import org.apache.hadoop.util.Shell;
import org.junit.After;
import org.junit.Before;
@@ -129,7 +130,7 @@ public abstract class FileContextURIBase {
}
@Test
- public void testCreateExistingFile() throws IOException {
+ public void testCreateExistingFile() throws Exception {
String fileName = "testCreateExistingFile";
Path testPath = qualifiedPath(fileName, fc2);
@@ -140,15 +141,11 @@ public abstract class FileContextURIBase {
createFile(fc1, testPath);
// Create same file with fc1
- try {
- createFile(fc2, testPath);
- Assert.fail("Create existing file should throw an IOException.");
- } catch (IOException e) {
- // expected
- }
+ LambdaTestUtils.intercept(IOException.class, () ->
+ createFile(fc2, testPath));
// Ensure fc2 has the created file
- Assert.assertTrue(exists(fc2, testPath));
+ fc2.getFileStatus(testPath);
}
@Test
@@ -167,7 +164,7 @@ public abstract class FileContextURIBase {
Assert.assertTrue(isDir(fc2, testPath.getParent()));
Assert.assertEquals("testCreateFileInNonExistingDirectory",
testPath.getParent().getName());
- Assert.assertTrue(exists(fc2, testPath));
+ fc2.getFileStatus(testPath);
}
@@ -216,10 +213,11 @@ public abstract class FileContextURIBase {
// TestCase - Create multiple directories
String dirNames[] = {
"createTest/testDir", "createTest/test Dir",
- "deleteTest/test*Dir", "deleteTest/test#Dir",
- "deleteTest/test1234", "deleteTest/test_DIr",
- "deleteTest/1234Test", "deleteTest/test)Dir",
- "deleteTest/()&^%$#@!~_+}{><?", " ", "^ " };
+ "createTest/test*Dir", "createTest/test#Dir",
+ "createTest/test1234", "createTest/test_DIr",
+ "createTest/1234Test", "createTest/test)Dir",
+ "createTest/()&^%$#@!~_+}{><?",
+ "createTest/ ", "createTest/^ " };
for (String f : dirNames) {
if (!isTestableFileNameOnPlatform(f)) {
@@ -237,6 +235,10 @@ public abstract class FileContextURIBase {
Assert.assertTrue(exists(fc2, testPath));
Assert.assertTrue(isDir(fc2, testPath));
}
+ // delete the parent directory and verify that the dir no longer exists
+ final Path parent = qualifiedPath("createTest", fc2);
+ fc2.delete(parent, true);
+ Assert.assertFalse(exists(fc2, parent));
}
@@ -392,7 +394,9 @@ public abstract class FileContextURIBase {
"deleteTest/test*Dir", "deleteTest/test#Dir",
"deleteTest/test1234", "deleteTest/1234Test",
"deleteTest/test)Dir", "deleteTest/test_DIr",
- "deleteTest/()&^%$#@!~_+}{><?", " ", "^ " };
+ "deleteTest/()&^%$#@!~_+}{><?",
+ "deleteTest/ ",
+ "deleteTest/^ " };
for (String f : dirNames) {
if (!isTestableFileNameOnPlatform(f)) {
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRootDirectoryTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRootDirectoryTest.java
index 924ebd0065d9..2988ebd215b8 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRootDirectoryTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRootDirectoryTest.java
@@ -29,7 +29,6 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
-import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
@@ -100,22 +99,18 @@ public abstract class AbstractContractRootDirectoryTest extends AbstractFSContra
final FileStatus[] originalChildren = listChildren(fs, root);
LambdaTestUtils.eventually(
OBJECTSTORE_RETRY_TIMEOUT,
- new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- FileStatus[] deleted = deleteChildren(fs, root, true);
- FileStatus[] children = listChildren(fs, root);
- if (children.length > 0) {
- fail(String.format(
- "After %d attempts: listing after rm /* not empty"
- + "\n%s\n%s\n%s",
- iterations.incrementAndGet(),
- dumpStats("final", children),
+ () -> {
+ iterations.incrementAndGet();
+ FileStatus[] deleted = deleteChildren(fs, root, true);
+ FileStatus[] children = listChildren(fs, root);
+ Assertions.assertThat(children)
+ .describedAs("After %d attempts: listing after rm /* not empty"
+ + "\ndeleted: %s\n: original %s",
+ iterations.get(),
dumpStats("deleted", deleted),
- dumpStats("original", originalChildren)));
- }
- return null;
- }
+ dumpStats("original", originalChildren))
+ .isEmpty();
+ return null;
},
new LambdaTestUtils.ProportionalRetryInterval(50, 1000));
// then try to delete the empty one
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
index b61abddd4342..13c86af08c4f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileRange;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathCapabilities;
@@ -34,6 +35,7 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.functional.RemoteIterators;
import org.apache.hadoop.util.functional.FutureIO;
+import org.assertj.core.api.Assertions;
import org.junit.Assert;
import org.junit.AssumptionViolatedException;
import org.slf4j.Logger;
@@ -61,6 +63,7 @@ import java.util.concurrent.TimeoutException;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
+import static org.apache.hadoop.util.functional.RemoteIterators.foreach;
/**
* Utilities used across test cases.
@@ -1506,19 +1509,39 @@ public class ContractTestUtils extends Assert {
*/
public static TreeScanResults treeWalk(FileSystem fs, Path path)
throws IOException {
- TreeScanResults dirsAndFiles = new TreeScanResults();
+ return treeWalk(fs, fs.getFileStatus(path), new TreeScanResults());
+ }
- FileStatus[] statuses = fs.listStatus(path);
- for (FileStatus status : statuses) {
- LOG.info("{}{}", status.getPath(), status.isDirectory() ? "*" : "");
- }
- for (FileStatus status : statuses) {
- dirsAndFiles.add(status);
- if (status.isDirectory()) {
- dirsAndFiles.add(treeWalk(fs, status.getPath()));
- }
+ /**
+ * Recursively list all entries, with a depth first traversal of the
+ * directory tree.
+ * @param dir status of the dir to scan
+ * @return the scan results
+ * @throws IOException IO problems
+ * @throws FileNotFoundException if the dir is not found and this FS does not
+ * have the listing inconsistency path capability/flag.
+ */
+ private static TreeScanResults treeWalk(FileSystem fs,
+ FileStatus dir,
+ TreeScanResults results) throws IOException {
+
+ Path path = dir.getPath();
+
+ try {
+ foreach(fs.listStatusIterator(path), (status) -> {
+ LOG.info("{}{}", status.getPath(), status.isDirectory() ? "*" : "");
+ if (status.isDirectory()) {
+ treeWalk(fs, status, results);
+ } else {
+ results.add(status);
+ }
+ });
+ // and ourselves
+ results.add(dir);
+ } catch (FileNotFoundException fnfe) {
+ FileUtil.maybeIgnoreMissingDirectory(fs, path, fnfe);
}
- return dirsAndFiles;
+ return results;
}
/**
@@ -1916,17 +1939,16 @@ public class ContractTestUtils extends Assert {
public void assertFieldsEquivalent(String fieldname,
TreeScanResults that,
List<Path> ours, List<Path> theirs) {
- String ourList = pathsToString(ours);
- String theirList = pathsToString(theirs);
- assertFalse("Duplicate " + fieldname + " in " + this
- +": " + ourList,
- containsDuplicates(ours));
- assertFalse("Duplicate " + fieldname + " in other " + that
- + ": " + theirList,
- containsDuplicates(theirs));
- assertTrue(fieldname + " mismatch: between " + ourList
- + " and " + theirList,
- collectionsEquivalent(ours, theirs));
+ Assertions.assertThat(ours).
+ describedAs("list of %s", fieldname)
+ .doesNotHaveDuplicates();
+ Assertions.assertThat(theirs).
+ describedAs("list of %s in %s", fieldname, that)
+ .doesNotHaveDuplicates();
+ Assertions.assertThat(ours)
+ .describedAs("Elements of %s", fieldname)
+ .containsExactlyInAnyOrderElementsOf(theirs);
+
}
public List<Path> getFiles() {
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LocatedFileStatusFetcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LocatedFileStatusFetcher.java
index 5810698607c4..fcaa02a65586 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LocatedFileStatusFetcher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LocatedFileStatusFetcher.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.mapred;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
@@ -56,6 +57,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
+import static org.apache.hadoop.fs.FileUtil.maybeIgnoreMissingDirectory;
import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.retrieveIOStatistics;
import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.snapshotIOStatistics;
@@ -304,23 +306,27 @@ public class LocatedFileStatusFetcher implements IOStatisticsSource {
Result result = new Result();
result.fs = fs;
LOG.debug("ProcessInputDirCallable {}", fileStatus);
- if (fileStatus.isDirectory()) {
- RemoteIterator<LocatedFileStatus> iter = fs
- .listLocatedStatus(fileStatus.getPath());
- while (iter.hasNext()) {
- LocatedFileStatus stat = iter.next();
- if (inputFilter.accept(stat.getPath())) {
- if (recursive && stat.isDirectory()) {
- result.dirsNeedingRecursiveCalls.add(stat);
- } else {
- result.locatedFileStatuses.add(stat);
+ try {
+ if (fileStatus.isDirectory()) {
+ RemoteIterator<LocatedFileStatus> iter = fs
+ .listLocatedStatus(fileStatus.getPath());
+ while (iter.hasNext()) {
+ LocatedFileStatus stat = iter.next();
+ if (inputFilter.accept(stat.getPath())) {
+ if (recursive && stat.isDirectory()) {
+ result.dirsNeedingRecursiveCalls.add(stat);
+ } else {
+ result.locatedFileStatuses.add(stat);
+ }
}
}
+ // aggregate any stats
+ result.stats = retrieveIOStatistics(iter);
+ } else {
+ result.locatedFileStatuses.add(fileStatus);
}
- // aggregate any stats
- result.stats = retrieveIOStatistics(iter);
- } else {
- result.locatedFileStatuses.add(fileStatus);
+ } catch (FileNotFoundException e) {
+ maybeIgnoreMissingDirectory(fs, fileStatus.getPath(), e);
}
return result;
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
index 2d88d13a1546..a7c837628857 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.mapreduce.lib.input;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.util.ArrayList;
@@ -48,8 +49,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import static org.apache.hadoop.fs.FileUtil.maybeIgnoreMissingDirectory;
-/**
+/**
* A base class for file-based {@link InputFormat}s.
*
* <p><code>FileInputFormat</code> is the base class for all file-based
@@ -357,16 +359,26 @@ public abstract class FileInputFormat<K, V> extends InputFormat<K, V> {
protected void addInputPathRecursively(List<FileStatus> result,
FileSystem fs, Path path, PathFilter inputFilter)
throws IOException {
- RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(path);
- while (iter.hasNext()) {
- LocatedFileStatus stat = iter.next();
- if (inputFilter.accept(stat.getPath())) {
- if (stat.isDirectory()) {
- addInputPathRecursively(result, fs, stat.getPath(), inputFilter);
- } else {
- result.add(stat);
+ // FNFE exceptions are caught whether raised in the list call,
+ // or in the hasNext() or next() calls, where async reporting
+ // may take place.
+ try {
+ RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(path);
+ while (iter.hasNext()) {
+ LocatedFileStatus stat = iter.next();
+ if (inputFilter.accept(stat.getPath())) {
+ if (stat.isDirectory()) {
+ addInputPathRecursively(result, fs, stat.getPath(), inputFilter);
+ } else {
+ result.add(stat);
+ }
}
}
+ } catch (FileNotFoundException e) {
+ // unless the store is capabile of list inconsistencies, rethrow.
+ // because this is recursive, the caller method may also end up catching
+ // and rethrowing, which is slighly inefficient but harmless.
+ maybeIgnoreMissingDirectory(fs, path, e);
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 140ea53ceb6f..b0369ac735f5 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -1462,9 +1462,18 @@ public final class Constants {
/**
* Stream supports multipart uploads to the given path.
*/
- public static final String STORE_CAPABILITY_DIRECTORY_MARKER_MULTIPART_UPLOAD_ENABLED =
+ public static final String STORE_CAPABILITY_MULTIPART_UPLOAD_ENABLED =
"fs.s3a.capability.multipart.uploads.enabled";
+ /**
+ * Stream supports multipart uploads to the given path.
+ * This name is wrong, but it has shipped so must be
+ * retained.
+ */
+ @Deprecated
+ public static final String STORE_CAPABILITY_DIRECTORY_MARKER_MULTIPART_UPLOAD_ENABLED
+ = STORE_CAPABILITY_MULTIPART_UPLOAD_ENABLED;
+
/**
* Prefetch max blocks count config.
* Value = {@value}
@@ -1509,4 +1518,11 @@ public final class Constants {
* Value: {@value}.
*/
public static final boolean OPTIMIZED_COPY_FROM_LOCAL_DEFAULT = true;
+
+ /**
+ * Is this a v2 SDK build? value {@value}.
+ */
+ public static final String STORE_CAPABILITY_AWS_V2 =
+ "fs.s3a.capability.aws.v2";
+
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
index 490deaaab04d..e0868a2e1308 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
@@ -259,7 +259,7 @@ public class Listing extends AbstractStoreOperation {
}
S3ListRequest request = createListObjectsRequest(key, "/", span);
- LOG.debug("listStatus: doing listObjects for directory {}", key);
+ LOG.debug("listStatus: doing listObjects for directory \"{}\"", key);
// return the results obtained from s3.
return createFileStatusListingIterator(
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 3e995fb1ddb4..f02c8495914b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -220,6 +220,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_
import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
+import static org.apache.hadoop.fs.CommonPathCapabilities.DIRECTORY_LISTING_INCONSISTENT;
import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs;
import static org.apache.hadoop.fs.s3a.Constants.*;
import static org.apache.hadoop.fs.s3a.Invoker.*;
@@ -250,6 +251,8 @@ import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_404_NOT_FOUND;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.UPLOAD_PART_COUNT_LIMIT;
import static org.apache.hadoop.fs.s3a.impl.NetworkBinding.fixBucketRegion;
import static org.apache.hadoop.fs.s3a.impl.NetworkBinding.logDnsLookup;
+import static org.apache.hadoop.fs.s3a.impl.S3ExpressStorage.STORE_CAPABILITY_S3_EXPRESS_STORAGE;
+import static org.apache.hadoop.fs.s3a.impl.S3ExpressStorage.isS3ExpressStore;
import static org.apache.hadoop.fs.s3a.s3guard.S3Guard.checkNoS3Guard;
import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.logIOStatisticsAtLevel;
import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_CONTINUE_LIST_REQUEST;
@@ -289,11 +292,16 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
public static final int DEFAULT_BLOCKSIZE = 32 * 1024 * 1024;
private URI uri;
+
private Path workingDir;
+
private String username;
+
private S3Client s3Client;
+
/** Async client is used for transfer manager and s3 select. */
private S3AsyncClient s3AsyncClient;
+
// initial callback policy is fail-once; it's there just to assist
// some mock tests and other codepaths trying to call the low level
// APIs on an uninitialized filesystem.
@@ -470,6 +478,21 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
*/
private boolean optimizedCopyFromLocal;
+ /**
+ * Is this an S3 Express store?
+ */
+ private boolean s3ExpressStore;
+
+ /**
+ * Store endpoint from configuration info or access point ARN.
+ */
+ private String endpoint;
+
+ /**
+ * Region from configuration info or access point ARN.
+ */
+ private String configuredRegion;
+
/** Add any deprecated keys. */
@SuppressWarnings("deprecation")
private static void addDeprecatedKeys() {
@@ -581,9 +604,23 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
//check but do not store the block size
longBytesOption(conf, FS_S3A_BLOCK_SIZE, DEFAULT_BLOCKSIZE, 1);
enableMultiObjectsDelete = conf.getBoolean(ENABLE_MULTI_DELETE, true);
- // should the delete also purge uploads.
+
+ // determine and cache the endpoints
+ endpoint = accessPoint == null
+ ? conf.getTrimmed(ENDPOINT, DEFAULT_ENDPOINT)
+ : accessPoint.getEndpoint();
+
+ configuredRegion = accessPoint == null
+ ? conf.getTrimmed(AWS_REGION)
+ : accessPoint.getRegion();
+
+ // is this an S3Express store?
+ s3ExpressStore = isS3ExpressStore(bucket, endpoint);
+
+ // should the delete also purge uploads?
+ // happens if explicitly enabled, or if the store is S3Express storage.
dirOperationsPurgeUploads = conf.getBoolean(DIRECTORY_OPERATIONS_PURGE_UPLOADS,
- DIRECTORY_OPERATIONS_PURGE_UPLOADS_DEFAULT);
+ s3ExpressStore);
this.isMultipartUploadEnabled = conf.getBoolean(MULTIPART_UPLOADS_ENABLED,
DEFAULT_MULTIPART_UPLOAD_ENABLED);
@@ -996,14 +1033,6 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
S3_CLIENT_FACTORY_IMPL, DEFAULT_S3_CLIENT_FACTORY_IMPL,
S3ClientFactory.class);
- String endpoint = accessPoint == null
- ? conf.getTrimmed(ENDPOINT, DEFAULT_ENDPOINT)
- : accessPoint.getEndpoint();
-
- String configuredRegion = accessPoint == null
- ? conf.getTrimmed(AWS_REGION)
- : accessPoint.getRegion();
-
S3ClientFactory.S3ClientCreationParameters parameters =
new S3ClientFactory.S3ClientCreationParameters()
.withCredentialSet(credentials)
@@ -1445,7 +1474,19 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
public boolean isMultipartCopyEnabled() {
return S3AFileSystem.this.isMultipartUploadEnabled;
}
- }
+
+ @Override
+ public long abortMultipartUploads(final Path path) throws IOException {
+ final String prefix = pathToKey(path);
+ try (AuditSpan span = createSpan("object_multipart_bulk_abort", prefix, null)) {
+ return S3AFileSystem.this.abortMultipartUploadsUnderPrefix(
+ createStoreContext(),
+ span,
+ prefix);
+ }
+ }
+
+ } // end S3AInternals
/**
* Get the input policy for this FS instance.
@@ -2483,19 +2524,34 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
@Retries.RetryTranslated
public long abortMultipartUploadsUnderPrefix(String prefix)
throws IOException {
- getAuditSpan().activate();
- // this deactivates the audit span somehow
- final RemoteIterator<MultipartUpload> uploads =
- S3AFileSystem.this.listUploadsUnderPrefix(storeContext, prefix);
- // so reactivate it.
- getAuditSpan().activate();
- return foreach(uploads, upload ->
- invoker.retry("Aborting multipart commit", upload.key(), true, () ->
- S3AFileSystem.this.abortMultipartUpload(upload)));
+ return S3AFileSystem.this.abortMultipartUploadsUnderPrefix(storeContext, auditSpan, prefix);
}
} // end OperationCallbacksImpl
+ /**
+ * Abort multipart uploads under a prefix.
+ * @param storeContext store context
+ * @param span span for the operations
+ * @param prefix prefix for uploads to abort
+ * @return a count of aborts
+ * @throws IOException trouble; FileNotFoundExceptions are swallowed.
+ */
+ private long abortMultipartUploadsUnderPrefix(StoreContext storeContext,
+ AuditSpan span,
+ String prefix) throws IOException {
+
+ span.activate();
+ // this deactivates the audit span somehow
+ final RemoteIterator<MultipartUpload> uploads =
+ listUploadsUnderPrefix(storeContext, prefix);
+ // so reactivate it.
+ span.activate();
+ return foreach(uploads, upload ->
+ invoker.retry("Aborting multipart commit", upload.key(), true, () ->
+ abortMultipartUpload(upload)));
+ }
+
/**
* Callbacks from {@link Listing}.
* Auditing: the listing object is long-lived; the audit span
@@ -3327,7 +3383,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
LOG.debug("Initiating delete operation for {} objects",
keysToDelete.size());
for (ObjectIdentifier objectIdentifier : keysToDelete) {
- LOG.debug(" {} {}", objectIdentifier.key(),
+ LOG.debug(" \"{}\" {}", objectIdentifier.key(),
objectIdentifier.versionId() != null ? objectIdentifier.versionId() : "");
}
}
@@ -5400,10 +5456,24 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
case STORE_CAPABILITY_DIRECTORY_MARKER_AWARE:
return true;
+ // multi object delete flag
+ case ENABLE_MULTI_DELETE:
+ return enableMultiObjectsDelete;
+
// Do directory operations purge uploads.
case DIRECTORY_OPERATIONS_PURGE_UPLOADS:
return dirOperationsPurgeUploads;
+ // this is a v2 sdk release.
+ case STORE_CAPABILITY_AWS_V2:
+ return true;
+
+ // is this store S3 Express?
+ // if so, note that directory listings may be inconsistent
+ case STORE_CAPABILITY_S3_EXPRESS_STORAGE:
+ case DIRECTORY_LISTING_INCONSISTENT:
+ return s3ExpressStore;
+
// etags are avaialable in listings, but they
// are not consistent across renames.
// therefore, only availability is declared
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java
index 18d6c1af586f..b4116068565c 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java
@@ -121,4 +121,14 @@ public interface S3AInternals {
* @return true if the transfer manager is used to copy files.
*/
boolean isMultipartCopyEnabled();
+
+ /**
+ * Abort multipart uploads under a path.
+ * @param path path to abort uploads under.
+ * @return a count of aborts
+ * @throws IOException trouble; FileNotFoundExceptions are swallowed.
+ */
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ long abortMultipartUploads(Path path) throws IOException;
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index 94fa2d35fc6f..703f64bfb6c2 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -270,6 +270,11 @@ public final class S3AUtils {
}
break;
+ // Caused by duplicate create bucket call.
+ case SC_409_CONFLICT:
+ ioe = new AWSBadRequestException(message, ase);
+ break;
+
// this also surfaces sometimes and is considered to
// be ~ a not found exception.
case SC_410_GONE:
@@ -280,12 +285,18 @@ public final class S3AUtils {
// errors which stores can return from requests which
// the store does not support.
case SC_405_METHOD_NOT_ALLOWED:
- case SC_412_PRECONDITION_FAILED:
case SC_415_UNSUPPORTED_MEDIA_TYPE:
case SC_501_NOT_IMPLEMENTED:
ioe = new AWSUnsupportedFeatureException(message, s3Exception);
break;
+ // precondition failure: the object is there, but the precondition
+ // (e.g. etag) didn't match. Assume remote file change during
+ // rename or status passed in to openfile had an etag which didn't match.
+ case SC_412_PRECONDITION_FAILED:
+ ioe = new RemoteFileChangedException(path, message, "", ase);
+ break;
+
// out of range. This may happen if an object is overwritten with
// a shorter one while it is being read.
case SC_416_RANGE_NOT_SATISFIABLE:
@@ -860,7 +871,7 @@ public final class S3AUtils {
*/
public static String stringify(S3Object s3Object) {
StringBuilder builder = new StringBuilder(s3Object.key().length() + 100);
- builder.append(s3Object.key()).append(' ');
+ builder.append("\"").append(s3Object.key()).append("\" ");
builder.append("size=").append(s3Object.size());
return builder.toString();
}
@@ -1633,4 +1644,5 @@ public final class S3AUtils {
public static String formatRange(long rangeStart, long rangeEnd) {
return String.format("bytes=%d-%d", rangeStart, rangeEnd);
}
+
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java
index 8a24a4e14db1..3cb8d9753244 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java
@@ -24,6 +24,7 @@ import software.amazon.awssdk.core.SdkRequest;
import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CreateSessionRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
import software.amazon.awssdk.services.s3.model.GetBucketLocationRequest;
@@ -196,7 +197,8 @@ public class AWSRequestAnalyzer {
isRequestNotAlwaysInSpan(final Object request) {
return request instanceof UploadPartCopyRequest
|| request instanceof CompleteMultipartUploadRequest
- || request instanceof GetBucketLocationRequest;
+ || request instanceof GetBucketLocationRequest
+ || request instanceof CreateSessionRequest;
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CustomSdkSigner.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CustomSdkSigner.java
new file mode 100644
index 000000000000..b37860216507
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CustomSdkSigner.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.auth;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.arns.Arn;
+import software.amazon.awssdk.auth.signer.Aws4Signer;
+import software.amazon.awssdk.auth.signer.AwsS3V4Signer;
+import software.amazon.awssdk.auth.signer.internal.AbstractAwsS3V4Signer;
+import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
+import software.amazon.awssdk.core.signer.Signer;
+import software.amazon.awssdk.http.SdkHttpFullRequest;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider;
+import org.apache.hadoop.security.UserGroupInformation;
+
+/**
+ * This class is for testing the SDK's signing: it
+ * can be declared as the signer class in the configuration
+ * and then the full test suite run with it.
+ * Derived from the inner class of {@code ITestCustomSigner}.
+ * <pre>
+ * fs.s3a.custom.signers=CustomSdkSigner:org.apache.hadoop.fs.s3a.auth.CustomSdkSigner
+ *
+ * fs.s3a.s3.signing-algorithm=CustomSdkSigner
+ * </pre>
+ */
+public class CustomSdkSigner extends AbstractAwsS3V4Signer implements Signer {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(CustomSdkSigner.class);
+
+ private static final AtomicInteger INSTANTIATION_COUNT =
+ new AtomicInteger(0);
+ private static final AtomicInteger INVOCATION_COUNT =
+ new AtomicInteger(0);
+
+ /**
+ * Signer for all S3 requests.
+ */
+ private final AwsS3V4Signer s3Signer = AwsS3V4Signer.create();
+
+ /**
+ * Signer for other services.
+ */
+ private final Aws4Signer aws4Signer = Aws4Signer.create();
+
+
+ public CustomSdkSigner() {
+ int c = INSTANTIATION_COUNT.incrementAndGet();
+ LOG.info("Creating Signer #{}", c);
+ }
+
+
+ /**
+ * Method to sign the incoming request with credentials.
+ * <p>
+ * NOTE: In case of Client-side encryption, we do a "Generate Key" POST
+ * request to AWSKMS service rather than S3, this was causing the test to
+ * break. When this request happens, we have the endpoint in form of
+ * "kms.[REGION].amazonaws.com", and bucket-name becomes "kms". We can't
+ * use AWSS3V4Signer for AWSKMS service as it contains a header
+ * "x-amz-content-sha256:UNSIGNED-PAYLOAD", which returns a 400 bad
+ * request because the signature calculated by the service doesn't match
+ * what we sent.
+ * @param request the request to sign.
+ * @param executionAttributes request executionAttributes which contain the credentials.
+ */
+ @Override
+ public SdkHttpFullRequest sign(SdkHttpFullRequest request,
+ ExecutionAttributes executionAttributes) {
+ int c = INVOCATION_COUNT.incrementAndGet();
+
+ String host = request.host();
+ LOG.debug("Signing request #{} against {}: class {}",
+ c, host, request.getClass());
+ String bucketName = parseBucketFromHost(host);
+ if (bucketName.equals("kms")) {
+ return aws4Signer.sign(request, executionAttributes);
+ } else {
+ return s3Signer.sign(request, executionAttributes);
+ }
+ }
+
+ /**
+ * Parse the bucket name from the host.
+ * @param host hostname
+ * @return the parsed bucket name; if "kms" is KMS signing.
+ */
+ static String parseBucketFromHost(String host) {
+ String[] hostBits = host.split("\\.");
+ String bucketName = hostBits[0];
+ String service = hostBits[1];
+
+ if (bucketName.equals("kms")) {
+ return bucketName;
+ }
+
+ if (service.contains("s3-accesspoint") || service.contains("s3-outposts")
+ || service.contains("s3-object-lambda")) {
+ // If AccessPoint then bucketName is of format `accessPoint-accountId`;
+ String[] accessPointBits = bucketName.split("-");
+ String accountId = accessPointBits[accessPointBits.length - 1];
+ // Extract the access point name from bucket name. eg: if bucket name is
+ // test-custom-signer-<accountId>, get the access point name test-custom-signer by removing
+ // -<accountId> from the bucket name.
+ String accessPointName =
+ bucketName.substring(0, bucketName.length() - (accountId.length() + 1));
+ Arn arn = Arn.builder()
+ .accountId(accountId)
+ .partition("aws")
+ .region(hostBits[2])
+ .resource("accesspoint" + "/" + accessPointName)
+ .service("s3").build();
+
+ bucketName = arn.toString();
+ }
+
+ return bucketName;
+ }
+
+ public static int getInstantiationCount() {
+ return INSTANTIATION_COUNT.get();
+ }
+
+ public static int getInvocationCount() {
+ return INVOCATION_COUNT.get();
+ }
+
+ public static String description() {
+ return "CustomSigner{"
+ + "invocations=" + INVOCATION_COUNT.get()
+ + ", instantiations=" + INSTANTIATION_COUNT.get()
+ + "}";
+ }
+
+ public static class Initializer implements AwsSignerInitializer {
+
+ @Override
+ public void registerStore(
+ final String bucketName,
+ final Configuration storeConf,
+ final DelegationTokenProvider dtProvider,
+ final UserGroupInformation storeUgi) {
+
+ LOG.debug("Registering store for bucket {}", bucketName);
+ }
+
+ @Override
+ public void unregisterStore(final String bucketName,
+ final Configuration storeConf,
+ final DelegationTokenProvider dtProvider,
+ final UserGroupInformation storeUgi) {
+ LOG.debug("Unregistering store for bucket {}", bucketName);
+ }
+ }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java
index 7cc94cf89203..b79cf7aef5d7 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java
@@ -263,7 +263,7 @@ public class DeleteOperation extends ExecutingStoreOperation<Boolean> {
final CompletableFuture<Long> abortUploads;
if (dirOperationsPurgeUploads) {
final StoreContext sc = getStoreContext();
- final String key = sc.pathToKey(path) + "/";
+ final String key = dirKey;
LOG.debug("All uploads under {} will be deleted", key);
abortUploads = submit(sc.getExecutor(), sc.getActiveAuditSpan(), () ->
callbacks.abortMultipartUploadsUnderPrefix(key));
@@ -439,14 +439,16 @@ public class DeleteOperation extends ExecutingStoreOperation<Boolean> {
.filter(e -> e.isDirMarker)
.map(e -> e.objectIdentifier)
.collect(Collectors.toList());
- LOG.debug("Deleting of {} directory markers", dirs.size());
- // This is invoked with deleteFakeDir.
- Invoker.once("Remove S3 Dir Markers",
- status.getPath().toString(),
- () -> callbacks.removeKeys(
- dirs,
- true
- ));
+ if (!dirs.isEmpty()) {
+ LOG.debug("Deleting {} directory markers", dirs.size());
+ // This is invoked with deleteFakeDir.
+ Invoker.once("Remove S3 Dir Markers",
+ status.getPath().toString(),
+ () -> callbacks.removeKeys(
+ dirs,
+ true
+ ));
+ }
}
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java
index 8b29313faf2d..4600f392e415 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.fs.s3a.impl;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -30,7 +31,24 @@ import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTest
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.s3a.Constants;
+import static org.apache.hadoop.fs.CommonPathCapabilities.DIRECTORY_LISTING_INCONSISTENT;
+import static org.apache.hadoop.fs.CommonPathCapabilities.ETAGS_AVAILABLE;
+import static org.apache.hadoop.fs.CommonPathCapabilities.FS_CHECKSUMS;
+import static org.apache.hadoop.fs.CommonPathCapabilities.FS_MULTIPART_UPLOADER;
import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_STANDARD_OPTIONS;
+import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_OPERATIONS_PURGE_UPLOADS;
+import static org.apache.hadoop.fs.s3a.Constants.ENABLE_MULTI_DELETE;
+import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
+import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE_ENABLED;
+import static org.apache.hadoop.fs.s3a.Constants.STORE_CAPABILITY_AWS_V2;
+import static org.apache.hadoop.fs.s3a.impl.S3ExpressStorage.STORE_CAPABILITY_S3_EXPRESS_STORAGE;
+import static org.apache.hadoop.fs.s3a.Constants.STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_DELETE;
+import static org.apache.hadoop.fs.s3a.Constants.STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_KEEP;
+import static org.apache.hadoop.fs.s3a.Constants.STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_AUTHORITATIVE;
+import static org.apache.hadoop.fs.s3a.Constants.STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_DELETE;
+import static org.apache.hadoop.fs.s3a.Constants.STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_KEEP;
+import static org.apache.hadoop.fs.s3a.Constants.STORE_CAPABILITY_MULTIPART_UPLOAD_ENABLED;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.STORE_CAPABILITY_MAGIC_COMMITTER;
/**
* Internal constants private only to the S3A codebase.
@@ -142,6 +160,9 @@ public final class InternalConstants {
/** 405 status code: Method Not Allowed. */
public static final int SC_405_METHOD_NOT_ALLOWED = 405;
+ /** 409 status code: Conflict. Example: creating a bucket twice. */
+ public static final int SC_409_CONFLICT = 409;
+
/** 410 status code: Gone. */
public static final int SC_410_GONE = 410;
@@ -239,6 +260,30 @@ public final class InternalConstants {
*/
public static final Set<String> CREATE_FILE_KEYS =
Collections.unmodifiableSet(
- new HashSet<>(Arrays.asList(Constants.FS_S3A_CREATE_PERFORMANCE)));
+ new HashSet<>(Arrays.asList(FS_S3A_CREATE_PERFORMANCE)));
+ /**
+ * Dynamic Path capabilities to be evaluated
+ * in the BucketInfo tool.
+ */
+ public static final List<String> S3A_DYNAMIC_CAPABILITIES =
+ Collections.unmodifiableList(Arrays.asList(
+ ETAGS_AVAILABLE,
+ FS_CHECKSUMS,
+ FS_MULTIPART_UPLOADER,
+ DIRECTORY_LISTING_INCONSISTENT,
+
+ // s3 specific
+ STORE_CAPABILITY_AWS_V2,
+ STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_KEEP,
+ STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_DELETE,
+ STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_AUTHORITATIVE,
+ STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_KEEP,
+ STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_DELETE,
+ STORE_CAPABILITY_MAGIC_COMMITTER,
+ STORE_CAPABILITY_MULTIPART_UPLOAD_ENABLED,
+ STORE_CAPABILITY_S3_EXPRESS_STORAGE,
+ FS_S3A_CREATE_PERFORMANCE_ENABLED,
+ DIRECTORY_OPERATIONS_PURGE_UPLOADS,
+ ENABLE_MULTI_DELETE));
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java
index 34b4049b0612..910723f909f5 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java
@@ -97,6 +97,17 @@ public final class NetworkBinding {
}
}
+ /**
+ * Is this an AWS endpoint? looks at end of FQDN.
+ * @param endpoint endpoint
+ * @return true if the endpoint matches the requirements for an aws endpoint.
+ */
+ public static boolean isAwsEndpoint(final String endpoint) {
+ return (endpoint.isEmpty()
+ || endpoint.endsWith(".amazonaws.com")
+ || endpoint.endsWith(".amazonaws.com.cn"));
+ }
+
/**
* Interface used to bind to the socket factory, allows the code which
* works with the shaded AWS libraries to exist in their own class.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java
index 0fddbe6726eb..ce6e8b3538f3 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java
@@ -599,7 +599,7 @@ public class RequestFactoryImpl implements RequestFactory {
return prepareRequest(DeleteObjectsRequest
.builder()
.bucket(bucket)
- .delete(d -> d.objects(keysToDelete).quiet(true)));
+ .delete(d -> d.objects(keysToDelete).quiet(!LOG.isTraceEnabled())));
}
@Override
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3ExpressStorage.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3ExpressStorage.java
new file mode 100644
index 000000000000..fde845c5d07d
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3ExpressStorage.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+import static org.apache.hadoop.fs.s3a.impl.NetworkBinding.isAwsEndpoint;
+
+/**
+ * Anything needed to support Amazon S3 Express One Zone Storage.
+ * These have bucket names like {@code s3a://bucket--usw2-az2--x-s3/}
+ */
+public final class S3ExpressStorage {
+
+ /**
+ * Is this S3Express storage? value {@value}.
+ */
+ public static final String STORE_CAPABILITY_S3_EXPRESS_STORAGE =
+ "fs.s3a.capability.s3express.storage";
+
+ /**
+ * What is the official product name? used for error messages and logging: {@value}.
+ */
+ public static final String PRODUCT_NAME = "Amazon S3 Express One Zone Storage";
+
+ private S3ExpressStorage() {
+ }
+
+ /**
+ * Minimum length of a region.
+ */
+ private static final int SUFFIX_LENGTH = "--usw2-az2--x-s3".length();
+
+ public static final int ZONE_LENGTH = "usw2-az2".length();
+
+ /**
+ * Suffix of S3Express storage bucket names..
+ */
+ public static final String S3EXPRESS_STORE_SUFFIX = "--x-s3";
+
+ /**
+ * Is a bucket an S3Express store?
+ * This may get confused against third party stores, so takes the endpoint
+ * and only supports aws endpoints round the world.
+ * @param bucket bucket to probe
+ * @param endpoint endpoint string.
+ * @return true if the store is S3 Express.
+ */
+ public static boolean isS3ExpressStore(String bucket, final String endpoint) {
+ return isAwsEndpoint(endpoint) && hasS3ExpressSuffix(bucket);
+ }
+
+ /**
+ * Check for a bucket name matching -does not look at endpoint.
+ * @param bucket bucket to probe.
+ * @return true if the suffix is present
+ */
+ public static boolean hasS3ExpressSuffix(final String bucket) {
+ return bucket.endsWith(S3EXPRESS_STORE_SUFFIX);
+ }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
index ea1ea908486e..41251d190c44 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants;
import org.apache.hadoop.fs.s3a.impl.DirectoryPolicy;
import org.apache.hadoop.fs.s3a.impl.DirectoryPolicyImpl;
import org.apache.hadoop.fs.s3a.select.SelectTool;
+import org.apache.hadoop.fs.s3a.tools.BucketTool;
import org.apache.hadoop.fs.s3a.tools.MarkerTool;
import org.apache.hadoop.fs.shell.CommandFormat;
import org.apache.hadoop.fs.statistics.IOStatistics;
@@ -74,6 +75,7 @@ import static org.apache.hadoop.fs.s3a.Constants.*;
import static org.apache.hadoop.fs.s3a.Invoker.LOG_EVENT;
import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
import static org.apache.hadoop.fs.s3a.commit.staging.StagingCommitterConstants.FILESYSTEM_TEMP_PATH;
+import static org.apache.hadoop.fs.s3a.impl.InternalConstants.S3A_DYNAMIC_CAPABILITIES;
import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToPrettyString;
import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.retrieveIOStatistics;
import static org.apache.hadoop.fs.statistics.StoreStatisticNames.MULTIPART_UPLOAD_ABORTED;
@@ -117,6 +119,7 @@ public abstract class S3GuardTool extends Configured implements Tool,
" [command] [OPTIONS] [s3a://BUCKET]\n\n" +
"Commands: \n" +
"\t" + BucketInfo.NAME + " - " + BucketInfo.PURPOSE + "\n" +
+ "\t" + BucketTool.NAME + " - " + BucketTool.PURPOSE + "\n" +
"\t" + MarkerTool.MARKERS + " - " + MarkerTool.PURPOSE + "\n" +
"\t" + SelectTool.NAME + " - " + SelectTool.PURPOSE + "\n" +
"\t" + Uploads.NAME + " - " + Uploads.PURPOSE + "\n";
@@ -164,8 +167,19 @@ public abstract class S3GuardTool extends Configured implements Tool,
* @param opts any boolean options to support
*/
protected S3GuardTool(Configuration conf, String... opts) {
+ this(conf, 0, Integer.MAX_VALUE, opts);
+ }
+
+ /**
+ * Constructor a S3Guard tool with HDFS configuration.
+ * @param conf Configuration.
+ * @param min min number of args
+ * @param max max number of args
+ * @param opts any boolean options to support
+ */
+ protected S3GuardTool(Configuration conf, int min, int max, String... opts) {
super(conf);
- commandFormat = new CommandFormat(0, Integer.MAX_VALUE, opts);
+ commandFormat = new CommandFormat(min, max, opts);
}
/**
@@ -208,7 +222,7 @@ public abstract class S3GuardTool extends Configured implements Tool,
return cliDelta;
}
- protected void addAgeOptions() {
+ protected final void addAgeOptions() {
CommandFormat format = getCommandFormat();
format.addOptionWithValue(DAYS_FLAG);
format.addOptionWithValue(HOURS_FLAG);
@@ -240,6 +254,22 @@ public abstract class S3GuardTool extends Configured implements Tool,
return getCommandFormat().parse(args, 1);
}
+ /**
+ * Process the arguments.
+ * @param args raw args
+ * @return process arg list.
+ * @throws ExitUtil.ExitException if there's an unknown option.
+ */
+ protected List<String> parseArgsWithErrorReporting(final String[] args)
+ throws ExitUtil.ExitException {
+ try {
+ return parseArgs(args);
+ } catch (CommandFormat.UnknownOptionException e) {
+ errorln(getUsage());
+ throw new ExitUtil.ExitException(EXIT_USAGE, e.getMessage(), e);
+ }
+ }
+
protected S3AFileSystem getFilesystem() {
return filesystem;
}
@@ -273,7 +303,7 @@ public abstract class S3GuardTool extends Configured implements Tool,
filesystem = null;
}
- protected CommandFormat getCommandFormat() {
+ protected final CommandFormat getCommandFormat() {
return commandFormat;
}
@@ -397,6 +427,7 @@ public abstract class S3GuardTool extends Configured implements Tool,
Configuration conf = fs.getConf();
URI fsUri = fs.getUri();
println(out, "Filesystem %s", fsUri);
+ final Path root = new Path("/");
try {
println(out, "Location: %s", fs.getBucketLocation());
} catch (IOException e) {
@@ -524,6 +555,13 @@ public abstract class S3GuardTool extends Configured implements Tool,
processMarkerOption(out, fs,
getCommandFormat().getOptValue(MARKERS_FLAG));
+ // and check for capabilitities
+ println(out, "%nStore Capabilities");
+ for (String capability : S3A_DYNAMIC_CAPABILITIES) {
+ out.printf("\t%s %s%n", capability,
+ fs.hasPathCapability(root, capability));
+ }
+ println(out, "");
// and finally flush the output and report a success.
out.flush();
return SUCCESS;
@@ -584,7 +622,7 @@ public abstract class S3GuardTool extends Configured implements Tool,
/**
* Command to list / abort pending multipart uploads.
*/
- static class Uploads extends S3GuardTool {
+ static final class Uploads extends S3GuardTool {
public static final String NAME = "uploads";
public static final String ABORT = "abort";
public static final String LIST = "list";
@@ -736,7 +774,7 @@ public abstract class S3GuardTool extends Configured implements Tool,
return ageDate.compareTo(Date.from(u.initiated())) >= 0;
}
- private void processArgs(List<String> args, PrintStream out)
+ protected void processArgs(List<String> args, PrintStream out)
throws IOException {
CommandFormat commands = getCommandFormat();
String err = "Can only specify one of -" + LIST + ", " +
@@ -947,9 +985,13 @@ public abstract class S3GuardTool extends Configured implements Tool,
throw s3guardUnsupported();
}
switch (subCommand) {
+
case BucketInfo.NAME:
command = new BucketInfo(conf);
break;
+ case BucketTool.NAME:
+ command = new BucketTool(conf);
+ break;
case MarkerTool.MARKERS:
command = new MarkerTool(conf);
break;
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/BucketTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/BucketTool.java
new file mode 100644
index 000000000000..6bfcbcf776f1
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/BucketTool.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.tools;
+
+import java.io.FileNotFoundException;
+import java.io.PrintStream;
+import java.net.URI;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.BiFunction;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.BucketType;
+import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration;
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+import software.amazon.awssdk.services.s3.model.DataRedundancy;
+import software.amazon.awssdk.services.s3.model.LocationInfo;
+import software.amazon.awssdk.services.s3.model.LocationType;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool;
+import org.apache.hadoop.fs.shell.CommandFormat;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.util.DurationInfo;
+import org.apache.hadoop.util.ExitUtil;
+
+import static org.apache.commons.lang3.StringUtils.isNotEmpty;
+import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION;
+import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT;
+import static org.apache.hadoop.fs.s3a.Constants.S3A_BUCKET_PROBE;
+import static org.apache.hadoop.fs.s3a.impl.S3ExpressStorage.PRODUCT_NAME;
+import static org.apache.hadoop.fs.s3a.impl.S3ExpressStorage.STORE_CAPABILITY_S3_EXPRESS_STORAGE;
+import static org.apache.hadoop.fs.s3a.Invoker.once;
+import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.REJECT_OUT_OF_SPAN_OPERATIONS;
+import static org.apache.hadoop.fs.s3a.impl.NetworkBinding.isAwsEndpoint;
+import static org.apache.hadoop.fs.s3a.impl.S3ExpressStorage.hasS3ExpressSuffix;
+import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_BAD_CONFIGURATION;
+import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_NOT_ACCEPTABLE;
+import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_USAGE;
+
+/**
+ * Bucket operations, e.g. create/delete/probe.
+ */
+public final class BucketTool extends S3GuardTool {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BucketTool.class);
+
+ /**
+ * Name of this tool: {@value}.
+ */
+ public static final String NAME = "bucket";
+
+ /**
+ * Purpose of this tool: {@value}.
+ */
+ public static final String PURPOSE =
+ "View and manipulate S3 buckets";
+
+ /**
+ * create command.
+ */
+ public static final String CREATE = "create";
+
+ /**
+ * region {@value}.
+ */
+ public static final String OPT_REGION = "region";
+
+ /**
+ * endpoint {@value}.
+ */
+ public static final String OPT_ENDPOINT = "endpoint";
+
+ /**
+ * Zone for a store.
+ */
+ public static final String OPT_ZONE = "zone";
+
+ /**
+ * Error message if -zone is set but the name doesn't match.
+ * Value {@value}.
+ */
+ static final String UNSUPPORTED_ZONE_ARG =
+ "The -zone option is only supported for " + PRODUCT_NAME;
+
+ /**
+ * Error message if the bucket is S3 Express but -zone wasn't set.
+ * Value {@value}.
+ */
+ static final String NO_ZONE_SUPPLIED = "Required option -zone missing for "
+ + PRODUCT_NAME + " bucket";
+
+ /**
+ * Error Message logged/thrown when the tool could not start as
+ * the bucket probe was not disabled and the probe (inevitably)
+ * failed.
+ */
+ public static final String PROBE_FAILURE =
+ "Initialization failed because the bucket existence probe"
+ + S3A_BUCKET_PROBE + " was not disabled. Check core-site settings.";
+
+ public BucketTool(final Configuration conf) {
+ super(conf, 1, 1,
+ CREATE);
+ CommandFormat format = getCommandFormat();
+ format.addOptionWithValue(OPT_REGION);
+ format.addOptionWithValue(OPT_ENDPOINT);
+ format.addOptionWithValue(OPT_ZONE);
+ }
+
+ public String getUsage() {
+ return "bucket "
+ + "-" + CREATE + " "
+ + "[-" + OPT_ENDPOINT + " <endpoint>] "
+ + "[-" + OPT_REGION + " <region>] "
+ + "[-" + OPT_ZONE + " <zone>] "
+ + " <s3a-URL>";
+ }
+
+ public String getName() {
+ return NAME;
+ }
+
+ private Optional<String> getOptionalString(String key) {
+ String value = getCommandFormat().getOptValue(key);
+ return isNotEmpty(value) ? Optional.of(value) : Optional.empty();
+ }
+
+ @VisibleForTesting
+ int exec(final String...args) throws Exception {
+ return run(args, System.out);
+ }
+
+ @Override
+ public int run(final String[] args, final PrintStream out)
+ throws Exception, ExitUtil.ExitException {
+
+ LOG.debug("Supplied arguments: {}", String.join(", ", args));
+ final List<String> parsedArgs = parseArgsWithErrorReporting(args);
+
+ CommandFormat command = getCommandFormat();
+ boolean create = command.getOpt(CREATE);
+
+ Optional<String> endpoint = getOptionalString(OPT_ENDPOINT);
+ Optional<String> region = getOptionalString(OPT_REGION);
+ Optional<String> zone = getOptionalString(OPT_ZONE);
+
+
+ final String bucketPath = parsedArgs.get(0);
+ final Path source = new Path(bucketPath);
+ URI fsURI = source.toUri();
+ String bucket = fsURI.getHost();
+
+ println(out, "Filesystem %s", fsURI);
+ if (!"s3a".equals(fsURI.getScheme())) {
+ throw new ExitUtil.ExitException(EXIT_USAGE, "Filesystem is not S3A URL: " + fsURI);
+ }
+ println(out, "Options region=%s endpoint=%s zone=%s s3a://%s",
+ region.orElse("(unset)"),
+ endpoint.orElse("(unset)"),
+ zone.orElse("(unset)"),
+ bucket);
+ if (!create) {
+ errorln(getUsage());
+ println(out, "Supplied arguments: ["
+ + String.join(", ", parsedArgs)
+ + "]");
+ throw new ExitUtil.ExitException(EXIT_USAGE,
+ "required option not found: -create");
+ }
+
+ final Configuration conf = getConf();
+
+ removeBucketOverrides(bucket, conf,
+ S3A_BUCKET_PROBE,
+ REJECT_OUT_OF_SPAN_OPERATIONS,
+ AWS_REGION,
+ ENDPOINT);
+ // stop any S3 calls taking place against a bucket which
+ // may not exist
+ String bucketPrefix = "fs.s3a.bucket." + bucket + '.';
+ conf.setInt(bucketPrefix + S3A_BUCKET_PROBE, 0);
+ conf.setBoolean(bucketPrefix + REJECT_OUT_OF_SPAN_OPERATIONS, false);
+ // propagate an option
+ BiFunction<String, Optional<String>, Boolean> propagate = (key, opt) ->
+ opt.map(v -> {
+ conf.set(key, v);
+ LOG.info("{} = {}", key, v);
+ return true;
+ }).orElse(false);
+
+
+ propagate.apply(AWS_REGION, region);
+ propagate.apply(ENDPOINT, endpoint);
+
+ // fail fast on third party store
+ if (hasS3ExpressSuffix(bucket) && !isAwsEndpoint(endpoint.orElse(""))) {
+ throw new ExitUtil.ExitException(EXIT_NOT_ACCEPTABLE, UNSUPPORTED_ZONE_ARG);
+ }
+ S3AFileSystem fs;
+ try {
+ fs = (S3AFileSystem) FileSystem.newInstance(fsURI, conf);
+ } catch (FileNotFoundException e) {
+ // this happens if somehow the probe wasn't disabled.
+ errorln(PROBE_FAILURE);
+ throw new ExitUtil.ExitException(EXIT_BAD_CONFIGURATION, PROBE_FAILURE);
+ }
+
+ try {
+
+ // now build the configuration
+ final CreateBucketConfiguration.Builder builder = CreateBucketConfiguration.builder();
+
+ if (fs.hasPathCapability(new Path("/"), STORE_CAPABILITY_S3_EXPRESS_STORAGE)) {
+ // S3 Express store requires a zone and some other other settings
+ final String az = zone.orElseThrow(() ->
+ new ExitUtil.ExitException(EXIT_USAGE, NO_ZONE_SUPPLIED + bucket));
+ builder.location(LocationInfo.builder()
+ .type(LocationType.AVAILABILITY_ZONE).name(az).build())
+ .bucket(software.amazon.awssdk.services.s3.model.BucketInfo.builder()
+ .type(BucketType.DIRECTORY)
+ .dataRedundancy(DataRedundancy.SINGLE_AVAILABILITY_ZONE).build());
+
+ } else {
+ if (zone.isPresent()) {
+ throw new ExitUtil.ExitException(EXIT_USAGE, UNSUPPORTED_ZONE_ARG + " not " + bucket);
+ }
+ region.ifPresent(builder::locationConstraint);
+ }
+
+ final CreateBucketRequest request = CreateBucketRequest.builder()
+ .bucket(bucket)
+ .createBucketConfiguration(builder.build())
+ .build();
+
+ println(out, "Creating bucket %s", bucket);
+
+ final S3Client s3Client = fs.getS3AInternals().getAmazonS3Client(NAME);
+ try (DurationInfo ignored = new DurationInfo(LOG,
+ "Create %sbucket %s in region %s",
+ (fs.hasPathCapability(new Path("/"),
+ STORE_CAPABILITY_S3_EXPRESS_STORAGE) ? (PRODUCT_NAME + " "): ""),
+ bucket, region.orElse("(unset)"))) {
+ once("create", source.toString(), () ->
+ s3Client.createBucket(request));
+ }
+ } finally {
+ IOUtils.closeStream(fs);
+ }
+
+ return 0;
+ }
+
+ /**
+ * Remove any values from a bucket.
+ * @param bucket bucket whose overrides are to be removed. Can be null/empty
+ * @param conf config
+ * @param options list of fs.s3a options to remove
+ */
+ public static void removeBucketOverrides(final String bucket,
+ final Configuration conf,
+ final String... options) {
+
+ if (StringUtils.isEmpty(bucket)) {
+ return;
+ }
+ final String bucketPrefix = "fs.s3a.bucket." + bucket + '.';
+ for (String option : options) {
+ final String stripped = option.substring("fs.s3a.".length());
+ String target = bucketPrefix + stripped;
+ String v = conf.get(target);
+ if (v != null) {
+ conf.unset(target);
+ }
+ String extended = bucketPrefix + option;
+ if (conf.get(extended) != null) {
+ conf.unset(extended);
+ }
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java
index fc60f4b623dd..b872d1431cb2 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java
@@ -237,13 +237,7 @@ public final class MarkerTool extends S3GuardTool {
public int run(final String[] args, final PrintStream stream)
throws ExitUtil.ExitException, Exception {
this.out = stream;
- final List<String> parsedArgs;
- try {
- parsedArgs = parseArgs(args);
- } catch (CommandFormat.UnknownOptionException e) {
- errorln(getUsage());
- throw new ExitUtil.ExitException(EXIT_USAGE, e.getMessage(), e);
- }
+ final List<String> parsedArgs = parseArgsWithErrorReporting(args);
if (parsedArgs.size() != 1) {
errorln(getUsage());
println(out, "Supplied arguments: ["
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md
index 138e060155c0..8a3fd16ec172 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md
@@ -288,6 +288,16 @@ public interface S3AInternals {
HeadObjectResponse getObjectMetadata(Path path) throws IOException;
AWSCredentialProviderList shareCredentials(final String purpose);
+
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ HeadBucketResponse getBucketMetadata() throws IOException;
+
+ boolean isMultipartCopyEnabled();
+
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ long abortMultipartUploads(Path path) throws IOException;
}
```
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/third_party_stores.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/third_party_stores.md
index 0216e46014c7..1aa6e83b110e 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/third_party_stores.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/third_party_stores.md
@@ -41,9 +41,18 @@ The features which may be unavailable include:
* List API to use (`fs.s3a.list.version = 1`)
* Bucket lifecycle rules to clean up pending uploads.
-## Configuring s3a to connect to a third party store
+### Disabling Change Detection
+The (default) etag-based change detection logic expects stores to provide an Etag header in HEAD/GET requests,
+and to support it as a precondition in subsequent GET and COPY calls.
+If a store does not do this, disable the checks.
+```xml
+<property>
+ <name>fs.s3a.change.detection.mode</name>
+ <value>none</value>
+</property>
+```
## Connecting to a third party object store over HTTPS
The core setting for a third party store is to change the endpoint in `fs.s3a.endpoint`.
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java
index 828eb5127cbf..583a41fa62c5 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java
@@ -49,6 +49,7 @@ public abstract class AbstractTestS3AEncryption extends AbstractS3ATestBase {
@Override
protected Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
+ skipIfEncryptionTestsDisabled(conf);
S3ATestUtils.disableFilesystemCaching(conf);
patchConfigurationEncryptionSettings(conf);
return conf;
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
index d9c847d4b2b0..8787fca431cc 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
@@ -131,7 +131,7 @@ public class ITestS3AConfiguration {
S3ATestConstants.CONFIGURATION_TEST_ENDPOINT, "");
if (endpoint.isEmpty()) {
LOG.warn("Custom endpoint test skipped as " +
- S3ATestConstants.CONFIGURATION_TEST_ENDPOINT + "config " +
+ S3ATestConstants.CONFIGURATION_TEST_ENDPOINT + " config " +
"setting was not detected");
} else {
conf.set(Constants.ENDPOINT, endpoint);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java
index 45b0c6c206f2..321f831c0a8d 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java
@@ -49,6 +49,7 @@ import static org.apache.hadoop.fs.s3a.S3ATestUtils.createTestPath;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfEncryptionTestsDisabled;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
/**
@@ -104,7 +105,6 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
@Override
protected Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
- disableFilesystemCaching(conf);
String bucketName = getTestBucketName(conf);
// directory marker options
removeBaseAndBucketOverrides(bucketName, conf,
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java
index 66a211181bdd..a375044add43 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java
@@ -19,18 +19,12 @@
package org.apache.hadoop.fs.s3a;
import java.io.ByteArrayInputStream;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.nio.charset.StandardCharsets;
-import java.nio.file.AccessDeniedException;
-import software.amazon.awssdk.services.s3.S3Client;
-import software.amazon.awssdk.services.s3.model.GetBucketEncryptionRequest;
-import software.amazon.awssdk.services.s3.model.GetBucketEncryptionResponse;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import org.assertj.core.api.Assertions;
-import org.junit.Assume;
import org.junit.Test;
@@ -45,7 +39,6 @@ import org.apache.hadoop.fs.store.audit.AuditSpan;
import org.apache.hadoop.fs.store.EtagChecksum;
import org.apache.hadoop.test.LambdaTestUtils;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertLacksPathCapabilities;
import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
@@ -55,7 +48,6 @@ import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_ALGORITH
import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_KEY;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.XA_ETAG;
-import static org.hamcrest.Matchers.nullValue;
/**
* Tests of the S3A FileSystem which don't have a specific home and can share
@@ -153,39 +145,6 @@ public class ITestS3AMiscOperations extends AbstractS3ATestBase {
return f;
}
- /**
- * The assumption here is that 0-byte files uploaded in a single PUT
- * always have the same checksum, including stores with encryption.
- * This will be skipped if the bucket has S3 default encryption enabled.
- * @throws Throwable on a failure
- */
- @Test
- public void testEmptyFileChecksums() throws Throwable {
- assumeNoDefaultEncryption();
- final S3AFileSystem fs = getFileSystem();
- Path file1 = touchFile("file1");
- EtagChecksum checksum1 = fs.getFileChecksum(file1, 0);
- LOG.info("Checksum for {}: {}", file1, checksum1);
- assertHasPathCapabilities(fs, file1,
- CommonPathCapabilities.FS_CHECKSUMS);
- assertNotNull("Null file 1 checksum", checksum1);
- assertNotEquals("file 1 checksum", 0, checksum1.getLength());
- assertEquals("checksums of empty files", checksum1,
- fs.getFileChecksum(touchFile("file2"), 0));
- Assertions.assertThat(fs.getXAttr(file1, XA_ETAG))
- .describedAs("etag from xattr")
- .isEqualTo(checksum1.getBytes());
- }
-
- /**
- * Skip a test if we can get the default encryption on a bucket and it is
- * non-null.
- */
- private void assumeNoDefaultEncryption() throws IOException {
- skipIfClientSideEncryption();
- Assume.assumeThat(getDefaultEncryption(), nullValue());
- }
-
/**
* Make sure that when checksums are disabled, the caller
* gets null back.
@@ -226,24 +185,6 @@ public class ITestS3AMiscOperations extends AbstractS3ATestBase {
.isEqualTo(checksum1.getBytes());
}
- /**
- * Verify that on an unencrypted store, the checksum of two non-empty
- * (single PUT) files is the same if the data is the same.
- * This will be skipped if the bucket has S3 default encryption enabled.
- * @throws Throwable failure
- */
- @Test
- public void testNonEmptyFileChecksumsUnencrypted() throws Throwable {
- Assume.assumeTrue(encryptionAlgorithm().equals(S3AEncryptionMethods.NONE));
- assumeNoDefaultEncryption();
- final S3AFileSystem fs = getFileSystem();
- final EtagChecksum checksum1 =
- fs.getFileChecksum(mkFile("file5", HELLO), 0);
- assertNotNull("file 3 checksum", checksum1);
- assertEquals("checksums", checksum1,
- fs.getFileChecksum(mkFile("file6", HELLO), 0));
- }
-
private S3AEncryptionMethods encryptionAlgorithm() {
return getFileSystem().getS3EncryptionAlgorithm();
}
@@ -399,22 +340,4 @@ public class ITestS3AMiscOperations extends AbstractS3ATestBase {
return o;
}
- /**
- * Gets default encryption settings for the bucket or returns null if default
- * encryption is disabled.
- */
- private GetBucketEncryptionResponse getDefaultEncryption() throws IOException {
- S3AFileSystem fs = getFileSystem();
- S3Client s3 = getS3AInternals().getAmazonS3Client("check default encryption");
- try (AuditSpan s = span()){
- return Invoker.once("getBucketEncryption()",
- fs.getBucket(),
- () -> s3.getBucketEncryption(GetBucketEncryptionRequest.builder()
- .bucket(fs.getBucket())
- .build()));
- } catch (FileNotFoundException | AccessDeniedException | AWSBadRequestException e) {
- return null;
- }
- }
-
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java
index 3f6870be46b2..20e5c12ec79c 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
import org.apache.hadoop.fs.store.audit.AuditSpan;
-import org.apache.hadoop.io.IOUtils;
import org.junit.Assert;
import org.slf4j.Logger;
@@ -35,22 +34,31 @@ import org.slf4j.LoggerFactory;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
import java.util.Date;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertFileHasLength;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
-import static org.apache.hadoop.fs.s3a.Invoker.LOG_EVENT;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_PATH_PREFIX;
/**
* Utilities for S3A multipart upload tests.
*/
public final class MultipartTestUtils {
+
private static final Logger LOG = LoggerFactory.getLogger(
MultipartTestUtils.class);
+ /**
+ * Target file of {@code createMagicFile()}.
+ */
+ public static final String MAGIC_FILE_TARGET = "subdir/file.txt";
+
/** Not instantiated. */
private MultipartTestUtils() { }
@@ -94,24 +102,10 @@ public final class MultipartTestUtils {
/** Delete any uploads under given path (recursive). Silent on failure. */
public static void clearAnyUploads(S3AFileSystem fs, Path path) {
- String key = fs.pathToKey(path);
- AuditSpan span = null;
try {
- RemoteIterator<MultipartUpload> uploads = fs.listUploads(key);
- span = fs.createSpan("multipart", path.toString(), null);
- final WriteOperationHelper helper
- = fs.getWriteOperationHelper();
- while (uploads.hasNext()) {
- MultipartUpload upload = uploads.next();
- LOG.debug("Cleaning up upload: {} {}", upload.key(),
- truncatedUploadId(upload.uploadId()));
- helper.abortMultipartUpload(upload.key(),
- upload.uploadId(), true, LOG_EVENT);
- }
+ fs.getS3AInternals().abortMultipartUploads(path);
} catch (IOException ioe) {
LOG.info("Ignoring exception: ", ioe);
- } finally {
- IOUtils.closeStream(span);
}
}
@@ -165,6 +159,33 @@ public final class MultipartTestUtils {
return fullId.substring(0, 12) + " ...";
}
+ /**
+ * Given a dir, return the name of the magic subdir.
+ * The naming has changed across versions; this isolates
+ * the changes.
+ * @param dir directory
+ * @return the magic subdir
+ */
+ public static Path magicPath(Path dir) {
+ return new Path(dir, MAGIC_PATH_PREFIX + "001/");
+ }
+
+ /**
+ * Create a magic file of "real" length more than 0 bytes long.
+ * @param fs filesystem
+ * @param dir directory
+ * @return the path
+ * @throws IOException creation failure.p
+ */
+ public static Path createMagicFile(final S3AFileSystem fs, final Path dir) throws IOException {
+ Path magicFile = new Path(magicPath(dir), "__base/" + MAGIC_FILE_TARGET);
+ createFile(fs, magicFile, true, "123".getBytes(StandardCharsets.UTF_8));
+
+ // the file exists but is a 0 byte marker file.
+ assertFileHasLength(fs, magicFile, 0);
+ return magicFile;
+ }
+
/** Struct of object key, upload ID. */
public static class IdKey {
private String key;
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
index aa38186c6503..dbe5aaf22288 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
@@ -22,6 +22,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.EtagSource;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileContext;
@@ -37,6 +38,7 @@ import org.apache.hadoop.fs.s3a.auth.MarshalledCredentials;
import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets;
import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy;
import org.apache.hadoop.fs.s3a.impl.ContextAccessors;
+import org.apache.hadoop.fs.s3a.impl.NetworkBinding;
import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum;
import org.apache.hadoop.fs.s3a.impl.StoreContext;
import org.apache.hadoop.fs.s3a.impl.StoreContextBuilder;
@@ -57,6 +59,8 @@ import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningE
import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors;
import org.apache.hadoop.util.BlockingThreadPoolExecutorService;
import org.apache.hadoop.util.DurationInfo;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Preconditions;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.functional.CallableRaisingIOE;
import org.apache.hadoop.util.functional.FutureIO;
@@ -89,6 +93,7 @@ import java.util.stream.Collectors;
import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.submit;
import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.waitForCompletion;
+import static org.apache.hadoop.fs.s3a.impl.S3ExpressStorage.STORE_CAPABILITY_S3_EXPRESS_STORAGE;
import static org.apache.hadoop.test.GenericTestUtils.buildPaths;
import static org.apache.hadoop.util.Preconditions.checkNotNull;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CREDENTIAL_PROVIDER_PATH;
@@ -98,6 +103,8 @@ import static org.apache.hadoop.fs.s3a.S3ATestConstants.*;
import static org.apache.hadoop.fs.s3a.Constants.*;
import static org.apache.hadoop.fs.s3a.S3AUtils.buildEncryptionSecrets;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.apache.hadoop.util.functional.RemoteIterators.mappingRemoteIterator;
+import static org.apache.hadoop.util.functional.RemoteIterators.toList;
import static org.junit.Assert.*;
/**
@@ -981,6 +988,54 @@ public final class S3ATestUtils {
}
}
+ /**
+ * Given a RemoteIterator to a list of file statuses, return a list of paths.
+ * @param i iterator
+ * @return list of paths
+ * @param <T> type of status
+ * @throws IOException failure retrieving values from the iterator
+ */
+ public static <T extends FileStatus> List<Path> toPathList(RemoteIterator<T> i)
+ throws IOException {
+ return toList(mappingRemoteIterator(i, FileStatus::getPath));
+ }
+
+ /**
+ * Expect an error code from the exception.
+ * @param code error code
+ * @param error exception
+ */
+ public static void expectErrorCode(final int code, final ExitUtil.ExitException error) {
+ if (error.getExitCode() != code) {
+ throw error;
+ }
+ }
+
+ /**
+ * Require a test case to be against Amazon S3 Express store.
+ */
+ public static void assumeS3ExpressFileSystem(final FileSystem fs) throws IOException {
+ assume("store is not S3 Express: " + fs.getUri(),
+ fs.hasPathCapability(new Path("/"), STORE_CAPABILITY_S3_EXPRESS_STORAGE));
+ }
+
+ /**
+ * Require a test case to be against a standard S3 store.
+ */
+ public static void assumeNotS3ExpressFileSystem(final FileSystem fs) throws IOException {
+ assume("store is S3 Express: " + fs.getUri(),
+ !fs.hasPathCapability(new Path("/"), STORE_CAPABILITY_S3_EXPRESS_STORAGE));
+ }
+
+ /**
+ * Require a store to be hosted by Amazon -i.e. not a third party store.
+ */
+ public static void assumeStoreAwsHosted(final FileSystem fs) {
+ assume("store is not AWS S3",
+ !NetworkBinding.isAwsEndpoint(fs.getConf()
+ .getTrimmed(ENDPOINT, DEFAULT_ENDPOINT)));
+ }
+
/**
* Helper class to do diffs of metrics.
*/
@@ -1566,5 +1621,26 @@ public final class S3ATestUtils {
return fs.hasPathCapability(new Path("/"), FS_S3A_CREATE_PERFORMANCE_ENABLED);
}
+ /**
+ * Is the filesystem connector bonded to S3Express storage?
+ * @param fs filesystem.
+ * @return true if the store has the relevant path capability.
+ * @throws IOException IO failure
+ */
+ public static boolean isS3ExpressStorage(FileSystem fs) throws IOException {
+ return fs.hasPathCapability(new Path("/"), STORE_CAPABILITY_S3_EXPRESS_STORAGE);
+ }
+ /**
+ * Get an etag from a FileStatus which must implement
+ * the {@link EtagSource} interface -which S3AFileStatus does.
+ *
+ * @param status the status.
+ * @return the etag
+ */
+ public static String etag(FileStatus status) {
+ Preconditions.checkArgument(status instanceof EtagSource,
+ "Not an EtagSource: %s", status);
+ return ((EtagSource) status).getEtag();
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java
index fe48cd891157..58bb2a5e491f 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java
@@ -20,14 +20,18 @@ package org.apache.hadoop.fs.s3a.auth;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicInteger;
-import software.amazon.awssdk.arns.Arn;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
import software.amazon.awssdk.auth.signer.Aws4Signer;
import software.amazon.awssdk.auth.signer.AwsS3V4Signer;
+import software.amazon.awssdk.auth.signer.internal.AbstractAwsS3V4Signer;
import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
import software.amazon.awssdk.core.signer.Signer;
import software.amazon.awssdk.http.SdkHttpFullRequest;
@@ -38,8 +42,10 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
import org.apache.hadoop.fs.s3a.Constants;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
@@ -48,12 +54,18 @@ import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider;
import org.apache.hadoop.security.UserGroupInformation;
import static org.apache.hadoop.fs.s3a.Constants.CUSTOM_SIGNERS;
+import static org.apache.hadoop.fs.s3a.Constants.ENABLE_MULTI_DELETE;
import static org.apache.hadoop.fs.s3a.Constants.SIGNING_ALGORITHM_S3;
+import static org.apache.hadoop.fs.s3a.MultipartTestUtils.createMagicFile;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
/**
* Tests for custom Signers and SignerInitializers.
+ * Because the v2 sdk has had some problems with bulk delete
+ * and custom signing, this suite is parameterized.
*/
+@RunWith(Parameterized.class)
public class ITestCustomSigner extends AbstractS3ATestBase {
private static final Logger LOG = LoggerFactory
@@ -62,10 +74,33 @@ public class ITestCustomSigner extends AbstractS3ATestBase {
private static final String TEST_ID_KEY = "TEST_ID_KEY";
private static final String TEST_REGION_KEY = "TEST_REGION_KEY";
+ /**
+ * Parameterization.
+ */
+ @Parameterized.Parameters(name = "{0}")
+ public static Collection<Object[]> params() {
+ return Arrays.asList(new Object[][]{
+ {"bulk delete", true},
+ {"simple-delete", false},
+ });
+ }
+
+ private final boolean bulkDelete;
+
+ private final UserGroupInformation ugi1 = UserGroupInformation.createRemoteUser("user1");
+
+ private final UserGroupInformation ugi2 = UserGroupInformation.createRemoteUser("user2");
+
private String regionName;
private String endpoint;
+ public ITestCustomSigner(
+ final String ignored,
+ final boolean bulkDelete) {
+ this.bulkDelete = bulkDelete;
+ }
+
@Override
public void setup() throws Exception {
super.setup();
@@ -79,6 +114,17 @@ public class ITestCustomSigner extends AbstractS3ATestBase {
}
LOG.info("Determined region name to be [{}] for bucket [{}]", regionName,
fs.getBucket());
+ CustomSignerInitializer.reset();
+ }
+
+ /**
+ * Teardown closes all filesystems for the test UGIs.
+ */
+ @Override
+ public void teardown() throws Exception {
+ super.teardown();
+ FileSystem.closeAllForUGI(ugi1);
+ FileSystem.closeAllForUGI(ugi2);
}
@Test
@@ -86,12 +132,10 @@ public class ITestCustomSigner extends AbstractS3ATestBase {
throws IOException, InterruptedException {
final Path basePath = path(getMethodName());
- UserGroupInformation ugi1 = UserGroupInformation.createRemoteUser("user1");
- FileSystem fs1 = runMkDirAndVerify(ugi1,
+ FileSystem fs1 = runStoreOperationsAndVerify(ugi1,
new Path(basePath, "customsignerpath1"), "id1");
- UserGroupInformation ugi2 = UserGroupInformation.createRemoteUser("user2");
- FileSystem fs2 = runMkDirAndVerify(ugi2,
+ FileSystem fs2 = runStoreOperationsAndVerify(ugi2,
new Path(basePath, "customsignerpath2"), "id2");
Assertions.assertThat(CustomSignerInitializer.knownStores.size())
@@ -104,14 +148,15 @@ public class ITestCustomSigner extends AbstractS3ATestBase {
.as("Num registered stores mismatch").isEqualTo(0);
}
- private FileSystem runMkDirAndVerify(UserGroupInformation ugi,
+ private S3AFileSystem runStoreOperationsAndVerify(UserGroupInformation ugi,
Path finalPath, String identifier)
throws IOException, InterruptedException {
Configuration conf = createTestConfig(identifier);
- return ugi.doAs((PrivilegedExceptionAction<FileSystem>) () -> {
+ return ugi.doAs((PrivilegedExceptionAction<S3AFileSystem>) () -> {
int instantiationCount = CustomSigner.getInstantiationCount();
int invocationCount = CustomSigner.getInvocationCount();
- FileSystem fs = finalPath.getFileSystem(conf);
+ S3AFileSystem fs = (S3AFileSystem)finalPath.getFileSystem(conf);
+
fs.mkdirs(finalPath);
Assertions.assertThat(CustomSigner.getInstantiationCount())
.as("CustomSigner Instantiation count lower than expected")
@@ -130,6 +175,22 @@ public class ITestCustomSigner extends AbstractS3ATestBase {
.as("Configuration TEST_KEY mismatch in %s", CustomSigner.description())
.isEqualTo(identifier);
+ // now do some more operations to make sure all is good.
+ final Path subdir = new Path(finalPath, "year=1970/month=1/day=1");
+ fs.mkdirs(subdir);
+
+ final Path file1 = new Path(subdir, "file1");
+ ContractTestUtils.touch(fs, new Path(subdir, "file1"));
+ fs.listStatus(subdir);
+ fs.delete(file1, false);
+ ContractTestUtils.touch(fs, new Path(subdir, "file1"));
+
+ // create a magic file.
+ createMagicFile(fs, subdir);
+ ContentSummary summary = fs.getContentSummary(finalPath);
+ fs.getS3AInternals().abortMultipartUploads(subdir);
+ fs.rename(subdir, new Path(finalPath, "renamed"));
+ fs.delete(finalPath, true);
return fs;
});
}
@@ -143,10 +204,15 @@ public class ITestCustomSigner extends AbstractS3ATestBase {
private Configuration createTestConfig(String identifier) {
Configuration conf = createConfiguration();
+ removeBaseAndBucketOverrides(conf,
+ CUSTOM_SIGNERS,
+ SIGNING_ALGORITHM_S3,
+ ENABLE_MULTI_DELETE);
conf.set(CUSTOM_SIGNERS,
"CustomS3Signer:" + CustomSigner.class.getName() + ":"
+ CustomSignerInitializer.class.getName());
conf.set(SIGNING_ALGORITHM_S3, "CustomS3Signer");
+ conf.setBoolean(ENABLE_MULTI_DELETE, bulkDelete);
conf.set(TEST_ID_KEY, identifier);
conf.set(TEST_REGION_KEY, regionName);
@@ -162,7 +228,7 @@ public class ITestCustomSigner extends AbstractS3ATestBase {
}
@Private
- public static final class CustomSigner implements Signer {
+ public static final class CustomSigner extends AbstractAwsS3V4Signer implements Signer {
private static final AtomicInteger INSTANTIATION_COUNT =
@@ -202,7 +268,8 @@ public class ITestCustomSigner extends AbstractS3ATestBase {
try {
lastStoreValue = CustomSignerInitializer
.getStoreValue(bucketName, UserGroupInformation.getCurrentUser());
- LOG.info("Store value for bucket {} is {}", bucketName, lastStoreValue);
+ LOG.info("Store value for bucket {} is {}", bucketName,
+ (lastStoreValue == null) ? "unset" : "set");
} catch (IOException e) {
throw new RuntimeException("Failed to get current Ugi " + e, e);
}
@@ -216,35 +283,7 @@ public class ITestCustomSigner extends AbstractS3ATestBase {
}
private String parseBucketFromHost(String host) {
- String[] hostBits = host.split("\\.");
- String bucketName = hostBits[0];
- String service = hostBits[1];
-
- if (bucketName.equals("kms")) {
- return bucketName;
- }
-
- if (service.contains("s3-accesspoint") || service.contains("s3-outposts")
- || service.contains("s3-object-lambda")) {
- // If AccessPoint then bucketName is of format `accessPoint-accountId`;
- String[] accessPointBits = bucketName.split("-");
- String accountId = accessPointBits[accessPointBits.length - 1];
- // Extract the access point name from bucket name. eg: if bucket name is
- // test-custom-signer-<accountId>, get the access point name test-custom-signer by removing
- // -<accountId> from the bucket name.
- String accessPointName =
- bucketName.substring(0, bucketName.length() - (accountId.length() + 1));
- Arn arn = Arn.builder()
- .accountId(accountId)
- .partition("aws")
- .region(hostBits[2])
- .resource("accesspoint" + "/" + accessPointName)
- .service("s3").build();
-
- bucketName = arn.toString();
- }
-
- return bucketName;
+ return CustomSdkSigner.parseBucketFromHost(host);
}
public static int getInstantiationCount() {
@@ -287,6 +326,13 @@ public class ITestCustomSigner extends AbstractS3ATestBase {
knownStores.remove(storeKey);
}
+ /**
+ * Clear the stores; invoke during test setup.
+ */
+ public static void reset() {
+ knownStores.clear();
+ }
+
public static StoreValue getStoreValue(String bucketName,
UserGroupInformation ugi) {
StoreKey storeKey = new StoreKey(bucketName, ugi);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java
index 64def00fd287..ee1e8328a771 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java
@@ -619,8 +619,10 @@ public class ITestCommitOperations extends AbstractCommitITest {
commits.add(commit1);
}
- assertPathDoesNotExist("destination dir", destDir);
- assertPathDoesNotExist("subdirectory", subdir);
+ if (!isS3ExpressStorage(fs)) {
+ assertPathDoesNotExist("destination dir", destDir);
+ assertPathDoesNotExist("subdirectory", subdir);
+ }
LOG.info("Initiating commit operations");
try (CommitContext commitContext
= actions.createCommitContextForTesting(destDir, JOB_ID, 0)) {
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestTreewalkProblems.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestTreewalkProblems.java
new file mode 100644
index 000000000000..286a92f78934
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestTreewalkProblems.java
@@ -0,0 +1,463 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+import software.amazon.awssdk.services.s3.model.MultipartUpload;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FsShell;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.performance.AbstractS3ACostTest;
+import org.apache.hadoop.fs.store.audit.AuditSpan;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.LocatedFileStatusFetcher;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.task.JobContextImpl;
+import org.apache.hadoop.tools.DistCpConstants;
+import org.apache.hadoop.tools.util.DistCpTestUtils;
+
+import static org.apache.hadoop.fs.CommonPathCapabilities.DIRECTORY_LISTING_INCONSISTENT;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.treeWalk;
+import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_OPERATIONS_PURGE_UPLOADS;
+import static org.apache.hadoop.fs.s3a.MultipartTestUtils.assertNoUploadsAt;
+import static org.apache.hadoop.fs.s3a.MultipartTestUtils.clearAnyUploads;
+import static org.apache.hadoop.fs.s3a.MultipartTestUtils.createMagicFile;
+import static org.apache.hadoop.fs.s3a.MultipartTestUtils.magicPath;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.toPathList;
+import static org.apache.hadoop.fs.s3a.S3AUtils.HIDDEN_FILE_FILTER;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_COMMITTER_ENABLED;
+import static org.apache.hadoop.mapreduce.lib.input.FileInputFormat.LIST_STATUS_NUM_THREADS;
+import static org.apache.hadoop.util.ToolRunner.run;
+import static org.apache.hadoop.util.functional.RemoteIterators.foreach;
+import static org.apache.hadoop.util.functional.RemoteIterators.remoteIteratorFromArray;
+import static org.apache.hadoop.util.functional.RemoteIterators.toList;
+
+/**
+ * Test behavior of treewalking when there are pending
+ * uploads. All commands MUST work.
+ * Currently, the only one which doesn't is distcp;
+ * some tests do have different assertions about directories
+ * found.
+ */
+public class ITestTreewalkProblems extends AbstractS3ACostTest {
+
+ /**
+ * Exit code to expect on a shell failure.
+ */
+ public static final int SHELL_FAILURE = 1;
+
+ /**
+ * Are directory listings potentially inconsistent?
+ */
+ private boolean listingInconsistent;
+
+ @Override
+ public Configuration createConfiguration() {
+ final Configuration conf = super.createConfiguration();
+ removeBaseAndBucketOverrides(conf,
+ DIRECTORY_OPERATIONS_PURGE_UPLOADS,
+ MAGIC_COMMITTER_ENABLED);
+ conf.setBoolean(DIRECTORY_OPERATIONS_PURGE_UPLOADS, true);
+ conf.setBoolean(MAGIC_COMMITTER_ENABLED, true);
+ return conf;
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ final S3AFileSystem fs = getFileSystem();
+ final Path path = methodPath();
+ assertHasPathCapabilities(fs, path, DIRECTORY_OPERATIONS_PURGE_UPLOADS);
+ listingInconsistent = fs.hasPathCapability(path, DIRECTORY_LISTING_INCONSISTENT);
+ clearAnyUploads(fs, path);
+ }
+
+ @Test
+ public void testListFilesDeep() throws Throwable {
+ final S3AFileSystem fs = getFileSystem();
+ final Path src = createDirWithUpload();
+
+ LOG.info("listFiles({}, true)", src);
+ foreach(fs.listFiles(src, true), e -> LOG.info("{}", e));
+
+ LOG.info("listFiles({}, true)", src);
+ foreach(fs.listLocatedStatus(src), e -> LOG.info("{}", e));
+
+ // and just verify a cleanup works
+ Assertions.assertThat(fs.getS3AInternals().abortMultipartUploads(src))
+ .describedAs("Aborted uploads under %s", src)
+ .isEqualTo(1);
+ assertNoUploadsAt(fs, src);
+ }
+
+ /**
+ * Create a directory methodPath()/src with a magic upload underneath,
+ * with the upload pointing at {@code src/subdir/file.txt}.
+ * @return the directory created
+ * @throws IOException creation problems
+ */
+ private Path createDirWithUpload() throws IOException {
+ final S3AFileSystem fs = getFileSystem();
+ final Path src = new Path(methodPath(), "src");
+ // create a magic file.
+ createMagicFile(fs, src);
+ fs.delete(magicPath(src), true);
+ return src;
+ }
+
+ @Test
+ public void testLocatedFileStatusFetcher() throws Throwable {
+ describe("Validate mapreduce LocatedFileStatusFetcher");
+
+ final Path src = createDirWithUpload();
+
+ Configuration listConfig = new Configuration(getConfiguration());
+ listConfig.setInt(LIST_STATUS_NUM_THREADS, 2);
+
+ LocatedFileStatusFetcher fetcher =
+ new LocatedFileStatusFetcher(listConfig, new Path[]{src}, true, HIDDEN_FILE_FILTER, true);
+ Assertions.assertThat(fetcher.getFileStatuses()).hasSize(0);
+ }
+
+ @Test
+ public void testGetContentSummaryDirsAndFiles() throws Throwable {
+ describe("FileSystem.getContentSummary()");
+ final S3AFileSystem fs = getFileSystem();
+ final Path src = createDirWithUpload();
+ fs.mkdirs(new Path(src, "child"));
+ final Path path = methodPath();
+ file(new Path(path, "file"));
+ final int dirs = listingInconsistent ? 3 : 3;
+ assertContentSummary(path, dirs, 1);
+ }
+
+ /**
+ * Execute getContentSummary() down a directory tree which only
+ * contains a single real directory.
+ * This test case has been a bit inconsistent between different store
+ * types.
+ */
+ @Test
+ public void testGetContentSummaryPendingDir() throws Throwable {
+ describe("FileSystem.getContentSummary() with pending dir");
+ assertContentSummary(createDirWithUpload(), 1, 0);
+ }
+
+ /**
+ * Make an assertions about the content summary of a path.
+ * @param path path to scan
+ * @param dirs number of directories to find.
+ * @param files number of files to find
+ * @throws IOException scanning problems
+ */
+ private void assertContentSummary(
+ final Path path,
+ final int dirs,
+ final int files) throws IOException {
+ ContentSummary summary = getFileSystem().getContentSummary(path);
+ Assertions.assertThat(summary.getDirectoryCount())
+ .describedAs("dir count under %s of %s", path, summary)
+ .isEqualTo(dirs);
+ Assertions.assertThat(summary.getFileCount())
+ .describedAs("filecount count under %s of %s", path, summary)
+ .isEqualTo(files);
+ }
+
+ /**
+ * Execute getContentSummary() down a directory tree which only
+ * contains a single real directory.
+ */
+ @Test
+ public void testGetContentSummaryFiles() throws Throwable {
+ describe("FileSystem.getContentSummary()");
+ final S3AFileSystem fs = getFileSystem();
+ final Path src = createDirWithUpload();
+ fs.mkdirs(new Path(src, "child"));
+ final Path base = methodPath();
+ touch(fs, new Path(base, "file"));
+ assertContentSummary(base, 3, 1);
+ }
+
+ /**
+ * Test all the various filesystem.list* calls.
+ * Bundled into one test case to reduce setup/teardown overhead.
+ */
+ @Test
+ public void testListStatusOperations() throws Throwable {
+ describe("FileSystem liststtus calls");
+ final S3AFileSystem fs = getFileSystem();
+ final Path base = methodPath();
+ final Path src = createDirWithUpload();
+ final Path file = new Path(base, "file");
+ final Path dir2 = new Path(base, "dir2");
+ // path in a child dir
+ final Path childfile = new Path(dir2, "childfile");
+ file(childfile);
+ file(file);
+ fs.mkdirs(dir2);
+
+ Assertions.assertThat(toPathList(fs.listStatusIterator(base)))
+ .describedAs("listStatusIterator(%s)", base)
+ .contains(src, dir2, file);
+
+ Assertions.assertThat(toPathList(remoteIteratorFromArray(fs.listStatus(base))))
+ .describedAs("listStatusIterator(%s, false)", false)
+ .contains(src, dir2, file);
+
+ Assertions.assertThat(toPathList(fs.listFiles(base, false)))
+ .describedAs("listfiles(%s, false)", false)
+ .containsExactly(file);
+
+ Assertions.assertThat(toPathList(fs.listFiles(base, true)))
+ .describedAs("listfiles(%s, true)", false)
+ .containsExactlyInAnyOrder(file, childfile);
+
+ Assertions.assertThat(toPathList(fs.listLocatedStatus(base, (p) -> true)))
+ .describedAs("listLocatedStatus(%s, true)", false)
+ .contains(src, dir2, file);
+ Assertions.assertThat(toPathList(fs.listLocatedStatus(base)))
+ .describedAs("listLocatedStatus(%s, true)", false)
+ .contains(src, dir2, file);
+ }
+
+ @Test
+ public void testShellList() throws Throwable {
+ describe("Validate hadoop fs -ls sorted and unsorted");
+ final S3AFileSystem fs = getFileSystem();
+ final Path base = methodPath();
+ createDirWithUpload();
+ fs.mkdirs(new Path(base, "dir2"));
+ // recursive not sorted
+ shell(base, "-ls", "-R", base.toUri().toString());
+
+ // recursive sorted
+ shell(base, "-ls", "-R", "-S", base.toUri().toString());
+ }
+
+ @Test
+ public void testShellDu() throws Throwable {
+ describe("Validate hadoop fs -du");
+ final Path base = methodPath();
+ createDirWithUpload();
+
+ shell(base, "-du", base.toUri().toString());
+ }
+
+ @Test
+ public void testShellDf() throws Throwable {
+ describe("Validate hadoop fs -df");
+ final Path base = methodPath();
+
+ final String p = base.toUri().toString();
+ shell(SHELL_FAILURE, base, "-df", p);
+ createDirWithUpload();
+
+ shell(base, "-df", p);
+ }
+
+ @Test
+ public void testShellFind() throws Throwable {
+ describe("Validate hadoop fs -ls -R");
+ final Path base = methodPath();
+ final String p = base.toUri().toString();
+ shell(SHELL_FAILURE, base, "-find", p, "-print");
+ createDirWithUpload();
+ shell(base, "-find", p, "-print");
+ }
+
+ @Test
+ public void testDistCp() throws Throwable {
+ describe("Validate distcp");
+ final S3AFileSystem fs = getFileSystem();
+ final Path base = methodPath();
+ final Path src = createDirWithUpload();
+ final Path dest = new Path(base, "dest");
+ file(new Path(src, "real-file"));
+ // distcp fails if uploads are visible
+ DistCpTestUtils.assertRunDistCp(DistCpConstants.SUCCESS, src.toString(), dest.toString(),
+ "-useiterator -update -delete -direct", getConfiguration());
+ }
+
+ @Test
+ public void testDistCpNoIterator() throws Throwable {
+ describe("Validate distcp");
+ final S3AFileSystem fs = getFileSystem();
+ final Path base = methodPath();
+ final Path src = createDirWithUpload();
+ final Path dest = new Path(base, "dest");
+ file(new Path(src, "real-file"));
+
+ // distcp fails if uploads are visible
+ DistCpTestUtils.assertRunDistCp(DistCpConstants.SUCCESS, src.toString(), dest.toString(),
+ "-update -delete -direct", getConfiguration());
+ }
+
+ /**
+ * CTU is also doing treewalking, though it's test only.
+ */
+ @Test
+ public void testContractTestUtilsTreewalk() throws Throwable {
+ final S3AFileSystem fs = getFileSystem();
+ final Path base = methodPath();
+ createDirWithUpload();
+ final ContractTestUtils.TreeScanResults treeWalk = treeWalk(fs, base);
+
+ ContractTestUtils.TreeScanResults listing =
+ new ContractTestUtils.TreeScanResults(fs.listFiles(base, true));
+ treeWalk.assertFieldsEquivalent("treewalk vs listFiles(/, true)", listing, treeWalk.getFiles(),
+ listing.getFiles());
+ }
+
+ /**
+ * Globber is already resilient to missing directories; a relic
+ * of the time when HEAD requests on s3 objects could leave the
+ * 404 in S3 front end cache.
+ */
+ @Test
+ public void testGlobberTreewalk() throws Throwable {
+ final S3AFileSystem fs = getFileSystem();
+ final Path base = methodPath();
+ final Path src = createDirWithUpload();
+ // this is the pending dir
+ final Path subdir = new Path(src, "subdir/");
+ final Path dest = new Path(base, "dest");
+ final Path monday = new Path(dest, "day=monday");
+ final Path realFile = file(new Path(monday, "real-file.parquet"));
+ assertGlob(fs, new Path(base, "*/*/*.parquet"), realFile);
+ if (listingInconsistent) {
+ assertGlob(fs, new Path(base, "*"), src, dest);
+ assertGlob(fs, new Path(base, "*/*"), subdir, monday);
+ } else {
+ assertGlob(fs, new Path(base, "*"), src, dest);
+ assertGlob(fs, new Path(base, "*/*"), monday);
+ }
+ }
+
+ private static void assertGlob(final S3AFileSystem fs,
+ final Path pattern,
+ final Path... expected)
+ throws IOException {
+ final FileStatus[] globbed = fs.globStatus(pattern,
+ (f) -> true);
+ final List<Path> paths = Arrays.stream(globbed).map(s -> s.getPath())
+ .collect(Collectors.toList());
+ Assertions.assertThat(paths)
+ .describedAs("glob(%s)", pattern)
+ .containsExactlyInAnyOrder(expected);
+ }
+
+ @Test
+ public void testFileInputFormatSplits() throws Throwable {
+ final S3AFileSystem fs = getFileSystem();
+ final Path base = methodPath();
+ final Path src = createDirWithUpload();
+ final Path dest = new Path(base, "dest");
+ final Path monday = new Path(dest, "day=monday");
+ final int count = 4;
+ List<Path> files = new ArrayList<>();
+ for (int i = 0; i < count; i++) {
+ files.add(file(new Path(monday, "file-0" + i + ".parquet")));
+ }
+ final JobContextImpl jobContext = new JobContextImpl(getConfiguration(), new JobID("job", 1));
+ final JobConf jc = (JobConf) jobContext.getConfiguration();
+ jc.set("mapreduce.input.fileinputformat.inputdir", base.toUri().toString());
+ jc.setBoolean("mapreduce.input.fileinputformat.input.dir.recursive", true);
+ final TextInputFormat inputFormat = new TextInputFormat();
+ final List<Path> paths = inputFormat.getSplits(jobContext).stream().map(s ->
+ ((FileSplit) s).getPath())
+ .collect(Collectors.toList());
+
+ Assertions.assertThat(paths)
+ .describedAs("input split of base directory")
+ .containsExactlyInAnyOrderElementsOf(files);
+
+
+ }
+
+ /**
+ * Exec a shell command; require it to succeed.
+ * @param base base dir
+ * @param command command sequence
+ * @throws Exception failure
+ */
+
+ private void shell(final Path base, final String... command) throws Exception {
+ shell(0, base, command);
+ }
+
+ /**
+ * Exec a shell command; require the result to match the expected outcome.
+ * @param expected expected outcome
+ * @param base base dir
+ * @param command command sequence
+ * @throws Exception failure
+ */
+
+ private void shell(int expected, final Path base, final String... command) throws Exception {
+ Assertions.assertThat(run(getConfiguration(), new FsShell(), command))
+ .describedAs("%s %s", command[0], base)
+ .isEqualTo(expected);
+ }
+
+ /**
+ * Assert the upload count under a dir is the expected value.
+ * Failure message will include the list of entries.
+ * @param dir dir
+ * @param expected expected count
+ * @throws IOException listing problem
+ */
+ private void assertUploadCount(final Path dir, final int expected) throws IOException {
+ Assertions.assertThat(toList(listUploads(dir)))
+ .describedAs("uploads under %s", dir)
+ .hasSize(expected);
+ }
+
+ /**
+ * List uploads; use the same APIs that the directory operations use,
+ * so implicitly validating them.
+ * @param dir directory to list
+ * @return full list of entries
+ * @throws IOException listing problem
+ */
+ private RemoteIterator<MultipartUpload> listUploads(Path dir) throws IOException {
+ final S3AFileSystem fs = getFileSystem();
+ try (AuditSpan ignored = span()) {
+ final StoreContext sc = fs.createStoreContext();
+ return fs.listUploadsUnderPrefix(sc, sc.pathToKey(dir));
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestUploadPurgeOnDirectoryOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestUploadPurgeOnDirectoryOperations.java
index 9e0702737598..80a44e22b8de 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestUploadPurgeOnDirectoryOperations.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestUploadPurgeOnDirectoryOperations.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.fs.s3a.impl;
import java.io.IOException;
-import java.nio.charset.StandardCharsets;
import org.assertj.core.api.Assertions;
import org.junit.Test;
@@ -32,21 +31,20 @@ import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.performance.AbstractS3ACostTest;
import org.apache.hadoop.fs.store.audit.AuditSpan;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.assertFileHasLength;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_OPERATIONS_PURGE_UPLOADS;
import static org.apache.hadoop.fs.s3a.MultipartTestUtils.clearAnyUploads;
+import static org.apache.hadoop.fs.s3a.MultipartTestUtils.createMagicFile;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.Statistic.MULTIPART_UPLOAD_LIST;
import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_MULTIPART_UPLOAD_ABORTED;
import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_MULTIPART_UPLOAD_LIST;
import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_COMMITTER_ENABLED;
-import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_PATH_PREFIX;
import static org.apache.hadoop.util.functional.RemoteIterators.toList;
/**
* Test behavior of purging uploads in rename and delete.
+ * S3 Express tests automatically set this; it is explicitly set for the rest.
*/
public class ITestUploadPurgeOnDirectoryOperations extends AbstractS3ACostTest {
@@ -117,22 +115,6 @@ public class ITestUploadPurgeOnDirectoryOperations extends AbstractS3ACostTest {
assertUploadCount(dir, 0);
}
- /**
- * Create a magic file of "real" length more than 0 bytes long.
- * @param fs filesystem
- * @param dir directory
- * @return the path
- * @throws IOException creation failure.p
- */
- private static Path createMagicFile(final S3AFileSystem fs, final Path dir) throws IOException {
- Path magicFile = new Path(dir, MAGIC_PATH_PREFIX + "001/file.txt");
- createFile(fs, magicFile, true, "123".getBytes(StandardCharsets.UTF_8));
-
- // the file exists but is a 0 byte marker file.
- assertFileHasLength(fs, magicFile, 0);
- return magicFile;
- }
-
/**
* Assert the upload count under a dir is the expected value.
* Failure message will include the list of entries.
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestS3ExpressStorage.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestS3ExpressStorage.java
new file mode 100644
index 000000000000..2d5d69d2c90e
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestS3ExpressStorage.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static org.apache.hadoop.fs.s3a.impl.S3ExpressStorage.isS3ExpressStore;
+
+/**
+ * Test S3 Express Storage methods.
+ */
+public class TestS3ExpressStorage extends AbstractHadoopTestBase {
+
+ private static final String S3EXPRESS_BUCKET = "bucket--usw2-az2--x-s3";
+ private static final String AZ = "usw2-az2";
+
+ @Test
+ public void testS3ExpressStateDefaultEndpoint() {
+ assertS3ExpressState(S3EXPRESS_BUCKET, true, "");
+ assertS3ExpressState("bucket", false, "");
+ }
+
+ @Test
+ public void testS3ExpressStateAwsRegions() {
+ assertS3ExpressState(S3EXPRESS_BUCKET, true, "s3.amazonaws.com");
+ assertS3ExpressState(S3EXPRESS_BUCKET, true, "s3.cn-northwest-1.amazonaws.com.cn");
+ assertS3ExpressState(S3EXPRESS_BUCKET, true, "s3-fips.us-gov-east-1.amazonaws.com");
+ assertS3ExpressState(S3EXPRESS_BUCKET, true,
+ "s3-accesspoint-fips.dualstack.us-east-1.amazonaws.com");
+ assertS3ExpressState(S3EXPRESS_BUCKET, true, "s3.ca-central-1.amazonaws.com");
+ }
+
+ @Test
+ public void testS3ExpressStateThirdPartyStore() {
+ assertS3ExpressState(S3EXPRESS_BUCKET, false, "storeendpoint.example.com");
+ }
+
+ private void assertS3ExpressState(final String bucket, final boolean expected, String endpoint) {
+ Assertions.assertThat(isS3ExpressStore(bucket, endpoint))
+ .describedAs("isS3ExpressStore(%s) with endpoint %s", bucket, endpoint)
+ .isEqualTo(expected);
+ }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java
index 8dbf5baaa620..e00130047434 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.concurrent.Callable;
import java.util.stream.Collectors;
+import org.assertj.core.api.Assertions;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.s3.S3Client;
@@ -684,7 +685,8 @@ public class ITestDirectoryMarkerListing extends AbstractS3ATestBase {
}
/**
- * Expect the list of status objects to match that of the paths.
+ * Expect the list of status objects to match that of the paths,
+ * without enforcing ordering of the values.
* @param statuses status object list
* @param paths ordered varargs list of paths
* @param <T> type of status objects
@@ -692,20 +694,11 @@ public class ITestDirectoryMarkerListing extends AbstractS3ATestBase {
private <T extends FileStatus> void assertContainsExactlyStatusOfPaths(
List<T> statuses, Path... paths) {
- String actual = statuses.stream()
- .map(Object::toString)
- .collect(Collectors.joining(";"));
- String expected = Arrays.stream(paths)
- .map(Object::toString)
- .collect(Collectors.joining(";"));
- String summary = "expected [" + expected + "]"
- + " actual = [" + actual + "]";
- assertEquals("mismatch in size of listing " + summary,
- paths.length, statuses.size());
- for (int i = 0; i < statuses.size(); i++) {
- assertEquals("Path mismatch at element " + i + " in " + summary,
- paths[i], statuses.get(i).getPath());
- }
+ final List<Path> pathList = statuses.stream()
+ .map(FileStatus::getPath)
+ .collect(Collectors.toList());
+ Assertions.assertThat(pathList)
+ .containsExactlyInAnyOrder(paths);
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
index 95f22644c91a..fd7a528a5d1b 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
@@ -219,8 +219,10 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase {
cmdR.getName(),
S3A_THIS_BUCKET_DOES_NOT_EXIST
};
- intercept(UnknownStoreException.class,
- () -> cmdR.run(argsR));
+ intercept(UnknownStoreException.class, () -> {
+ final int e = cmdR.run(argsR);
+ return String.format("Outcome of %s on missing bucket: %d", tool, e);
+ });
}
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java
index 628b56704182..53d6a4e0f99c 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java
@@ -18,10 +18,12 @@
package org.apache.hadoop.fs.s3a.scale;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
+import java.util.NoSuchElementException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.IntFunction;
@@ -72,6 +74,7 @@ import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyStatis
import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsSourceToString;
import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToPrettyString;
import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_WRITE_BLOCK_UPLOADS;
+import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteIterator;
/**
* Scale test which creates a huge file.
@@ -103,8 +106,8 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
public void setup() throws Exception {
super.setup();
scaleTestDir = new Path(getTestPath(), getTestSuiteName());
- hugefile = new Path(scaleTestDir, "hugefile");
- hugefileRenamed = new Path(scaleTestDir, "hugefileRenamed");
+ hugefile = new Path(scaleTestDir, "src/hugefile");
+ hugefileRenamed = new Path(scaleTestDir, "dest/hugefile");
uploadBlockSize = uploadBlockSize();
filesize = getTestPropertyBytes(getConf(), KEY_HUGE_FILESIZE,
DEFAULT_HUGE_FILESIZE);
@@ -490,7 +493,40 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
ContractTestUtils.assertPathExists(fs, "Huge file", hugefile);
FileStatus status = fs.getFileStatus(hugefile);
ContractTestUtils.assertIsFile(hugefile, status);
+ LOG.info("Huge File Status: {}", status);
assertEquals("File size in " + status, filesize, status.getLen());
+
+ // now do some etag status checks asserting they are always the same
+ // across listing operations.
+ final Path path = hugefile;
+ final FileStatus listStatus = listFile(hugefile);
+ LOG.info("List File Status: {}", listStatus);
+
+ Assertions.assertThat(listStatus.getLen())
+ .describedAs("List file status length %s", listStatus)
+ .isEqualTo(filesize);
+ Assertions.assertThat(etag(listStatus))
+ .describedAs("List file status etag %s", listStatus)
+ .isEqualTo(etag(status));
+ }
+
+ /**
+ * Get a filestatus by listing the parent directory.
+ * @param path path
+ * @return status
+ * @throws IOException failure to read, file not found
+ */
+ private FileStatus listFile(final Path path)
+ throws IOException {
+ try {
+ return filteringRemoteIterator(
+ getFileSystem().listStatusIterator(path.getParent()),
+ st -> st.getPath().equals(path))
+ .next();
+ } catch (NoSuchElementException e) {
+ throw (FileNotFoundException)(new FileNotFoundException("Not found: " + path)
+ .initCause(e));
+ }
}
/**
@@ -509,7 +545,7 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
String filetype = encrypted ? "encrypted file" : "file";
describe("Positioned reads of %s %s", filetype, hugefile);
S3AFileSystem fs = getFileSystem();
- FileStatus status = fs.getFileStatus(hugefile);
+ FileStatus status = listFile(hugefile);
long size = status.getLen();
int ops = 0;
final int bufferSize = 8192;
@@ -518,7 +554,11 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
ContractTestUtils.NanoTimer readAtByte0, readAtByte0Again, readAtEOF;
- try (FSDataInputStream in = fs.open(hugefile, uploadBlockSize)) {
+ try (FSDataInputStream in = fs.openFile(hugefile)
+ .withFileStatus(status)
+ .opt(FS_OPTION_OPENFILE_READ_POLICY, "random")
+ .opt(FS_OPTION_OPENFILE_BUFFER_SIZE, uploadBlockSize)
+ .build().get()) {
readAtByte0 = new ContractTestUtils.NanoTimer();
in.readFully(0, buffer);
readAtByte0.end("time to read data at start of file");
@@ -661,27 +701,45 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
S3AFileSystem fs = getFileSystem();
FileStatus status = fs.getFileStatus(hugefile);
long size = status.getLen();
- fs.delete(hugefileRenamed, false);
ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
- fs.rename(hugefile, hugefileRenamed);
+ renameFile(hugefile, hugefileRenamed);
long mb = Math.max(size / _1MB, 1);
timer.end("time to rename file of %d MB", mb);
LOG.info("Time per MB to rename = {} nS",
toHuman(timer.nanosPerOperation(mb)));
bandwidth(timer, size);
+ assertPathExists("renamed file", hugefileRenamed);
logFSState();
FileStatus destFileStatus = fs.getFileStatus(hugefileRenamed);
assertEquals(size, destFileStatus.getLen());
// rename back
ContractTestUtils.NanoTimer timer2 = new ContractTestUtils.NanoTimer();
- fs.rename(hugefileRenamed, hugefile);
+ renameFile(hugefileRenamed, hugefile);
+
timer2.end("Renaming back");
LOG.info("Time per MB to rename = {} nS",
toHuman(timer2.nanosPerOperation(mb)));
bandwidth(timer2, size);
}
+ /**
+ * Rename a file.
+ * Subclasses may do this differently.
+ * @param src source file
+ * @param dest dest file
+ * @throws IOException IO failure
+ */
+ protected void renameFile(final Path src,
+ final Path dest) throws IOException {
+ final S3AFileSystem fs = getFileSystem();
+ fs.delete(dest, false);
+ final boolean renamed = fs.rename(src, dest);
+ Assertions.assertThat(renamed)
+ .describedAs("rename(%s, %s)", src, dest)
+ .isTrue();
+ }
+
/**
* Test to verify target file encryption key.
* @throws IOException
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesByteBufferBlocks.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesByteBufferBlocks.java
index b1323c49a8f6..1e74d715b88f 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesByteBufferBlocks.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesByteBufferBlocks.java
@@ -18,12 +18,20 @@
package org.apache.hadoop.fs.s3a.scale;
+import java.io.IOException;
+
+import org.assertj.core.api.Assertions;
+
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.s3a.Constants;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
import static org.apache.hadoop.fs.s3a.Constants.FAST_UPLOAD_BYTEBUFFER;
/**
* Use {@link Constants#FAST_UPLOAD_BYTEBUFFER} for buffering.
+ * This also renames by parent directory, so validates parent
+ * dir renaming of huge files.
*/
public class ITestS3AHugeFilesByteBufferBlocks
extends AbstractSTestS3AHugeFiles {
@@ -31,4 +39,25 @@ public class ITestS3AHugeFilesByteBufferBlocks
protected String getBlockOutputBufferName() {
return FAST_UPLOAD_BYTEBUFFER;
}
+
+ /**
+ * Rename the parent directory, rather than the file itself.
+ * @param src source file
+ * @param dest dest file
+ * @throws IOException
+ */
+ @Override
+ protected void renameFile(final Path src, final Path dest) throws IOException {
+
+ final S3AFileSystem fs = getFileSystem();
+
+ final Path srcDir = src.getParent();
+ final Path destDir = dest.getParent();
+ fs.delete(destDir, true);
+ fs.mkdirs(destDir, null);
+ final boolean renamed = fs.rename(srcDir, destDir);
+ Assertions.assertThat(renamed)
+ .describedAs("rename(%s, %s)", src, dest)
+ .isTrue();
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/ITestBucketTool.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/ITestBucketTool.java
new file mode 100644
index 000000000000..3025646d239b
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/ITestBucketTool.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.tools;
+
+import java.io.IOException;
+import java.net.UnknownHostException;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.AWSBadRequestException;
+import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.apache.hadoop.util.ExitUtil;
+
+import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.assumeNotS3ExpressFileSystem;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.assumeS3ExpressFileSystem;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.assumeStoreAwsHosted;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.expectErrorCode;
+import static org.apache.hadoop.fs.s3a.impl.S3ExpressStorage.STORE_CAPABILITY_S3_EXPRESS_STORAGE;
+import static org.apache.hadoop.fs.s3a.tools.BucketTool.CREATE;
+import static org.apache.hadoop.fs.s3a.tools.BucketTool.NO_ZONE_SUPPLIED;
+import static org.apache.hadoop.fs.s3a.tools.BucketTool.OPT_ENDPOINT;
+import static org.apache.hadoop.fs.s3a.tools.BucketTool.OPT_REGION;
+import static org.apache.hadoop.fs.s3a.tools.BucketTool.OPT_ZONE;
+import static org.apache.hadoop.fs.s3a.tools.BucketTool.UNSUPPORTED_ZONE_ARG;
+import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_NOT_ACCEPTABLE;
+import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_USAGE;
+import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Test {@link BucketTool}.
+ * This is a tricky little test as it doesn't want to create any new bucket,
+ * including on third party stores.
+ */
+public class ITestBucketTool extends AbstractS3ATestBase {
+
+ public static final String USWEST_2 = "us-west-2";
+
+ private static final Logger LOG = LoggerFactory.getLogger(ITestBucketTool.class);
+
+ /**
+ * This is yours.
+ */
+ public static final String OWNED = "BucketAlreadyOwnedByYouException";
+
+ /**
+ * This is not yours. Not *yet* used in any tests.
+ */
+ public static final String EXISTS = "BucketAlreadyExists";
+
+ /**
+ * Sample S3 Express bucket name. {@value}.
+ */
+ private static final String S3EXPRESS_BUCKET = "bucket--usw2-az2--x-s3";
+
+ /**
+ * Sample S3 Express bucket URI. {@value}.
+ */
+ public static final String S3A_S3EXPRESS = "s3a://" + S3EXPRESS_BUCKET;
+
+ /**
+ * US west az. {@value}
+ */
+ public static final String USWEST_AZ_2 = "usw2-az2";
+
+ private String bucketName;
+
+ private boolean s3ExpressStore;
+
+ private BucketTool bucketTool;
+
+ private String fsURI;
+
+ private String region;
+
+ private S3AFileSystem fs;
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ fs = getFileSystem();
+ final Configuration conf = fs.getConf();
+ bucketName = S3ATestUtils.getTestBucketName(conf);
+ fsURI = fs.getUri().toString();
+ s3ExpressStore = fs.hasPathCapability(new Path("/"), STORE_CAPABILITY_S3_EXPRESS_STORAGE);
+ bucketTool = new BucketTool(conf);
+ region = conf.get(AWS_REGION, "");
+ }
+
+ @Test
+ public void testRecreateTestBucketS3Express() throws Throwable {
+ assumeS3ExpressFileSystem(getFileSystem());
+ final IOException ex = intercept(IOException.class,
+ () -> bucketTool.exec("bucket", d(CREATE), d(OPT_ZONE), USWEST_AZ_2, d(OPT_REGION), region,
+ fsURI));
+ if (ex instanceof AWSBadRequestException) {
+ // owned error
+ assertExceptionContains(OWNED, ex);
+ } else if (ex instanceof UnknownHostException) {
+ // endpoint region stuff, expect the error to include the s3express endpoint
+ // name
+ assertExceptionContains("s3express-control", ex);
+ }
+ }
+
+ /**
+ * Try and recreate the test bucket.
+ * Third party stores may allow this, so skip test on them.
+ */
+ @Test
+ public void testRecreateTestBucketNonS3Express() throws Throwable {
+ assumeNotS3ExpressFileSystem(fs);
+ assumeStoreAwsHosted(fs);
+ intercept(AWSBadRequestException.class, OWNED,
+ () -> bucketTool.exec("bucket", d(CREATE),
+ d(OPT_REGION), region,
+ fsURI));
+ }
+
+ @Test
+ public void testSimpleBucketWithZoneParam() throws Throwable {
+ expectErrorCode(EXIT_USAGE,
+ intercept(ExitUtil.ExitException.class, UNSUPPORTED_ZONE_ARG, () ->
+ bucketTool.exec("bucket", d(CREATE),
+ d(OPT_ZONE), USWEST_AZ_2,
+ "s3a://simple/")));
+ }
+
+ @Test
+ public void testS3ExpressBucketWithoutZoneParam() throws Throwable {
+ expectErrorCode(EXIT_USAGE,
+ intercept(ExitUtil.ExitException.class, NO_ZONE_SUPPLIED, () ->
+ bucketTool.exec("bucket", d(CREATE),
+ S3A_S3EXPRESS)));
+ }
+
+ /**
+ * To avoid accidentally trying to create S3 Express stores on third party
+ * endpoints, the tool rejects such attempts.
+ */
+ @Test
+ public void testS3ExpressStorageOnlyOnAwsEndpoint() throws Throwable {
+ describe("The tool only supports S3 Express bucket names on aws endpoints");
+ expectErrorCode(EXIT_NOT_ACCEPTABLE,
+ intercept(ExitUtil.ExitException.class, () ->
+ bucketTool.exec("bucket", d(CREATE),
+ d(OPT_ENDPOINT), "s3.example.org",
+ S3A_S3EXPRESS)));
+ }
+
+ /**
+ * Add a dash to a string.
+ * @param s source.
+ * @return "-s"
+ */
+ private String d(String s) {
+ return "-" + s;
+ }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
index e6c3eeed3d7e..f871369ed571 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
+++ b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
@@ -108,7 +108,7 @@
<property>
<name>ireland.endpoint</name>
- <value>s3-eu-west-1.amazonaws.com</value>
+ <value>s3.eu-west-1.amazonaws.com</value>
</property>
<property>
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties b/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties
index 15c3b571b126..25247aaaabd3 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties
+++ b/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties
@@ -92,3 +92,7 @@ log4j.logger.org.apache.hadoop.fs.s3a.S3AStorageStatistics=INFO
# uncomment this to trace where context entries are set
# log4j.logger.org.apache.hadoop.fs.audit.CommonAuditContext=TRACE
+
+# uncomment this to get S3 Delete requests to return the list of deleted objects
+# log4.logger.org.apache.hadoop.fs.s3a.impl.RequestFactoryImpl=TRACE
+
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/DistCpTestUtils.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/DistCpTestUtils.java
index 624f7d5a0ef7..15211e261ee7 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/DistCpTestUtils.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/DistCpTestUtils.java
@@ -24,9 +24,13 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.tools.DistCp;
import org.apache.hadoop.util.ToolRunner;
+import java.util.Arrays;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.stream.Collectors;
+
+import org.assertj.core.api.Assertions;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
@@ -91,7 +95,9 @@ public class DistCpTestUtils {
optsArr[optsArr.length - 2] = src;
optsArr[optsArr.length - 1] = dst;
- assertEquals(exitCode,
- ToolRunner.run(conf, distCp, optsArr));
+ Assertions.assertThat(ToolRunner.run(conf, distCp, optsArr))
+ .describedAs("Exit code of distcp %s",
+ Arrays.stream(optsArr).collect(Collectors.joining(" ")))
+ .isEqualTo(exitCode);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 08/17: HADOOP-18946. TestErrorTranslation failure (#6205)
Posted by ah...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ahmar pushed a commit to branch feature-HADOOP-18073-sdk-v2-upgrade-3.6-aws
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit f9d14259c52a84c5f34be86edff56025fbf5e764
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Fri Oct 20 10:13:05 2023 +0100
HADOOP-18946. TestErrorTranslation failure (#6205)
Fixes TestErrorTranslation.testMultiObjectExceptionFilledIn() failure
which came in with HADOOP-18939.
Contributed by Steve Loughran
---
.../test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java | 2 --
1 file changed, 2 deletions(-)
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java
index dd4abe64c65c..a1fdbabb9479 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java
@@ -127,8 +127,6 @@ public class TestErrorTranslation extends AbstractHadoopTestBase {
.exception(ase)
.build();
RetryOnErrorCodeCondition retry = RetryOnErrorCodeCondition.create("");
- assertTrue("retry policy of MultiObjectException",
- retry.shouldRetry(context));
Assertions.assertThat(retry.shouldRetry(context))
.describedAs("retry policy of MultiObjectException")
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 04/17: HADOOP-18889. S3A v2 SDK third party support (#6141)
Posted by ah...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ahmar pushed a commit to branch feature-HADOOP-18073-sdk-v2-upgrade-3.6-aws
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit c11b6ab8e43d800cd75b2b34b6e7309b381817e2
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Thu Oct 12 17:47:44 2023 +0100
HADOOP-18889. S3A v2 SDK third party support (#6141)
Tune AWS v2 SDK changes based on testing with third party stores
including GCS.
Contains HADOOP-18889. S3A v2 SDK error translations and troubleshooting docs
* Changes needed to work with multiple third party stores
* New third_party_stores document on how to bind to and test
third party stores, including google gcs (which works!)
* Troubleshooting docs mostly updated for v2 SDK
Exception translation/resilience
* New AWSUnsupportedFeatureException for unsupported/unavailable errors
* Handle 501 method unimplemented as one of these
* Error codes > 500 mapped to the AWSStatus500Exception if no explicit
handler.
* Precondition errors handled a bit better
* GCS throttle exception also recognized.
* GCS raises 404 on a delete of a file which doesn't exist: swallow it.
* Error translation uses reflection to create IOE of the right type.
All IOEs at the bottom of an AWS stack chain are regenerated.
then a new exception of that specific type is created, with the top level ex
its cause. This is done to retain the whole stack chain.
* Reduce the number of retries within the AWS SDK
* And those of s3a code.
* S3ARetryPolicy explicitly declare SocketException as connectivity failure
but subclasses BindException
* SocketTimeoutException also considered connectivity
* Log at debug whenever retry policies looked up
* Reorder exceptions to alphabetical order, with commentary
* Review use of the Invoke.retry() method
The reduction in retries is because its clear when you try to create a bucket
which doesn't resolve that the time for even an UnknownHostException to
eventually fail over 90s, which then hit the s3a retry code.
- Reducing the SDK retries means these escalate to our code better.
- Cutting back on our own retries makes it a bit more responsive for most real
deployments.
- maybeTranslateNetworkException() and s3a retry policy means that
unknown host exception is recognised and fails fast.
Contributed by Steve Loughran
---
.../src/main/resources/core-default.xml | 10 +-
.../hadoop/fs/s3a/AWSBadRequestException.java | 4 +-
.../hadoop/fs/s3a/AWSStatus500Exception.java | 16 +-
...on.java => AWSUnsupportedFeatureException.java} | 20 +-
.../java/org/apache/hadoop/fs/s3a/Constants.java | 7 +-
.../org/apache/hadoop/fs/s3a/MultipartUtils.java | 8 +
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 52 +-
.../org/apache/hadoop/fs/s3a/S3ARetryPolicy.java | 125 ++--
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 39 +-
.../org/apache/hadoop/fs/s3a/S3ListResult.java | 6 +-
.../fs/s3a/SimpleAWSCredentialsProvider.java | 2 +-
.../java/org/apache/hadoop/fs/s3a/Statistic.java | 2 +-
.../apache/hadoop/fs/s3a/auth/SignerFactory.java | 42 +-
.../apache/hadoop/fs/s3a/auth/SignerManager.java | 15 +-
.../apache/hadoop/fs/s3a/impl/AWSClientConfig.java | 14 +-
.../hadoop/fs/s3a/impl/ErrorTranslation.java | 70 +-
.../fs/s3a/impl/InstantiationIOException.java | 12 +-
.../hadoop/fs/s3a/impl/InternalConstants.java | 11 +-
.../hadoop/fs/s3a/impl/RequestFactoryImpl.java | 3 +-
.../apache/hadoop/fs/s3a/s3guard/S3GuardTool.java | 6 +-
.../org/apache/hadoop/fs/s3a/tools/MarkerTool.java | 27 +-
.../markdown/tools/hadoop-aws/assumed_roles.md | 195 +-----
.../src/site/markdown/tools/hadoop-aws/index.md | 629 ++++++++++++-----
.../src/site/markdown/tools/hadoop-aws/testing.md | 78 ++-
.../tools/hadoop-aws/third_party_stores.md | 415 +++++++++++
.../tools/hadoop-aws/troubleshooting_s3a.md | 757 ++++-----------------
.../hadoop/fs/s3a/ITestS3ABucketExistence.java | 16 +-
.../apache/hadoop/fs/s3a/ITestS3ACannedACLs.java | 4 +
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 12 +-
.../hadoop/fs/s3a/ITestS3AContentEncoding.java | 39 +-
.../s3a/ITestS3AContractGetFileStatusV1List.java | 4 +
.../hadoop/fs/s3a/ITestS3AEndpointRegion.java | 99 ++-
.../hadoop/fs/s3a/ITestS3AFailureHandling.java | 21 +-
.../hadoop/fs/s3a/ITestS3AMiscOperations.java | 12 +-
.../hadoop/fs/s3a/ITestS3AMultipartUtils.java | 20 +-
.../apache/hadoop/fs/s3a/ITestS3AStorageClass.java | 7 +-
.../apache/hadoop/fs/s3a/MultipartTestUtils.java | 8 +
.../org/apache/hadoop/fs/s3a/S3ATestConstants.java | 15 +
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 57 +-
.../java/org/apache/hadoop/fs/s3a/TestInvoker.java | 3 +-
.../hadoop/fs/s3a/auth/ITestCustomSigner.java | 49 +-
.../hadoop/fs/s3a/auth/TestSignerManager.java | 31 +-
.../s3a/commit/magic/ITestS3AHugeMagicCommits.java | 16 +
.../apache/hadoop/fs/s3a/impl/ITestXAttrCost.java | 12 +-
.../hadoop/fs/s3a/impl/TestErrorTranslation.java | 115 ++++
.../fs/s3a/performance/AbstractS3ACostTest.java | 5 +-
.../performance/ITestDirectoryMarkerListing.java | 16 +-
.../fs/s3a/performance/ITestS3ADeleteCost.java | 16 +-
.../s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java | 7 +-
.../s3a/scale/ITestS3AHugeFilesStorageClass.java | 7 +-
.../hadoop-aws/src/test/resources/core-site.xml | 12 +
.../hadoop-aws/src/test/resources/log4j.properties | 8 +-
52 files changed, 1946 insertions(+), 1230 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 6af4f793732a..1a391c9cf22f 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1398,8 +1398,14 @@
<property>
<name>fs.s3a.attempts.maximum</name>
- <value>20</value>
- <description>How many times we should retry commands on transient errors.</description>
+ <value>5</value>
+ <description>
+ Number of times the AWS client library should retry errors before
+ escalating to the S3A code: {@value}.
+ The S3A connector does its own selective retries; the only time the AWS
+ SDK operations are not wrapped is during multipart copy via the AWS SDK
+ transfer manager.
+ </description>
</property>
<property>
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java
index c5867eeab4f4..dfa1725acd11 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.fs.s3a;
import software.amazon.awssdk.awscore.exception.AwsServiceException;
+import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_400_BAD_REQUEST;
+
/**
* A 400 "Bad Request" exception was received.
* This is the general "bad parameters, headers, whatever" failure.
@@ -28,7 +30,7 @@ public class AWSBadRequestException extends AWSServiceIOException {
/**
* HTTP status code which signals this failure mode was triggered: {@value}.
*/
- public static final int STATUS_CODE = 400;
+ public static final int STATUS_CODE = SC_400_BAD_REQUEST;
/**
* Instantiate.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java
index ecfe5da1455c..f7c72f853095 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java
@@ -21,13 +21,13 @@ package org.apache.hadoop.fs.s3a;
import software.amazon.awssdk.awscore.exception.AwsServiceException;
/**
- * A 500 response came back from a service.
- * This is considered <i>probably</i> retriable, That is, we assume
- * <ol>
- * <li>whatever error happened in the service itself to have happened
- * before the infrastructure committed the operation.</li>
- * <li>Nothing else got through either.</li>
- * </ol>
+ * A 5xx response came back from a service.
+ * The 500 error considered retriable by the AWS SDK, which will have already
+ * tried it {@code fs.s3a.attempts.maximum} times before reaching s3a
+ * code.
+ * How it handles other 5xx errors is unknown: S3A FS code will treat them
+ * as unrecoverable on the basis that they indicate some third-party store
+ * or gateway problem.
*/
public class AWSStatus500Exception extends AWSServiceIOException {
public AWSStatus500Exception(String operation,
@@ -37,6 +37,6 @@ public class AWSStatus500Exception extends AWSServiceIOException {
@Override
public boolean retryable() {
- return true;
+ return false;
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSUnsupportedFeatureException.java
similarity index 70%
copy from hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java
copy to hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSUnsupportedFeatureException.java
index c5867eeab4f4..dd1cebf45f9c 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSUnsupportedFeatureException.java
@@ -21,22 +21,26 @@ package org.apache.hadoop.fs.s3a;
import software.amazon.awssdk.awscore.exception.AwsServiceException;
/**
- * A 400 "Bad Request" exception was received.
- * This is the general "bad parameters, headers, whatever" failure.
+ * A store returned an error indicating that it does not support a
+ * specific S3 feature such as the chosen ChangeDetectionPolicy or
+ * other AWS-S3 feature that the third-party store does not support.
+ * The workaround is to disable use of the feature.
+ * Unrecoverable.
*/
-public class AWSBadRequestException extends AWSServiceIOException {
- /**
- * HTTP status code which signals this failure mode was triggered: {@value}.
- */
- public static final int STATUS_CODE = 400;
+public class AWSUnsupportedFeatureException extends AWSServiceIOException {
/**
* Instantiate.
* @param operation operation which triggered this
* @param cause the underlying cause
*/
- public AWSBadRequestException(String operation,
+ public AWSUnsupportedFeatureException(String operation,
AwsServiceException cause) {
super(operation, cause);
}
+
+ @Override
+ public boolean retryable() {
+ return false;
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index e52e4a711a6d..aa836ffb6fd1 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -228,6 +228,9 @@ public final class Constants {
/**
* Number of times the AWS client library should retry errors before
* escalating to the S3A code: {@value}.
+ * The S3A connector does its own selective retries; the only time the AWS
+ * SDK operations are not wrapped is during multipart copy via the AWS SDK
+ * transfer manager.
*/
public static final String MAX_ERROR_RETRIES = "fs.s3a.attempts.maximum";
@@ -235,7 +238,7 @@ public final class Constants {
* Default number of times the AWS client library should retry errors before
* escalating to the S3A code: {@value}.
*/
- public static final int DEFAULT_MAX_ERROR_RETRIES = 10;
+ public static final int DEFAULT_MAX_ERROR_RETRIES = 5;
/**
* Experimental/Unstable feature: should the AWS client library retry
@@ -264,7 +267,7 @@ public final class Constants {
// milliseconds until we give up trying to establish a connection to s3
public static final String ESTABLISH_TIMEOUT =
"fs.s3a.connection.establish.timeout";
- public static final int DEFAULT_ESTABLISH_TIMEOUT = 50000;
+ public static final int DEFAULT_ESTABLISH_TIMEOUT = 5000;
// milliseconds until we give up on a connection to s3
public static final String SOCKET_TIMEOUT = "fs.s3a.connection.timeout";
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java
index 296ec18dcf18..efca093204c2 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java
@@ -223,6 +223,14 @@ public final class MultipartUtils {
/** Iterator over the current listing. */
private ListIterator<MultipartUpload> batchIterator;
+ /**
+ * Construct an iterator to list uploads under a path.
+ * @param storeContext store context
+ * @param s3 s3 client
+ * @param maxKeys max # of keys to list per batch
+ * @param prefix prefix
+ * @throws IOException listing failure.
+ */
@Retries.RetryTranslated
public UploadIterator(
final StoreContext storeContext,
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index da828123ec77..f6a9df4cdb27 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -334,6 +334,10 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
public static final Logger LOG = LoggerFactory.getLogger(S3AFileSystem.class);
/** Exactly once log to warn about setting the region in config to avoid probe. */
private static final LogExactlyOnce SET_REGION_WARNING = new LogExactlyOnce(LOG);
+
+ /** Log to warn of storage class configuration problems. */
+ private static final LogExactlyOnce STORAGE_CLASS_WARNING = new LogExactlyOnce(LOG);
+
private static final Logger PROGRESS =
LoggerFactory.getLogger("org.apache.hadoop.fs.s3a.S3AFileSystem.Progress");
private LocalDirAllocator directoryAllocator;
@@ -1075,7 +1079,8 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
if (exception.statusCode() == SC_404_NOT_FOUND) {
throw new UnknownStoreException("s3a://" + bucket + "/",
- " Bucket does " + "not exist");
+ " Bucket does not exist: " + exception,
+ exception);
}
throw exception;
@@ -1176,6 +1181,9 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
// Any encoding type
String contentEncoding = getConf().getTrimmed(CONTENT_ENCODING, null);
+ if (contentEncoding != null) {
+ LOG.debug("Using content encoding set in {} = {}", CONTENT_ENCODING, contentEncoding);
+ }
String storageClassConf = getConf()
.getTrimmed(STORAGE_CLASS, "")
@@ -1183,10 +1191,11 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
StorageClass storageClass = null;
if (!storageClassConf.isEmpty()) {
storageClass = StorageClass.fromValue(storageClassConf);
-
+ LOG.debug("Using storage class {}", storageClass);
if (storageClass.equals(StorageClass.UNKNOWN_TO_SDK_VERSION)) {
- LOG.warn("Unknown storage class property {}: {}; falling back to default storage class",
- STORAGE_CLASS, storageClassConf);
+ STORAGE_CLASS_WARNING.warn("Unknown storage class \"{}\" from option: {};"
+ + " falling back to default storage class",
+ storageClassConf, STORAGE_CLASS);
storageClass = null;
}
@@ -1432,7 +1441,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
public String getBucketLocation(String bucketName) throws IOException {
final String region = trackDurationAndSpan(
STORE_EXISTS_PROBE, bucketName, null, () ->
- invoker.retry("getBucketLocation()", bucketName, true, () ->
+ once("getBucketLocation()", bucketName, () ->
// If accessPoint then region is known from Arn
accessPoint != null
? accessPoint.getRegion()
@@ -2990,7 +2999,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
"deleting %s", key)) {
invoker.retryUntranslated(String.format("Delete %s:/%s", bucket, key),
DELETE_CONSIDERED_IDEMPOTENT,
- ()-> {
+ () -> {
incrementStatistic(OBJECT_DELETE_OBJECTS);
trackDurationOfInvocation(getDurationTrackerFactory(),
OBJECT_DELETE_REQUEST.getSymbol(),
@@ -2999,6 +3008,12 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
.build()));
return null;
});
+ } catch (AwsServiceException ase) {
+ // 404 errors get swallowed; this can be raised by
+ // third party stores (GCS).
+ if (!isObjectNotFound(ase)) {
+ throw ase;
+ }
}
}
@@ -4284,13 +4299,13 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
}
/**
- * Verify that the input stream is open. Non blocking; this gives
+ * Verify that the filesystem has not been closed. Non blocking; this gives
* the last state of the volatile {@link #closed} field.
- * @throws IOException if the connection is closed.
+ * @throws PathIOException if the FS is closed.
*/
- private void checkNotClosed() throws IOException {
+ private void checkNotClosed() throws PathIOException {
if (isClosed) {
- throw new IOException(uri + ": " + E_FS_CLOSED);
+ throw new PathIOException(uri.toString(), E_FS_CLOSED);
}
}
@@ -4440,7 +4455,6 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
// This means the File was deleted since LIST enumerated it.
LOG.debug("getObjectMetadata({}) failed to find an expected file",
srcKey, e);
- // We create an exception, but the text depends on the S3Guard state
throw new RemoteFileChangedException(
keyToQualifiedPath(srcKey).toString(),
action,
@@ -4451,6 +4465,8 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
CopyObjectRequest.Builder copyObjectRequestBuilder =
getRequestFactory().newCopyObjectRequestBuilder(srcKey, dstKey, srcom);
changeTracker.maybeApplyConstraint(copyObjectRequestBuilder);
+ final CopyObjectRequest copyRequest = copyObjectRequestBuilder.build();
+ LOG.debug("Copy Request: {}", copyRequest);
CopyObjectResponse response;
// transfer manager is skipped if disabled or the file is too small to worry about
@@ -4465,7 +4481,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
Copy copy = transferManager.copy(
CopyRequest.builder()
- .copyObjectRequest(copyObjectRequestBuilder.build())
+ .copyObjectRequest(copyRequest)
.build());
try {
@@ -4474,6 +4490,8 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
} catch (CompletionException e) {
Throwable cause = e.getCause();
if (cause instanceof SdkException) {
+ // if this is a 412 precondition failure, it may
+ // be converted to a RemoteFileChangedException
SdkException awsException = (SdkException)cause;
changeTracker.processException(awsException, "copy");
throw awsException;
@@ -4490,7 +4508,15 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
() -> {
LOG.debug("copyFile: single part copy {} -> {} of size {}", srcKey, dstKey, size);
incrementStatistic(OBJECT_COPY_REQUESTS);
- return s3Client.copyObject(copyObjectRequestBuilder.build());
+ try {
+ return s3Client.copyObject(copyRequest);
+ } catch (SdkException awsException) {
+ // if this is a 412 precondition failure, it may
+ // be converted to a RemoteFileChangedException
+ changeTracker.processException(awsException, "copy");
+ // otherwise, rethrow
+ throw awsException;
+ }
});
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
index fdb4591476c5..cc30da4fbe04 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
@@ -22,7 +22,10 @@ import java.io.EOFException;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InterruptedIOException;
+import java.net.BindException;
+import java.net.ConnectException;
import java.net.NoRouteToHostException;
+import java.net.SocketException;
import java.net.SocketTimeoutException;
import java.net.UnknownHostException;
import java.nio.file.AccessDeniedException;
@@ -31,7 +34,6 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import software.amazon.awssdk.core.exception.SdkException;
-import org.apache.hadoop.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,13 +47,14 @@ import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.net.ConnectTimeoutException;
+import static java.util.Objects.requireNonNull;
import static org.apache.hadoop.io.retry.RetryPolicies.*;
-
import static org.apache.hadoop.fs.s3a.Constants.*;
+
/**
* The S3A request retry policy.
- *
+ * <p>
* This uses the retry options in the configuration file to determine retry
* count and delays for "normal" retries and separately, for throttling;
* the latter is best handled for longer with an exponential back-off.
@@ -66,20 +69,25 @@ import static org.apache.hadoop.fs.s3a.Constants.*;
* For non-idempotent operations, only failures due to throttling or
* from failures which are known to only arise prior to talking to S3
* are retried.
- *
+ * <p>
* The retry policy is all built around that of the normal IO exceptions,
* particularly those extracted from
* {@link S3AUtils#translateException(String, Path, SdkException)}.
* Because the {@link #shouldRetry(Exception, int, int, boolean)} method
* does this translation if an {@code SdkException} is processed,
* the policy defined for the IOEs also applies to the original exceptions.
- *
+ * <p>
* Put differently: this retry policy aims to work for handlers of the
* untranslated exceptions, as well as the translated ones.
+ * <p>
+ * Note that because delete is considered idempotent, all s3a operations currently
+ * declare themselves idempotent.
+ * This means the retry policy here is more complex than it needs to be -but it
+ * does force us to consider when retrying operations would not be safe.
* @see <a href="http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html">S3 Error responses</a>
- * @see <a href="http://docs.aws.amazon.com/AmazonS3/latest/dev/ErrorBestPractices.html">Amazon S3 Error Best Practices</a>
+ * @see <a href="http://docs.aws.amazon.com/AmazonS3/latest/dev/ErrorBestPractices.html"> Amazon S3 Error Best Practices</a>
*/
-@SuppressWarnings("visibilitymodifier") // I want a struct of finals, for real.
+@SuppressWarnings("visibilitymodifier") // we want a struct of finals, for real.
public class S3ARetryPolicy implements RetryPolicy {
private static final Logger LOG = LoggerFactory.getLogger(
@@ -122,8 +130,7 @@ public class S3ARetryPolicy implements RetryPolicy {
* @param conf configuration to read.
*/
public S3ARetryPolicy(Configuration conf) {
- Preconditions.checkArgument(conf != null, "Null configuration");
- this.configuration = conf;
+ this.configuration = requireNonNull(conf, "Null configuration");
// base policy from configuration
int limit = conf.getInt(RETRY_LIMIT, RETRY_LIMIT_DEFAULT);
@@ -188,35 +195,57 @@ public class S3ARetryPolicy implements RetryPolicy {
// inherit policies.
Map<Class<? extends Exception>, RetryPolicy> policyMap = new HashMap<>();
- // failfast exceptions which we consider unrecoverable
- policyMap.put(UnknownHostException.class, fail);
- policyMap.put(NoRouteToHostException.class, fail);
- policyMap.put(InterruptedException.class, fail);
- // note this does not pick up subclasses (like socket timeout)
- policyMap.put(InterruptedIOException.class, fail);
// Access denial and auth exceptions are not retried
policyMap.put(AccessDeniedException.class, fail);
- policyMap.put(NoAuthWithAWSException.class, fail);
- policyMap.put(FileNotFoundException.class, fail);
- policyMap.put(UnknownStoreException.class, fail);
- policyMap.put(InvalidRequestException.class, fail);
- // once the file has changed, trying again is not going to help
- policyMap.put(RemoteFileChangedException.class, fail);
+ // Treated as an immediate failure
+ policyMap.put(AWSBadRequestException.class, fail);
- // likely only recovered by changing the policy configuration or s3
- // implementation
- policyMap.put(NoVersionAttributeException.class, fail);
+ // use specific retry policy for aws client exceptions
+ // nested IOExceptions will already have been extracted and used
+ // in this map.
+ policyMap.put(AWSClientIOException.class, retryAwsClientExceptions);
+
+ // server didn't respond.
+ policyMap.put(AWSNoResponseException.class, retryIdempotentCalls);
// should really be handled by resubmitting to new location;
// that's beyond the scope of this retry policy
policyMap.put(AWSRedirectException.class, fail);
+ // generic exception from the service
+ policyMap.put(AWSServiceIOException.class, retryAwsClientExceptions);
+
// throttled requests are can be retried, always
policyMap.put(AWSServiceThrottledException.class, throttlePolicy);
+ // Status 5xx error code is an immediate failure
+ // this is sign of a server-side problem, and while
+ // rare in AWS S3, it does happen on third party stores.
+ // (out of disk space, etc).
+ // by the time we get here, the aws sdk will have
+ // already retried.
+ // there is specific handling for some 5XX codes (501, 503);
+ // this is for everything else
+ policyMap.put(AWSStatus500Exception.class, fail);
+
+ // subclass of AWSServiceIOException whose cause is always S3Exception
+ policyMap.put(AWSS3IOException.class, retryIdempotentCalls);
+
+ // server doesn't support a feature.
+ // raised from a number of HTTP error codes -mostly from
+ // third-party stores which only support a subset of AWS S3
+ // operations.
+ policyMap.put(AWSUnsupportedFeatureException.class, fail);
+
+ // socket exception subclass we consider unrecoverable
+ // though this is normally only found when opening a port for listening.
+ // which is never done in S3A.
+ policyMap.put(BindException.class, fail);
+
// connectivity problems are retried without worrying about idempotency
policyMap.put(ConnectTimeoutException.class, connectivityFailure);
+ policyMap.put(ConnectException.class, connectivityFailure);
// this can be a sign of an HTTP connection breaking early.
// which can be reacted to by another attempt if the request was idempotent.
@@ -224,27 +253,38 @@ public class S3ARetryPolicy implements RetryPolicy {
// which isn't going to be recovered from
policyMap.put(EOFException.class, retryIdempotentCalls);
- // policy on a 400/bad request still ambiguous.
- // Treated as an immediate failure
- policyMap.put(AWSBadRequestException.class, fail);
+ // object not found. 404 when not unknown bucket; 410 "gone"
+ policyMap.put(FileNotFoundException.class, fail);
- // Status 500 error code is also treated as a connectivity problem
- policyMap.put(AWSStatus500Exception.class, connectivityFailure);
+ // Interrupted, usually by other threads
+ policyMap.put(InterruptedException.class, fail);
+ // note this does not pick up subclasses (like socket timeout)
+ policyMap.put(InterruptedIOException.class, fail);
+ policyMap.put(InvalidRequestException.class, fail);
- // server didn't respond.
- policyMap.put(AWSNoResponseException.class, retryIdempotentCalls);
+ // auth failure. Possibly recoverable by reattempting with
+ // the credential provider, but not covered here.
+ policyMap.put(NoAuthWithAWSException.class, fail);
- // use specific retry policy for aws client exceptions
- policyMap.put(AWSClientIOException.class, retryAwsClientExceptions);
- policyMap.put(AWSServiceIOException.class, retryAwsClientExceptions);
+ // network routing.
+ policyMap.put(NoRouteToHostException.class, fail);
- // other operations
- policyMap.put(AWSS3IOException.class, retryIdempotentCalls);
- policyMap.put(SocketTimeoutException.class, retryIdempotentCalls);
+ // likely only recovered by changing the policy configuration or s3
+ // implementation
+ policyMap.put(NoVersionAttributeException.class, fail);
+ // once the file has changed, trying again is not going to help
+ policyMap.put(RemoteFileChangedException.class, fail);
+ // general socket exceptions
+ policyMap.put(SocketException.class, connectivityFailure);
+ policyMap.put(SocketTimeoutException.class, connectivityFailure);
+ // assume that DNS wil not recover; SDK is likely to have retried.
+ policyMap.put(UnknownHostException.class, fail);
+ policyMap.put(UnknownStoreException.class, fail);
// Unsupported requests do not work, however many times you try
policyMap.put(UnsupportedRequestException.class, fail);
+
return policyMap;
}
@@ -253,14 +293,19 @@ public class S3ARetryPolicy implements RetryPolicy {
int retries,
int failovers,
boolean idempotent) throws Exception {
- Preconditions.checkArgument(exception != null, "Null exception");
- Exception ex = exception;
+ Exception ex = requireNonNull(exception, "Null exception");
if (exception instanceof SdkException) {
// update the sdk exception for the purpose of exception
// processing.
ex = S3AUtils.translateException("", "", (SdkException) exception);
}
- return retryPolicy.shouldRetry(ex, retries, failovers, idempotent);
+ LOG.debug("Retry probe for {} with {} retries and {} failovers,"
+ + " idempotent={}, due to {}",
+ ex.getClass().getSimpleName(), retries, failovers, idempotent, ex, ex);
+ // look in the retry policy map
+ final RetryAction action = retryPolicy.shouldRetry(ex, retries, failovers, idempotent);
+ LOG.debug("Retry action is {}", action);
+ return action;
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index 62be975b656f..a8a7172d8a43 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -80,6 +80,7 @@ import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.isAbstract;
import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.isNotInstanceOf;
import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.unsupportedConstructor;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.*;
+import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractNetworkException;
import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteIterator;
@@ -189,10 +190,14 @@ public final class S3AUtils {
ioe = maybeTranslateCredentialException(path, exception);
if (ioe != null) {
return ioe;
- } else {
- // no custom handling.
- return new AWSClientIOException(message, exception);
}
+ // network problems covered by an IOE inside the exception chain.
+ ioe = maybeExtractNetworkException(path, exception);
+ if (ioe != null) {
+ return ioe;
+ }
+ // no custom handling.
+ return new AWSClientIOException(message, exception);
} else {
// "error response returned by an S3 or other service."
// These contain more details and should be translated based
@@ -207,6 +212,8 @@ public final class S3AUtils {
if (ase.awsErrorDetails() != null) {
message = message + ":" + ase.awsErrorDetails().errorCode();
}
+
+ // big switch on the HTTP status code.
switch (status) {
case SC_301_MOVED_PERMANENTLY:
@@ -240,7 +247,8 @@ public final class S3AUtils {
// this is a missing bucket
ioe = new UnknownStoreException(path, message, ase);
} else {
- // a normal unknown object
+ // a normal unknown object.
+ // Can also be raised by third-party stores when aborting an unknown multipart upload
ioe = new FileNotFoundException(message);
ioe.initCause(ase);
}
@@ -253,10 +261,13 @@ public final class S3AUtils {
ioe.initCause(ase);
break;
- // method not allowed; seen on S3 Select.
- // treated as a bad request
+ // errors which stores can return from requests which
+ // the store does not support.
case SC_405_METHOD_NOT_ALLOWED:
- ioe = new AWSBadRequestException(message, s3Exception);
+ case SC_412_PRECONDITION_FAILED:
+ case SC_415_UNSUPPORTED_MEDIA_TYPE:
+ case SC_501_NOT_IMPLEMENTED:
+ ioe = new AWSUnsupportedFeatureException(message, s3Exception);
break;
// out of range. This may happen if an object is overwritten with
@@ -275,7 +286,8 @@ public final class S3AUtils {
break;
// throttling
- case SC_503_SERVICE_UNAVAILABLE:
+ case SC_429_TOO_MANY_REQUESTS_GCS: // google cloud through this connector
+ case SC_503_SERVICE_UNAVAILABLE: // AWS
ioe = new AWSServiceThrottledException(message, ase);
break;
@@ -293,8 +305,15 @@ public final class S3AUtils {
// other 200: FALL THROUGH
default:
- // no specific exit code. Choose an IOE subclass based on the class
- // of the caught exception
+ // no specifically handled exit code.
+
+ // convert all unknown 500+ errors to a 500 exception
+ if (status > SC_500_INTERNAL_SERVER_ERROR) {
+ ioe = new AWSStatus500Exception(message, ase);
+ break;
+ }
+
+ // Choose an IOE subclass based on the class of the caught exception
ioe = s3Exception != null
? new AWSS3IOException(message, s3Exception)
: new AWSServiceIOException(message, ase);
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java
index c77311211abc..0300b78bec25 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java
@@ -29,6 +29,8 @@ import software.amazon.awssdk.services.s3.model.S3Object;
import org.slf4j.Logger;
+import static java.util.Objects.requireNonNull;
+
/**
* API version-independent container for S3 List responses.
*/
@@ -47,7 +49,7 @@ public class S3ListResult {
* @return new list result container
*/
public static S3ListResult v1(ListObjectsResponse result) {
- return new S3ListResult(result, null);
+ return new S3ListResult(requireNonNull(result), null);
}
/**
@@ -56,7 +58,7 @@ public class S3ListResult {
* @return new list result container
*/
public static S3ListResult v2(ListObjectsV2Response result) {
- return new S3ListResult(null, result);
+ return new S3ListResult(null, requireNonNull(result));
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
index 13c951e83b9c..087d034975ad 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
@@ -91,7 +91,7 @@ public class SimpleAWSCredentialsProvider implements AwsCredentialsProvider {
public String toString() {
return "SimpleAWSCredentialsProvider{" +
"accessKey.empty=" + accessKey.isEmpty() +
- ", secretKey.empty'" + secretKey.isEmpty() +
+ ", secretKey.empty=" + secretKey.isEmpty() +
'}';
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
index b5f7fcbcd016..f54113af6c5c 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
@@ -582,7 +582,7 @@ public enum Statistic {
TYPE_COUNTER),
MULTIPART_UPLOAD_ABORT_UNDER_PATH_INVOKED(
StoreStatisticNames.MULTIPART_UPLOAD_ABORT_UNDER_PATH_INVOKED,
- "Multipart Upload Abort Unner Path Invoked",
+ "Multipart Upload Abort Under Path Invoked",
TYPE_COUNTER),
MULTIPART_UPLOAD_COMPLETED(
StoreStatisticNames.MULTIPART_UPLOAD_COMPLETED,
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
index c786086947fa..5d34688cebe1 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
@@ -31,7 +31,10 @@ import software.amazon.awssdk.core.signer.NoOpSigner;
import software.amazon.awssdk.core.signer.Signer;
import org.apache.hadoop.fs.s3a.S3AUtils;
+import org.apache.hadoop.fs.s3a.impl.InstantiationIOException;
+import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.unavailable;
+import static org.apache.hadoop.util.Preconditions.checkArgument;
/**
* Signer factory used to register and create signers.
@@ -44,6 +47,9 @@ public final class SignerFactory {
public static final String NO_OP_SIGNER = "NoOpSignerType";
private static final String S3_V4_SIGNER = "AWSS3V4SignerType";
+ /** The v2 signer is no longer available: {@value}. */
+ public static final String S3_V2_SIGNER = "S3SignerType";
+
private static final Map<String, Class<? extends Signer>> SIGNERS
= new ConcurrentHashMap<>();
@@ -69,12 +75,8 @@ public final class SignerFactory {
final String signerType,
final Class<? extends Signer> signerClass) {
- if (signerType == null) {
- throw new IllegalArgumentException("signerType cannot be null");
- }
- if (signerClass == null) {
- throw new IllegalArgumentException("signerClass cannot be null");
- }
+ checkArgument(signerType != null, "signerType cannot be null");
+ checkArgument(signerClass != null, "signerClass cannot be null");
SIGNERS.put(signerType, signerClass);
}
@@ -82,14 +84,21 @@ public final class SignerFactory {
/**
* Check if the signer has already been registered.
* @param signerType signer to get
+ * @throws IllegalArgumentException if the signer type is unknown.
*/
public static void verifySignerRegistered(String signerType) {
- Class<? extends Signer> signerClass = SIGNERS.get(signerType);
- if (signerClass == null) {
- throw new IllegalArgumentException("unknown signer type: " + signerType);
- }
+ checkArgument(isSignerRegistered(signerType),
+ "unknown signer type: %s", signerType);
}
+ /**
+ * Check if the signer has already been registered.
+ * @param signerType signer to get
+ * @return true if the signer is registered.
+ */
+ public static boolean isSignerRegistered(String signerType) {
+ return SIGNERS.containsKey(signerType);
+ }
/**
* Create an instance of the given signer.
@@ -97,13 +106,22 @@ public final class SignerFactory {
* @param signerType The signer type.
* @param configKey Config key used to configure the signer.
* @return The new signer instance.
- * @throws IOException on any problem.
+ * @throws InstantiationIOException instantiation problems.
+ * @throws IOException on any other problem.
+ *
*/
public static Signer createSigner(String signerType, String configKey) throws IOException {
+ if (S3_V2_SIGNER.equals(signerType)) {
+ throw unavailable(null, null, configKey, S3_V2_SIGNER + " is no longer supported");
+ }
+ if (!isSignerRegistered(signerType)) {
+ throw unavailable(null, null, configKey, "unknown signer type: " + signerType);
+ }
Class<?> signerClass = SIGNERS.get(signerType);
+
String className = signerClass.getName();
- LOG.debug("Signer class is {}", className);
+ LOG.debug("Signer class from {} and key {} is {}", signerType, configKey, className);
Signer signer =
S3AUtils.getInstanceFromReflection(className, null, null, Signer.class, "create",
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java
index 198bc66133a9..f0a71007b4cf 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java
@@ -109,17 +109,20 @@ public class SignerManager implements Closeable {
}
}
- /*
- * Make sure the signer class is registered once with the AWS SDK
+ /**
+ * Make sure the signer class is registered once with the AWS SDK.
+ * @param signerName signer name
+ * @param signerClassName classname
+ * @param conf source configuration
+ * @throws RuntimeException if the class is not found
*/
private static void maybeRegisterSigner(String signerName,
String signerClassName, Configuration conf) {
- try {
- SignerFactory.verifySignerRegistered(signerName);
- } catch (IllegalArgumentException e) {
+
+ if (!SignerFactory.isSignerRegistered(signerName)) {
// Signer is not registered with the AWS SDK.
// Load the class and register the signer.
- Class<? extends Signer> clazz = null;
+ Class<? extends Signer> clazz;
try {
clazz = (Class<? extends Signer>) conf.getClassByName(signerClassName);
} catch (ClassNotFoundException cnfe) {
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java
index 4ff2ec0b0b9f..3400daff509b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java
@@ -117,14 +117,14 @@ public final class AWSClientConfig {
S3AUtils.intOption(conf, ESTABLISH_TIMEOUT, DEFAULT_ESTABLISH_TIMEOUT, 0);
int socketTimeout = S3AUtils.intOption(conf, SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT, 0);
- httpClientBuilder.connectionTimeout(Duration.ofSeconds(connectionEstablishTimeout));
- httpClientBuilder.socketTimeout(Duration.ofSeconds(socketTimeout));
+ httpClientBuilder.connectionTimeout(Duration.ofMillis(connectionEstablishTimeout));
+ httpClientBuilder.socketTimeout(Duration.ofMillis(socketTimeout));
// set the connection TTL irrespective of whether the connection is in use or not.
// this can balance requests over different S3 servers, and avoid failed
// connections. See HADOOP-18845.
long ttl = longOption(conf, CONNECTION_TTL, DEFAULT_CONNECTION_TTL, -1);
- httpClientBuilder.connectionTimeToLive(Duration.ofSeconds(ttl));
+ httpClientBuilder.connectionTimeToLive(Duration.ofMillis(ttl));
NetworkBinding.bindSSLChannelMode(conf, httpClientBuilder);
@@ -148,15 +148,15 @@ public final class AWSClientConfig {
S3AUtils.intOption(conf, ESTABLISH_TIMEOUT, DEFAULT_ESTABLISH_TIMEOUT, 0);
int socketTimeout = S3AUtils.intOption(conf, SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT, 0);
- httpClientBuilder.connectionTimeout(Duration.ofSeconds(connectionEstablishTimeout));
- httpClientBuilder.readTimeout(Duration.ofSeconds(socketTimeout));
- httpClientBuilder.writeTimeout(Duration.ofSeconds(socketTimeout));
+ httpClientBuilder.connectionTimeout(Duration.ofMillis(connectionEstablishTimeout));
+ httpClientBuilder.readTimeout(Duration.ofMillis(socketTimeout));
+ httpClientBuilder.writeTimeout(Duration.ofMillis(socketTimeout));
// set the connection TTL irrespective of whether the connection is in use or not.
// this can balance requests over different S3 servers, and avoid failed
// connections. See HADOOP-18845.
long ttl = longOption(conf, CONNECTION_TTL, DEFAULT_CONNECTION_TTL, -1);
- httpClientBuilder.connectionTimeToLive(Duration.ofSeconds(ttl));
+ httpClientBuilder.connectionTimeToLive(Duration.ofMillis(ttl));
// TODO: Don't think you can set a socket factory for the netty client.
// NetworkBinding.bindSSLChannelMode(conf, awsConf);
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java
index 54a91323bc2e..7b5190becc48 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java
@@ -18,8 +18,13 @@
package org.apache.hadoop.fs.s3a.impl;
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+
import software.amazon.awssdk.awscore.exception.AwsServiceException;
+import org.apache.hadoop.fs.PathIOException;
+
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_404_NOT_FOUND;
/**
@@ -35,7 +40,7 @@ import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_404_NOT_FOUND;
* The existing code las been left in S3AUtils it is to avoid cherry-picking
* problems on backports.
*/
-public class ErrorTranslation {
+public final class ErrorTranslation {
/**
* Private constructor for utility class.
@@ -66,6 +71,69 @@ public class ErrorTranslation {
return e.statusCode() == SC_404_NOT_FOUND && !isUnknownBucket(e);
}
+ /**
+ * Translate an exception if it or its inner exception is an
+ * IOException.
+ * If this condition is not met, null is returned.
+ * @param path path of operation.
+ * @param thrown exception
+ * @return a translated exception or null.
+ */
+ public static IOException maybeExtractNetworkException(String path, Throwable thrown) {
+
+ if (thrown == null) {
+ return null;
+ }
+
+ // look inside
+ Throwable cause = thrown.getCause();
+ while (cause != null && cause.getCause() != null) {
+ cause = cause.getCause();
+ }
+ if (!(cause instanceof IOException)) {
+ return null;
+ }
+
+ // the cause can be extracted to an IOE.
+ // rather than just return it, we try to preserve the stack trace
+ // of the outer exception.
+ // as a new instance is created through reflection, the
+ // class of the returned instance will be that of the innermost,
+ // unless no suitable constructor is available.
+ return wrapWithInnerIOE(path, thrown, (IOException) cause);
+
+ }
+
+ /**
+ * Given an outer and an inner exception, create a new IOE
+ * of the inner type, with the outer exception as the cause.
+ * The message is derived from both.
+ * This only works if the inner exception has a constructor which
+ * takes a string; if not a PathIOException is created.
+ * <p>
+ * See {@code NetUtils}.
+ * @param <T> type of inner exception.
+ * @param path path of the failure.
+ * @param outer outermost exception.
+ * @param inner inner exception.
+ * @return the new exception.
+ */
+ @SuppressWarnings("unchecked")
+ private static <T extends IOException> IOException wrapWithInnerIOE(
+ String path,
+ Throwable outer,
+ T inner) {
+ String msg = outer.toString() + ": " + inner.getMessage();
+ Class<? extends Throwable> clazz = inner.getClass();
+ try {
+ Constructor<? extends Throwable> ctor = clazz.getConstructor(String.class);
+ Throwable t = ctor.newInstance(msg);
+ return (T) (t.initCause(outer));
+ } catch (Throwable e) {
+ return new PathIOException(path, msg, outer);
+ }
+ }
+
/**
* AWS error codes explicitly recognized and processes specially;
* kept in their own class for isolation.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InstantiationIOException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InstantiationIOException.java
index 435db879fabf..e09571fc338d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InstantiationIOException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InstantiationIOException.java
@@ -73,12 +73,14 @@ public class InstantiationIOException extends PathIOException {
public InstantiationIOException(
Kind kind,
- @Nullable URI uri, String classname,
+ @Nullable URI uri,
+ @Nullable String classname,
@Nullable String key,
String message,
- Throwable cause) {
+ @Nullable Throwable cause) {
super(uri!= null ? uri.toString() : "",
- "Class " + classname + " " + message
+ (classname != null ? ("Class " + classname + " ") : "")
+ + message
+ (key != null ? (" (configuration key " + key + ")") : ""),
cause);
this.kind = kind;
@@ -137,8 +139,8 @@ public class InstantiationIOException extends PathIOException {
*/
public static InstantiationIOException unavailable(
@Nullable URI uri,
- String classname,
- String key,
+ @Nullable String classname,
+ @Nullable String key,
String text) {
return new InstantiationIOException(Kind.Unavailable,
uri, classname, key, text, null);
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java
index 7bf4e619e0a0..3409bd3cc79e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java
@@ -148,9 +148,15 @@ public final class InternalConstants {
/** 412 status code: Precondition Failed. */
public static final int SC_412_PRECONDITION_FAILED = 412;
+ /** 415 status code: Content type unsupported by this store. */
+ public static final int SC_415_UNSUPPORTED_MEDIA_TYPE = 415;
+
/** 416 status code: Range Not Satisfiable. */
public static final int SC_416_RANGE_NOT_SATISFIABLE = 416;
+ /** 429 status code: This is the google GCS throttle message. */
+ public static final int SC_429_TOO_MANY_REQUESTS_GCS = 429;
+
/** 443 status code: No Response (unofficial). */
public static final int SC_443_NO_RESPONSE = 443;
@@ -160,7 +166,10 @@ public final class InternalConstants {
/** 500 status code: Internal Server Error. */
public static final int SC_500_INTERNAL_SERVER_ERROR = 500;
- /** 503 status code: Service Unavailable. */
+ /** 501 status code: method not implemented. */
+ public static final int SC_501_NOT_IMPLEMENTED = 501;
+
+ /** 503 status code: Service Unavailable. on AWS S3: throttle response. */
public static final int SC_503_SERVICE_UNAVAILABLE = 503;
/** Name of the log for throttling events. Value: {@value}. */
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java
index 170bd2b3345c..0fddbe6726eb 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java
@@ -63,6 +63,7 @@ import static org.apache.commons.lang3.StringUtils.isNotEmpty;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DEFAULT_UPLOAD_PART_COUNT_LIMIT;
import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument;
import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static software.amazon.awssdk.services.s3.model.StorageClass.UNKNOWN_TO_SDK_VERSION;
/**
* The standard implementation of the request factory.
@@ -242,7 +243,7 @@ public class RequestFactoryImpl implements RequestFactory {
.metadataDirective(MetadataDirective.REPLACE)
.acl(cannedACL);
- if (srcom.storageClass() != null) {
+ if (srcom.storageClass() != null && srcom.storageClass() != UNKNOWN_TO_SDK_VERSION) {
copyObjectRequestBuilder.storageClass(srcom.storageClass());
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
index ec68168bd0ff..330b92186dde 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
@@ -24,7 +24,6 @@ import java.io.IOException;
import java.io.PrintStream;
import java.net.URI;
import java.net.URISyntaxException;
-import java.nio.file.AccessDeniedException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
@@ -400,9 +399,10 @@ public abstract class S3GuardTool extends Configured implements Tool,
println(out, "Filesystem %s", fsUri);
try {
println(out, "Location: %s", fs.getBucketLocation());
- } catch (AccessDeniedException e) {
+ } catch (IOException e) {
// Caller cannot get the location of this bucket due to permissions
- // in their role or the bucket itself.
+ // in their role or the bucket itself, or it is not an operation
+ // supported by this store.
// Note and continue.
LOG.debug("failed to get bucket location", e);
println(out, LOCATION_UNKNOWN);
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java
index 2ce08bb2552e..fc60f4b623dd 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java
@@ -35,6 +35,7 @@ import java.util.stream.Collectors;
import software.amazon.awssdk.awscore.exception.AwsServiceException;
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.fs.s3a.AWSBadRequestException;
import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -679,8 +680,30 @@ public final class MarkerTool extends S3GuardTool {
int count = 0;
boolean result = true;
- RemoteIterator<S3AFileStatus> listing = operations
- .listObjects(path, storeContext.pathToKey(path));
+
+ // the path/key stuff loses any trailing / passed in.
+ // but this may actually be needed.
+ RemoteIterator<S3AFileStatus> listing = null;
+ String listkey = storeContext.pathToKey(path);
+ if (listkey.isEmpty()) {
+ // root. always give it a path to keep ranger happy.
+ listkey = "/";
+ }
+
+ try {
+ listing = operations.listObjects(path, listkey);
+ } catch (AWSBadRequestException e) {
+ // endpoint was unhappy. this is generally unrecoverable, but some
+ // third party stores do insist on a / here.
+ LOG.debug("Failed to list \"{}\"", listkey, e);
+ // now retry with a trailing / in case that works
+ if (listkey.endsWith("/")) {
+ // already has a trailing /, so fail
+ throw e;
+ }
+ // try again.
+ listing = operations.listObjects(path, listkey + "/");
+ }
while (listing.hasNext()) {
count++;
S3AFileStatus status = listing.next();
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md
index ea53b2e1fa9e..7da23c9fe7cf 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md
@@ -371,16 +371,7 @@ The Assumed Role Credential Provider is enabled, but `fs.s3a.assumed.role.arn` i
```
java.io.IOException: Unset property fs.s3a.assumed.role.arn
at org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider.<init>(AssumedRoleCredentialProvider.java:76)
- at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
- at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
- at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
- at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
- at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:583)
- at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet(S3AUtils.java:520)
- at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.createS3Client(DefaultS3ClientFactory.java:52)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:252)
- at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3354)
- at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:474)
+
```
### <a name="not_authorized_for_assumed_role"></a> "Not authorized to perform sts:AssumeRole"
@@ -390,17 +381,9 @@ or one to which the caller has no access.
```
java.nio.file.AccessDeniedException: : Instantiate org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider
- on : com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
Not authorized to perform sts:AssumeRole (Service: AWSSecurityTokenService; Status Code: 403;
Error Code: AccessDenied; Request ID: aad4e59a-f4b0-11e7-8c78-f36aaa9457f6):AccessDenied
- at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:215)
- at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:616)
- at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet(S3AUtils.java:520)
- at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.createS3Client(DefaultS3ClientFactory.java:52)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:252)
- at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3354)
- at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:474)
- at org.apache.hadoop.fs.Path.getFileSystem(Path.java:361)
+
```
### <a name="root_account"></a> "Roles may not be assumed by root accounts"
@@ -411,31 +394,15 @@ the role.
```
java.nio.file.AccessDeniedException: : Instantiate org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider
- on : com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
+
Roles may not be assumed by root accounts. (Service: AWSSecurityTokenService; Status Code: 403; Error Code: AccessDenied;
Request ID: e86dfd8f-e758-11e7-88e7-ad127c04b5e2):
No AWS Credentials provided by AssumedRoleCredentialProvider :
- com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
+ software.amazon.awssdk.services.sts.model.StsException:
Roles may not be assumed by root accounts. (Service: AWSSecurityTokenService;
Status Code: 403; Error Code: AccessDenied; Request ID: e86dfd8f-e758-11e7-88e7-ad127c04b5e2)
- at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:215)
- at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:616)
- at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet(S3AUtils.java:520)
- at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.createS3Client(DefaultS3ClientFactory.java:52)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:252)
- at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3354)
- at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:474)
- at org.apache.hadoop.fs.Path.getFileSystem(Path.java:361)
- ... 22 more
-Caused by: com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
- Roles may not be assumed by root accounts.
- (Service: AWSSecurityTokenService; Status Code: 403; Error Code: AccessDenied;
- Request ID: e86dfd8f-e758-11e7-88e7-ad127c04b5e2)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1303)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1055)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:743)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:717)
+
+
```
### <a name="invalid_duration"></a> `Member must have value greater than or equal to 900`
@@ -444,7 +411,7 @@ The value of `fs.s3a.assumed.role.session.duration` is too low.
```
org.apache.hadoop.fs.s3a.AWSBadRequestException: request role credentials:
-com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
+software.amazon.awssdk.services.sts.model.StsException:
1 validation error detected: Value '20' at 'durationSeconds' failed to satisfy constraint:
Member must have value greater than or equal to 900 (Service: AWSSecurityTokenService;
Status Code: 400; Error Code: ValidationError;
@@ -459,7 +426,7 @@ The value of `fs.s3a.assumed.role.session.duration` is too high.
```
org.apache.hadoop.fs.s3a.AWSBadRequestException: request role credentials:
- com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
+ software.amazon.awssdk.services.sts.model.StsException:
The requested DurationSeconds exceeds the MaxSessionDuration set for this role.
(Service: AWSSecurityTokenService; Status Code: 400;
Error Code: ValidationError; Request ID: 17875165-d0a7-11e8-b85f-d15a599a7f6d)
@@ -478,7 +445,7 @@ any role for up to 12h; attempting to use a larger number will fail.
```
-Caused by: com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
+Caused by: software.amazon.awssdk.services.sts.model.StsException:
1 validation error detected:
Value '345600' at 'durationSeconds' failed to satisfy constraint:
Member must have value less than or equal to 43200
@@ -492,7 +459,7 @@ For full sessions, the duration limit is 129600 seconds: 36h.
```
org.apache.hadoop.fs.s3a.AWSBadRequestException: request session credentials:
-com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
+software.amazon.awssdk.services.sts.model.StsException:
1 validation error detected: Value '345600' at 'durationSeconds' failed to satisfy constraint:
Member must have value less than or equal to 129600
(Service: AWSSecurityTokenService; Status Code: 400; Error Code: ValidationError;
@@ -510,26 +477,12 @@ AWS specification of Role Policies.
```
org.apache.hadoop.fs.s3a.AWSBadRequestException: Instantiate org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider on :
- com.amazonaws.services.securitytoken.model.MalformedPolicyDocumentException:
+ software.amazon.awssdk.services.sts.model.MalformedPolicyDocumentException:
The policy is not in the valid JSON format. (Service: AWSSecurityTokenService; Status Code: 400;
Error Code: MalformedPolicyDocument; Request ID: baf8cb62-f552-11e7-9768-9df3b384e40c):
MalformedPolicyDocument: The policy is not in the valid JSON format.
(Service: AWSSecurityTokenService; Status Code: 400; Error Code: MalformedPolicyDocument;
Request ID: baf8cb62-f552-11e7-9768-9df3b384e40c)
- at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:209)
- at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:616)
- at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet(S3AUtils.java:520)
- at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.createS3Client(DefaultS3ClientFactory.java:52)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:252)
- at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3354)
- at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:474)
- at org.apache.hadoop.fs.Path.getFileSystem(Path.java:361)
-Caused by: com.amazonaws.services.securitytoken.model.MalformedPolicyDocumentException:
- The policy is not in the valid JSON format.
- (Service: AWSSecurityTokenService; Status Code: 400;
- Error Code: MalformedPolicyDocument; Request ID: baf8cb62-f552-11e7-9768-9df3b384e40c)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1303)
```
### <a name="policy_syntax_error"></a> `MalformedPolicyDocumentException` "Syntax errors in policy"
@@ -539,27 +492,13 @@ The policy set in `fs.s3a.assumed.role.policy` is not valid JSON.
```
org.apache.hadoop.fs.s3a.AWSBadRequestException:
Instantiate org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider on :
- com.amazonaws.services.securitytoken.model.MalformedPolicyDocumentException:
+ software.amazon.awssdk.services.sts.model.MalformedPolicyDocumentException:
Syntax errors in policy. (Service: AWSSecurityTokenService;
Status Code: 400; Error Code: MalformedPolicyDocument;
Request ID: 24a281e8-f553-11e7-aa91-a96becfb4d45):
MalformedPolicyDocument: Syntax errors in policy.
(Service: AWSSecurityTokenService; Status Code: 400; Error Code: MalformedPolicyDocument;
Request ID: 24a281e8-f553-11e7-aa91-a96becfb4d45)
- at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:209)
- at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:616)
- at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet(S3AUtils.java:520)
- at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.createS3Client(DefaultS3ClientFactory.java:52)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:252)
- at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3354)
- at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:474)
- at org.apache.hadoop.fs.Path.getFileSystem(Path.java:361)
- (Service: AWSSecurityTokenService; Status Code: 400; Error Code: MalformedPolicyDocument;
- Request ID: 24a281e8-f553-11e7-aa91-a96becfb4d45)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1303)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1055)
- ... 19 more
```
### <a name="recursive_auth"></a> `IOException`: "AssumedRoleCredentialProvider cannot be in fs.s3a.assumed.role.credentials.provider"
@@ -591,7 +530,7 @@ inner authentication which is breaking signature creation.
```
org.apache.hadoop.fs.s3a.AWSBadRequestException: Instantiate org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider
- on : com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
+ on : software.amazon.awssdk.services.sts.model.StsException:
'valid/20180109/us-east-1/sts/aws4_request' not a valid key=value pair (missing equal-sign) in Authorization header:
'AWS4-HMAC-SHA256 Credential=not valid/20180109/us-east-1/sts/aws4_request,
SignedHeaders=amz-sdk-invocation-id;amz-sdk-retry;host;user-agent;x-amz-date.
@@ -601,21 +540,7 @@ inner authentication which is breaking signature creation.
in Authorization header: 'AWS4-HMAC-SHA256 Credential=not valid/20180109/us-east-1/sts/aws4_request,
SignedHeaders=amz-sdk-invocation-id;amz-sdk-retry;host;user-agent;x-amz-date,
(Service: AWSSecurityTokenService; Status Code: 400; Error Code: IncompleteSignature;
- at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:209)
- at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:616)
- at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet(S3AUtils.java:520)
- at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.createS3Client(DefaultS3ClientFactory.java:52)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:252)
- at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3354)
- at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:474)
- at org.apache.hadoop.fs.Path.getFileSystem(Path.java:361)
-Caused by: com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
- 'valid/20180109/us-east-1/sts/aws4_request' not a valid key=value pair (missing equal-sign)
- in Authorization header: 'AWS4-HMAC-SHA256 Credential=not valid/20180109/us-east-1/sts/aws4_request,
- SignedHeaders=amz-sdk-invocation-id;amz-sdk-retry;host;user-agent;x-amz-date,
- (Service: AWSSecurityTokenService; Status Code: 400; Error Code: IncompleteSignature;
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638)
```
### <a name="invalid_token"></a> `AccessDeniedException/InvalidClientTokenId`: "The security token included in the request is invalid"
@@ -625,27 +550,11 @@ The credentials used to authenticate with the AWS Security Token Service are inv
```
[ERROR] Failures:
[ERROR] java.nio.file.AccessDeniedException: : Instantiate org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider on :
- com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
+ software.amazon.awssdk.services.sts.model.StsException:
The security token included in the request is invalid.
(Service: AWSSecurityTokenService; Status Code: 403; Error Code: InvalidClientTokenId;
Request ID: 74aa7f8a-f557-11e7-850c-33d05b3658d7):InvalidClientTokenId
- at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:215)
- at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:616)
- at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet(S3AUtils.java:520)
- at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.createS3Client(DefaultS3ClientFactory.java:52)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:252)
- at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3354)
- at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:474)
-
-Caused by: com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
-The security token included in the request is invalid.
- (Service: AWSSecurityTokenService; Status Code: 403; Error Code: InvalidClientTokenId;
- Request ID: 74aa7f8a-f557-11e7-850c-33d05b3658d7)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1303)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1055)
- ... 25 more
-```
+ ```
### <a name="invalid_session"></a> `AWSSecurityTokenServiceExceptiond`: "Member must satisfy regular expression pattern: `[\w+=,.@-]*`"
@@ -659,7 +568,7 @@ If set explicitly, it must be valid.
```
org.apache.hadoop.fs.s3a.AWSBadRequestException:
Instantiate org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider on
- com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
+ software.amazon.awssdk.services.sts.model.StsException:
1 validation error detected: Value 'Session Names cannot Hava Spaces!' at 'roleSessionName'
failed to satisfy constraint: Member must satisfy regular expression pattern: [\w+=,.@-]*
(Service: AWSSecurityTokenService; Status Code: 400; Error Code: ValidationError;
@@ -667,23 +576,7 @@ org.apache.hadoop.fs.s3a.AWSBadRequestException:
1 validation error detected: Value 'Session Names cannot Hava Spaces!' at 'roleSessionName'
failed to satisfy constraint: Member must satisfy regular expression pattern: [\w+=,.@-]*
(Service: AWSSecurityTokenService; Status Code: 400; Error Code: ValidationError;
- at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:209)
- at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:616)
- at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet(S3AUtils.java:520)
- at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.createS3Client(DefaultS3ClientFactory.java:52)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:252)
- at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3354)
- at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:474)
- at org.apache.hadoop.fs.Path.getFileSystem(Path.java:361)
-
-Caused by: com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
- 1 validation error detected: Value 'Session Names cannot Hava Spaces!' at 'roleSessionName'
- failed to satisfy constraint:
- Member must satisfy regular expression pattern: [\w+=,.@-]*
- (Service: AWSSecurityTokenService; Status Code: 400; Error Code: ValidationError;
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1303)
-```
+ ```
### <a name="access_denied"></a> `java.nio.file.AccessDeniedException` within a FileSystem API call
@@ -692,23 +585,11 @@ If an operation fails with an `AccessDeniedException`, then the role does not ha
the permission for the S3 Operation invoked during the call.
```
-java.nio.file.AccessDeniedException: s3a://bucket/readonlyDir:
- rename(s3a://bucket/readonlyDir, s3a://bucket/renameDest)
- on s3a://bucket/readonlyDir:
- com.amazonaws.services.s3.model.AmazonS3Exception: Access Denied
- (Service: Amazon S3; Status Code: 403; Error Code: AccessDenied; Request ID: 2805F2ABF5246BB1;
- S3 Extended Request ID: iEXDVzjIyRbnkAc40MS8Sjv+uUQNvERRcqLsJsy9B0oyrjHLdkRKwJ/phFfA17Kjn483KSlyJNw=),
- S3 Extended Request ID: iEXDVzjIyRbnkAc40MS8Sjv+uUQNvERRcqLsJsy9B0oyrjHLdkRKwJ/phFfA17Kjn483KSlyJNw=:AccessDenied
- at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:216)
- at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:143)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.rename(S3AFileSystem.java:853)
- ...
-Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Access Denied
- (Service: Amazon S3; Status Code: 403; Error Code: AccessDenied; Request ID: 2805F2ABF5246BB1;
- S3 Extended Request ID: iEXDVzjIyRbnkAc40MS8Sjv+uUQNvERRcqLsJsy9B0oyrjHLdkRKwJ/phFfA17Kjn483KSlyJNw=),
- S3 Extended Request ID: iEXDVzjIyRbnkAc40MS8Sjv+uUQNvERRcqLsJsy9B0oyrjHLdkRKwJ/phFfA17Kjn483KSlyJNw=
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1303)
+> hadoop fs -touch s3a://landsat-pds/a
+
+java.nio.file.AccessDeniedException: a: Writing Object on a:
+ software.amazon.awssdk.services.s3.model.S3Exception: Access Denied
+ (Service: S3, Status Code: 403, Request ID: F08EV50F85AYKF1V, Extended Request ID: 75vMz9xWNP5/lYplPSZfm/i4yQ5q0G32SIwOwfaj6a8gNCRj9tLBAqcLaaexT/aNg2DhWZQPvDU=):AccessDenied
```
This is the policy restriction behaving as intended: the caller is trying to
@@ -743,28 +624,9 @@ If the client does have write access to the bucket, verify that the caller has
```
java.nio.file.AccessDeniedException: test/testDTFileSystemClient: Writing Object on test/testDTFileSystemClient:
- com.amazonaws.services.s3.model.AmazonS3Exception: Access Denied (Service: Amazon S3; Status Code: 403;
+ software.amazon.awssdk.services.s3.model.S3Exception: Access Denied (Service: Amazon S3; Status Code: 403;
Error Code: AccessDenied; Request ID: E86544FF1D029857)
- at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:243)
- at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:111)
- at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$4(Invoker.java:314)
- at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:406)
- at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:310)
- at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:285)
- at org.apache.hadoop.fs.s3a.WriteOperationHelper.retry(WriteOperationHelper.java:150)
- at org.apache.hadoop.fs.s3a.WriteOperationHelper.putObject(WriteOperationHelper.java:460)
- at org.apache.hadoop.fs.s3a.S3ABlockOutputStream.lambda$putObject$0(S3ABlockOutputStream.java:438)
- at org.apache.hadoop.util.SemaphoredDelegatingExecutor$CallableWithPermitRelease.call(SemaphoredDelegatingExecutor.java:219)
- at org.apache.hadoop.util.SemaphoredDelegatingExecutor$CallableWithPermitRelease.call(SemaphoredDelegatingExecutor.java:219)
- at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:125)
- at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:57)
- at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:78)
- at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
- at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
- at java.lang.Thread.run(Thread.java:748)
-Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Access Denied (Service: Amazon S3; Status Code: 403;
- Error Code: AccessDenied; Request ID: E86544FF1D029857)
```
Note: the ability to read encrypted data in the store does not guarantee that the caller can encrypt new data.
@@ -779,14 +641,9 @@ This is a low-level networking error. Possible causes include:
```
org.apache.hadoop.fs.s3a.AWSClientIOException: request session credentials:
- com.amazonaws.SdkClientException:
Unable to execute HTTP request: null: Unable to execute HTTP request: null
-at com.amazonaws.thirdparty.apache.http.impl.conn.DefaultRoutePlanner.determineRoute(DefaultRoutePlanner.java:88)
-at com.amazonaws.thirdparty.apache.http.impl.client.InternalHttpClient.determineRoute(InternalHttpClient.java:124)
-at com.amazonaws.thirdparty.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:183)
-at com.amazonaws.thirdparty.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
-at com.amazonaws.thirdparty.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
+
```
### <a name="credential_scope"></a> Error "Credential should be scoped to a valid region"
@@ -800,7 +657,7 @@ Variant 1: `Credential should be scoped to a valid region, not 'us-west-1'` (or
```
java.nio.file.AccessDeniedException: : request session credentials:
-com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
+software.amazon.awssdk.services.sts.model.StsException:
Credential should be scoped to a valid region, not 'us-west-1'.
(Service: AWSSecurityTokenService; Status Code: 403; Error Code: SignatureDoesNotMatch; Request ID: d9065cc4-e2b9-11e8-8b7b-f35cb8d7aea4):SignatureDoesNotMatch
```
@@ -817,7 +674,7 @@ Variant 2: `Credential should be scoped to a valid region, not ''`
```
java.nio.file.AccessDeniedException: : request session credentials:
-com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
+software.amazon.awssdk.services.sts.model.StsException:
Credential should be scoped to a valid region, not ''. (
Service: AWSSecurityTokenService; Status Code: 403; Error Code: SignatureDoesNotMatch;
Request ID: bd3e5121-e2ac-11e8-a566-c1a4d66b6a16):SignatureDoesNotMatch
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 895c901f5284..076ea0fd5b8a 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -35,19 +35,20 @@ full details.
* [Encryption](./encryption.html)
* [Performance](./performance.html)
-* [S3Guard](./s3guard.html)
+* [The upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html)
+* [Working with Third-party S3 Stores](./third_party_stores.html)
* [Troubleshooting](./troubleshooting_s3a.html)
+* [Prefetching](./prefetching.html)
* [Controlling the S3A Directory Marker Behavior](directory_markers.html).
+* [Auditing](./auditing.html).
* [Committing work to S3 with the "S3A Committers"](./committers.html)
* [S3A Committers Architecture](./committer_architecture.html)
* [Working with IAM Assumed Roles](./assumed_roles.html)
* [S3A Delegation Token Support](./delegation_tokens.html)
* [S3A Delegation Token Architecture](delegation_token_architecture.html).
-* [Auditing](./auditing.html).
* [Auditing Architecture](./auditing_architecture.html).
* [Testing](./testing.html)
-* [Prefetching](./prefetching.html)
-* [Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html)
+* [S3Guard](./s3guard.html)
## <a name="overview"></a> Overview
@@ -79,16 +80,15 @@ and compatible implementations.
* Supports partitioned uploads for many-GB objects.
* Offers a high-performance random IO mode for working with columnar data such
as Apache ORC and Apache Parquet files.
-* Uses Amazon's Java S3 SDK with support for latest S3 features and authentication
+* Uses Amazon's Java V2 SDK with support for latest S3 features and authentication
schemes.
* Supports authentication via: environment variables, Hadoop configuration
properties, the Hadoop key management store and IAM roles.
* Supports per-bucket configuration.
* Supports S3 "Server Side Encryption" for both reading and writing:
SSE-S3, SSE-KMS and SSE-C.
+* Supports S3-CSE client side encryption.
* Instrumented with Hadoop metrics.
-* Before S3 was consistent, provided a consistent view of inconsistent storage
- through [S3Guard](./s3guard.html).
* Actively maintained by the open source community.
@@ -97,19 +97,17 @@ properties, the Hadoop key management store and IAM roles.
There other Hadoop connectors to S3. Only S3A is actively maintained by
the Hadoop project itself.
-1. Apache's Hadoop's original `s3://` client. This is no longer included in Hadoop.
1. Amazon EMR's `s3://` client. This is from the Amazon EMR team, who actively
maintain it.
-1. Apache's Hadoop's [`s3n:` filesystem client](./s3n.html).
- This connector is no longer available: users must migrate to the newer `s3a:` client.
+
## <a name="getting_started"></a> Getting Started
S3A depends upon two JARs, alongside `hadoop-common` and its dependencies.
-* `hadoop-aws` JAR.
-* `aws-java-sdk-bundle` JAR.
+* `hadoop-aws` JAR. This contains the S3A connector.
+* `bundle` JAR. This contains the full shaded AWS V2 SDK.
The versions of `hadoop-common` and `hadoop-aws` must be identical.
@@ -189,7 +187,8 @@ of work, as opposed to HDFS or other "real" filesystem.
The [S3A committers](./committers.html) are the sole mechanism available
to safely save the output of queries directly into S3 object stores
-through the S3A filesystem.
+through the S3A filesystem when the filesystem structure is
+how the table is represented.
### Warning #2: Object stores have different authorization models
@@ -199,10 +198,7 @@ authorization model of HDFS and traditional file systems.
The S3A client simply reports stub information from APIs that would query this metadata:
* File owner is reported as the current user.
-* File group also is reported as the current user. Prior to Apache Hadoop
-2.8.0, file group was reported as empty (no group associated), which is a
-potential incompatibility problem for scripts that perform positional parsing of
-shell output and other clients that expect to find a well-defined group.
+* File group also is reported as the current user.
* Directory permissions are reported as 777.
* File permissions are reported as 666.
@@ -239,11 +235,6 @@ However, with the upcoming upgrade to AWS Java SDK V2, these classes will need t
updated to implement `software.amazon.awssdk.auth.credentials.AwsCredentialsProvider`.
For more information see [Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html).
-*Important*: The S3A connector no longer supports username and secrets
-in URLs of the form `s3a://key:secret@bucket/`.
-It is near-impossible to stop those secrets being logged —which is why
-a warning has been printed since Hadoop 2.8 whenever such a URL was used.
-
### Authentication properties
```xml
@@ -369,13 +360,13 @@ There are a number of AWS Credential Providers inside the `hadoop-aws` JAR:
| `org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider` | EC2/k8s instance credentials |
-There are also many in the Amazon SDKs, with the common ones being.
+There are also many in the Amazon SDKs, with the common ones being as follows
-| classname | description |
-|-----------|-------------|
-| `software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider` | AWS Environment Variables |
-| `software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider`| EC2 Metadata Credentials |
-| `software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider`| EC2/k8s Metadata Credentials |
+| classname | description |
+|----------------------------------------------------------------------------------|------------------------------|
+| `software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider` | AWS Environment Variables |
+| `software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider` | EC2 Metadata Credentials |
+| `software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider` | EC2/k8s Metadata Credentials |
@@ -716,16 +707,145 @@ Here are the S3A properties for use in production; some testing-related
options are covered in [Testing](./testing.md).
```xml
+
+<property>
+ <name>fs.s3a.aws.credentials.provider</name>
+ <value>
+ org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider,
+ org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,
+ software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider,
+ org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider
+ </value>
+ <description>
+ Comma-separated class names of credential provider classes which implement
+ software.amazon.awssdk.auth.credentials.AwsCredentialsProvider.
+
+ When S3A delegation tokens are not enabled, this list will be used
+ to directly authenticate with S3 and other AWS services.
+ When S3A Delegation tokens are enabled, depending upon the delegation
+ token binding it may be used
+ to communicate wih the STS endpoint to request session/role
+ credentials.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.security.credential.provider.path</name>
+ <value />
+ <description>
+ Optional comma separated list of credential providers, a list
+ which is prepended to that set in hadoop.security.credential.provider.path
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.assumed.role.arn</name>
+ <value />
+ <description>
+ AWS ARN for the role to be assumed.
+ Required if the fs.s3a.aws.credentials.provider contains
+ org.apache.hadoop.fs.s3a.AssumedRoleCredentialProvider
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.assumed.role.session.name</name>
+ <value />
+ <description>
+ Session name for the assumed role, must be valid characters according to
+ the AWS APIs.
+ Only used if AssumedRoleCredentialProvider is the AWS credential provider.
+ If not set, one is generated from the current Hadoop/Kerberos username.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.assumed.role.policy</name>
+ <value/>
+ <description>
+ JSON policy to apply to the role.
+ Only used if AssumedRoleCredentialProvider is the AWS credential provider.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.assumed.role.session.duration</name>
+ <value>30m</value>
+ <description>
+ Duration of assumed roles before a refresh is attempted.
+ Used when session tokens are requested.
+ Range: 15m to 1h
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.assumed.role.sts.endpoint</name>
+ <value/>
+ <description>
+ AWS Security Token Service Endpoint.
+ If unset, uses the default endpoint.
+ Only used if AssumedRoleCredentialProvider is the AWS credential provider.
+ Used by the AssumedRoleCredentialProvider and in Session and Role delegation
+ tokens.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.assumed.role.sts.endpoint.region</name>
+ <value></value>
+ <description>
+ AWS Security Token Service Endpoint's region;
+ Needed if fs.s3a.assumed.role.sts.endpoint points to an endpoint
+ other than the default one and the v4 signature is used.
+ Used by the AssumedRoleCredentialProvider and in Session and Role delegation
+ tokens.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.assumed.role.credentials.provider</name>
+ <value>org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider</value>
+ <description>
+ List of credential providers to authenticate with the STS endpoint and
+ retrieve short-lived role credentials.
+ Only used if AssumedRoleCredentialProvider is the AWS credential provider.
+ If unset, uses "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider".
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.delegation.token.binding</name>
+ <value></value>
+ <description>
+ The name of a class to provide delegation tokens support in S3A.
+ If unset: delegation token support is disabled.
+
+ Note: for job submission to actually collect these tokens,
+ Kerberos must be enabled.
+
+ Bindings available in hadoop-aws are:
+ org.apache.hadoop.fs.s3a.auth.delegation.SessionTokenBinding
+ org.apache.hadoop.fs.s3a.auth.delegation.FullCredentialsTokenBinding
+ org.apache.hadoop.fs.s3a.auth.delegation.RoleTokenBinding
+ </description>
+</property>
+
<property>
<name>fs.s3a.connection.maximum</name>
- <value>15</value>
- <description>Controls the maximum number of simultaneous connections to S3.</description>
+ <value>96</value>
+ <description>Controls the maximum number of simultaneous connections to S3.
+ This must be bigger than the value of fs.s3a.threads.max so as to stop
+ threads being blocked waiting for new HTTPS connections.
+ Why not equal? The AWS SDK transfer manager also uses these connections.
+ </description>
</property>
<property>
<name>fs.s3a.connection.ssl.enabled</name>
<value>true</value>
- <description>Enables or disables SSL connections to S3.</description>
+ <description>Enables or disables SSL connections to AWS services.
+ Also sets the default port to use for the s3a proxy settings,
+ when not explicitly set in fs.s3a.proxy.port.</description>
</property>
<property>
@@ -736,16 +856,6 @@ options are covered in [Testing](./testing.md).
</description>
</property>
-<property>
- <name>fs.s3a.endpoint.region</name>
- <description>AWS S3 region for a bucket, which bypasses the parsing of
- fs.s3a.endpoint to know the region. Would be helpful in avoiding errors
- while using privateLink URL and explicitly set the bucket region.
- If set to a blank string (or 1+ space), falls back to the
- (potentially brittle) SDK region resolution process.
- </description>
-</property>
-
<property>
<name>fs.s3a.path.style.access</name>
<value>false</value>
@@ -788,8 +898,14 @@ options are covered in [Testing](./testing.md).
<property>
<name>fs.s3a.attempts.maximum</name>
- <value>20</value>
- <description>How many times we should retry commands on transient errors.</description>
+ <value>5</value>
+ <description>
+ Number of times the AWS client library should retry errors before
+ escalating to the S3A code: {@value}.
+ The S3A connector does its own selective retries; the only time the AWS
+ SDK operations are not wrapped is during multipart copy via the AWS SDK
+ transfer manager.
+ </description>
</property>
<property>
@@ -804,20 +920,6 @@ options are covered in [Testing](./testing.md).
<description>Socket connection timeout in milliseconds.</description>
</property>
-<property>
- <name>fs.s3a.paging.maximum</name>
- <value>5000</value>
- <description>How many keys to request from S3 when doing
- directory listings at a time.</description>
-</property>
-
-<property>
- <name>fs.s3a.threads.max</name>
- <value>10</value>
- <description> Maximum number of concurrent active (part)uploads,
- which each use a thread from the threadpool.</description>
-</property>
-
<property>
<name>fs.s3a.socket.send.buffer</name>
<value>8192</value>
@@ -830,6 +932,20 @@ options are covered in [Testing](./testing.md).
<description>Socket receive buffer hint to amazon connector. Represented in bytes.</description>
</property>
+<property>
+ <name>fs.s3a.paging.maximum</name>
+ <value>5000</value>
+ <description>How many keys to request from S3 when doing
+ directory listings at a time.</description>
+</property>
+
+<property>
+ <name>fs.s3a.threads.max</name>
+ <value>64</value>
+ <description>The total number of threads available in the filesystem for data
+ uploads *or any other queued filesystem operation*.</description>
+</property>
+
<property>
<name>fs.s3a.threads.keepalivetime</name>
<value>60</value>
@@ -839,9 +955,25 @@ options are covered in [Testing](./testing.md).
<property>
<name>fs.s3a.max.total.tasks</name>
- <value>5</value>
- <description>Number of (part)uploads allowed to the queue before
- blocking additional uploads.</description>
+ <value>32</value>
+ <description>The number of operations which can be queued for execution.
+ This is in addition to the number of active threads in fs.s3a.threads.max.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.executor.capacity</name>
+ <value>16</value>
+ <description>The maximum number of submitted tasks which is a single
+ operation (e.g. rename(), delete()) may submit simultaneously for
+ execution -excluding the IO-heavy block uploads, whose capacity
+ is set in "fs.s3a.fast.upload.active.blocks"
+
+ All tasks are submitted to the shared thread pool whose size is
+ set in "fs.s3a.threads.max"; the value of capacity should be less than that
+ of the thread pool itself, as the goal is to stop a single operation
+ from overloading that thread pool.
+ </description>
</property>
<property>
@@ -854,7 +986,7 @@ options are covered in [Testing](./testing.md).
<property>
<name>fs.s3a.multipart.threshold</name>
- <value>128MB</value>
+ <value>128M</value>
<description>How big (in bytes) to split upload or copy operations up into.
This also controls the partition size in renamed files, as rename() involves
copying the source file(s).
@@ -874,23 +1006,52 @@ options are covered in [Testing](./testing.md).
<property>
<name>fs.s3a.acl.default</name>
<description>Set a canned ACL for newly created and copied objects. Value may be Private,
- PublicRead, PublicReadWrite, AuthenticatedRead, LogDeliveryWrite, BucketOwnerRead,
- or BucketOwnerFullControl.
+ PublicRead, PublicReadWrite, AuthenticatedRead, LogDeliveryWrite, BucketOwnerRead,
+ or BucketOwnerFullControl.
If set, caller IAM role must have "s3:PutObjectAcl" permission on the bucket.
- </description>
+ </description>
</property>
<property>
<name>fs.s3a.multipart.purge</name>
<value>false</value>
<description>True if you want to purge existing multipart uploads that may not have been
- completed/aborted correctly</description>
+ completed/aborted correctly. The corresponding purge age is defined in
+ fs.s3a.multipart.purge.age.
+ If set, when the filesystem is instantiated then all outstanding uploads
+ older than the purge age will be terminated -across the entire bucket.
+ This will impact multipart uploads by other applications and users. so should
+ be used sparingly, with an age value chosen to stop failed uploads, without
+ breaking ongoing operations.
+ </description>
</property>
<property>
<name>fs.s3a.multipart.purge.age</name>
<value>86400</value>
- <description>Minimum age in seconds of multipart uploads to purge</description>
+ <description>Minimum age in seconds of multipart uploads to purge
+ on startup if "fs.s3a.multipart.purge" is true
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.encryption.algorithm</name>
+ <description>Specify a server-side encryption or client-side
+ encryption algorithm for s3a: file system. Unset by default. It supports the
+ following values: 'AES256' (for SSE-S3), 'SSE-KMS', 'SSE-C', and 'CSE-KMS'
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.encryption.key</name>
+ <description>Specific encryption key to use if fs.s3a.encryption.algorithm
+ has been set to 'SSE-KMS', 'SSE-C' or 'CSE-KMS'. In the case of SSE-C
+ , the value of this property should be the Base64 encoded key. If you are
+ using SSE-KMS and leave this property empty, you'll be using your default's
+ S3 KMS key, otherwise you should set this property to the specific KMS key
+ id. In case of 'CSE-KMS' this value needs to be the AWS-KMS Key ID
+ generated from AWS console.
+ </description>
</property>
<property>
@@ -900,23 +1061,19 @@ options are covered in [Testing](./testing.md).
</property>
<property>
- <name>fs.s3a.encryption.algorithm</name>
- <description>Specify a server-side encryption or client-side
- encryption algorithm for s3a: file system. Unset by default. It supports the
- following values: 'AES256' (for SSE-S3), 'SSE-KMS', 'SSE-C', and 'CSE-KMS'
- </description>
+ <name>fs.s3a.accesspoint.required</name>
+ <value>false</value>
+ <description>Require that all S3 access is made through Access Points and not through
+ buckets directly. If enabled, use per-bucket overrides to allow bucket access to a specific set
+ of buckets.</description>
</property>
<property>
- <name>fs.s3a.encryption.key</name>
- <description>Specific encryption key to use if fs.s3a.encryption.algorithm
- has been set to 'SSE-KMS', 'SSE-C' or 'CSE-KMS'. In the case of SSE-C
- , the value of this property should be the Base64 encoded key. If you are
- using SSE-KMS and leave this property empty, you'll be using your default's
- S3 KMS key, otherwise you should set this property to the specific KMS key
- id. In case of 'CSE-KMS' this value needs to be the AWS-KMS Key ID
- generated from AWS console.
- </description>
+ <name>fs.s3a.block.size</name>
+ <value>32M</value>
+ <description>Block size to use when reading files using s3a: file system.
+ A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
+ </description>
</property>
<property>
@@ -930,9 +1087,51 @@ options are covered in [Testing](./testing.md).
</property>
<property>
- <name>fs.s3a.block.size</name>
- <value>32M</value>
- <description>Block size to use when reading files using s3a: file system.
+ <name>fs.s3a.fast.upload.buffer</name>
+ <value>disk</value>
+ <description>
+ The buffering mechanism to for data being written.
+ Values: disk, array, bytebuffer.
+
+ "disk" will use the directories listed in fs.s3a.buffer.dir as
+ the location(s) to save data prior to being uploaded.
+
+ "array" uses arrays in the JVM heap
+
+ "bytebuffer" uses off-heap memory within the JVM.
+
+ Both "array" and "bytebuffer" will consume memory in a single stream up to the number
+ of blocks set by:
+
+ fs.s3a.multipart.size * fs.s3a.fast.upload.active.blocks.
+
+ If using either of these mechanisms, keep this value low
+
+ The total number of threads performing work across all threads is set by
+ fs.s3a.threads.max, with fs.s3a.max.total.tasks values setting the number of queued
+ work items.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.fast.upload.active.blocks</name>
+ <value>4</value>
+ <description>
+ Maximum Number of blocks a single output stream can have
+ active (uploading, or queued to the central FileSystem
+ instance's pool of queued operations.
+
+ This stops a single stream overloading the shared thread pool.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.readahead.range</name>
+ <value>64K</value>
+ <description>Bytes to read ahead during a seek() before closing and
+ re-opening the S3 HTTP connection. This option will be overridden if
+ any call to setReadahead() is made to an open stream.
+ A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
</description>
</property>
@@ -958,118 +1157,232 @@ options are covered in [Testing](./testing.md).
</property>
<property>
- <name>fs.AbstractFileSystem.s3a.impl</name>
- <value>org.apache.hadoop.fs.s3a.S3A</value>
- <description>The implementation class of the S3A AbstractFileSystem.</description>
+ <name>fs.s3a.retry.limit</name>
+ <value>7</value>
+ <description>
+ Number of times to retry any repeatable S3 client request on failure,
+ excluding throttling requests.
+ </description>
</property>
<property>
- <name>fs.s3a.readahead.range</name>
- <value>64K</value>
- <description>Bytes to read ahead during a seek() before closing and
- re-opening the S3 HTTP connection. This option will be overridden if
- any call to setReadahead() is made to an open stream.</description>
+ <name>fs.s3a.retry.interval</name>
+ <value>500ms</value>
+ <description>
+ Initial retry interval when retrying operations for any reason other
+ than S3 throttle errors.
+ </description>
</property>
<property>
- <name>fs.s3a.input.async.drain.threshold</name>
- <value>64K</value>
- <description>Bytes to read ahead during a seek() before closing and
- re-opening the S3 HTTP connection. This option will be overridden if
- any call to setReadahead() is made to an open stream.</description>
+ <name>fs.s3a.retry.throttle.limit</name>
+ <value>20</value>
+ <description>
+ Number of times to retry any throttled request.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.retry.throttle.interval</name>
+ <value>100ms</value>
+ <description>
+ Initial between retry attempts on throttled requests, +/- 50%. chosen at random.
+ i.e. for an intial value of 3000ms, the initial delay would be in the range 1500ms to 4500ms.
+ Backoffs are exponential; again randomness is used to avoid the thundering heard problem.
+ 500ms is the default value used by the AWS S3 Retry policy.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.committer.name</name>
+ <value>file</value>
+ <description>
+ Committer to create for output to S3A, one of:
+ "file", "directory", "partitioned", "magic".
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.committer.magic.enabled</name>
+ <value>true</value>
+ <description>
+ Enable support in the S3A filesystem for the "Magic" committer.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.committer.threads</name>
+ <value>8</value>
+ <description>
+ Number of threads in committers for parallel operations on files
+ (upload, commit, abort, delete...)
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.committer.staging.tmp.path</name>
+ <value>tmp/staging</value>
+ <description>
+ Path in the cluster filesystem for temporary data.
+ This is for HDFS, not the local filesystem.
+ It is only for the summary data of each file, not the actual
+ data being committed.
+ Using an unqualified path guarantees that the full path will be
+ generated relative to the home directory of the user creating the job,
+ hence private (assuming home directory permissions are secure).
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.committer.staging.unique-filenames</name>
+ <value>true</value>
+ <description>
+ Option for final files to have a unique name through job attempt info,
+ or the value of fs.s3a.committer.staging.uuid
+ When writing data with the "append" conflict option, this guarantees
+ that new data will not overwrite any existing data.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.committer.staging.conflict-mode</name>
+ <value>append</value>
+ <description>
+ Staging committer conflict resolution policy.
+ Supported: "fail", "append", "replace".
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.committer.abort.pending.uploads</name>
+ <value>true</value>
+ <description>
+ Should the committers abort all pending uploads to the destination
+ directory?
+
+ Set to false if more than one job is writing to the same directory tree.
+ </description>
</property>
<property>
<name>fs.s3a.list.version</name>
<value>2</value>
- <description>Select which version of the S3 SDK's List Objects API to use.
- Currently support 2 (default) and 1 (older API).</description>
+ <description>
+ Select which version of the S3 SDK's List Objects API to use. Currently
+ support 2 (default) and 1 (older API).
+ </description>
</property>
<property>
<name>fs.s3a.connection.request.timeout</name>
<value>0</value>
<description>
- Time out on HTTP requests to the AWS service; 0 means no timeout.
- Measured in seconds; the usual time suffixes are all supported
+ Time out on HTTP requests to the AWS service; 0 means no timeout.
+ Measured in seconds; the usual time suffixes are all supported
- Important: this is the maximum duration of any AWS service call,
- including upload and copy operations. If non-zero, it must be larger
- than the time to upload multi-megabyte blocks to S3 from the client,
- and to rename many-GB files. Use with care.
+ Important: this is the maximum duration of any AWS service call,
+ including upload and copy operations. If non-zero, it must be larger
+ than the time to upload multi-megabyte blocks to S3 from the client,
+ and to rename many-GB files. Use with care.
- Values that are larger than Integer.MAX_VALUE milliseconds are
- converged to Integer.MAX_VALUE milliseconds
+ Values that are larger than Integer.MAX_VALUE milliseconds are
+ converged to Integer.MAX_VALUE milliseconds
</description>
</property>
<property>
- <name>fs.s3a.bucket.probe</name>
- <value>0</value>
+ <name>fs.s3a.etag.checksum.enabled</name>
+ <value>false</value>
<description>
- The value can be 0 (default), 1 or 2.
- When set to 0, bucket existence checks won't be done
- during initialization thus making it faster.
- Though it should be noted that when the bucket is not available in S3,
- or if fs.s3a.endpoint points to the wrong instance of a private S3 store
- consecutive calls like listing, read, write etc. will fail with
- an UnknownStoreException.
- When set to 1, the bucket existence check will be done using the
- V1 API of the S3 protocol which doesn't verify the client's permissions
- to list or read data in the bucket.
- When set to 2, the bucket existence check will be done using the
- V2 API of the S3 protocol which does verify that the
- client has permission to read the bucket.
+ Should calls to getFileChecksum() return the etag value of the remote
+ object.
+ WARNING: if enabled, distcp operations between HDFS and S3 will fail unless
+ -skipcrccheck is set.
</description>
</property>
<property>
- <name>fs.s3a.object.content.encoding</name>
- <value></value>
+ <name>fs.s3a.change.detection.source</name>
+ <value>etag</value>
<description>
- Content encoding: gzip, deflate, compress, br, etc.
- This will be set in the "Content-Encoding" header of the object,
- and returned in HTTP HEAD/GET requests.
+ Select which S3 object attribute to use for change detection.
+ Currently support 'etag' for S3 object eTags and 'versionid' for
+ S3 object version IDs. Use of version IDs requires object versioning to be
+ enabled for each S3 bucket utilized. Object versioning is disabled on
+ buckets by default. When version ID is used, the buckets utilized should
+ have versioning enabled before any data is written.
</description>
</property>
<property>
- <name>fs.s3a.create.storage.class</name>
- <value></value>
+ <name>fs.s3a.change.detection.mode</name>
+ <value>server</value>
<description>
- Storage class: standard, reduced_redundancy, intelligent_tiering, etc.
- Specify the storage class for S3A PUT object requests.
- If not set the storage class will be null
- and mapped to default standard class on S3.
+ Determines how change detection is applied to alert to inconsistent S3
+ objects read during or after an overwrite. Value 'server' indicates to apply
+ the attribute constraint directly on GetObject requests to S3. Value 'client'
+ means to do a client-side comparison of the attribute value returned in the
+ response. Value 'server' would not work with third-party S3 implementations
+ that do not support these constraints on GetObject. Values 'server' and
+ 'client' generate RemoteObjectChangedException when a mismatch is detected.
+ Value 'warn' works like 'client' but generates only a warning. Value 'none'
+ will ignore change detection completely.
</description>
</property>
<property>
- <name>fs.s3a.prefetch.enabled</name>
- <value>false</value>
+ <name>fs.s3a.change.detection.version.required</name>
+ <value>true</value>
+ <description>
+ Determines if S3 object version attribute defined by
+ fs.s3a.change.detection.source should be treated as required. If true and the
+ referred attribute is unavailable in an S3 GetObject response,
+ NoVersionAttributeException is thrown. Setting to 'true' is encouraged to
+ avoid potential for inconsistent reads with third-party S3 implementations or
+ against S3 buckets that have object versioning disabled.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.ssl.channel.mode</name>
+ <value>default_jsse</value>
<description>
- Enables prefetching and caching when reading from input stream.
+ If secure connections to S3 are enabled, configures the SSL
+ implementation used to encrypt connections to S3. Supported values are:
+ "default_jsse", "default_jsse_with_gcm", "default", and "openssl".
+ "default_jsse" uses the Java Secure Socket Extension package (JSSE).
+ However, when running on Java 8, the GCM cipher is removed from the list
+ of enabled ciphers. This is due to performance issues with GCM in Java 8.
+ "default_jsse_with_gcm" uses the JSSE with the default list of cipher
+ suites. "default_jsse_with_gcm" is equivalent to the behavior prior to
+ this feature being introduced. "default" attempts to use OpenSSL rather
+ than the JSSE for SSL encryption, if OpenSSL libraries cannot be loaded,
+ it falls back to the "default_jsse" behavior. "openssl" attempts to use
+ OpenSSL as well, but fails if OpenSSL libraries cannot be loaded.
</description>
</property>
<property>
- <name>fs.s3a.prefetch.block.size</name>
- <value>8MB</value>
+ <name>fs.s3a.downgrade.syncable.exceptions</name>
+ <value>true</value>
<description>
- The size of a single prefetched block of data.
- Decreasing this will increase the number of prefetches required, and may negatively impact performance.
+ Warn but continue when applications use Syncable.hsync when writing
+ to S3A.
</description>
</property>
+<!--
+The switch to turn S3A auditing on or off.
+-->
<property>
- <name>fs.s3a.prefetch.block.count</name>
- <value>8</value>
+ <name>fs.s3a.audit.enabled</name>
+ <value>true</value>
<description>
- Maximum number of blocks prefetched concurrently at any given time.
+ Should auditing of S3A requests be enabled?
</description>
</property>
-```
+```
## <a name="retry_and_recovery"></a>Retry and Recovery
The S3A client makes a best-effort attempt at recovering from network failures;
@@ -1089,8 +1402,10 @@ not the failing operation is idempotent.
* Interruptions: `InterruptedIOException`, `InterruptedException`.
* Rejected HTTP requests: `InvalidRequestException`
-These are all considered unrecoverable: S3A will make no attempt to recover
-from them.
+These and others are all considered unrecoverable: S3A will make no attempt to recover
+from them. The AWS SDK itself may retry before the S3A connector sees the exception.
+As an example, the SDK will retry on `UnknownHostException` in case it is a transient
+DNS error.
### Possibly Recoverable Problems: Retry
@@ -1141,17 +1456,9 @@ only succeed if the first `delete()` call has already succeeded.
Because S3 is eventually consistent *and* doesn't support an
atomic create-no-overwrite operation, the choice is more ambiguous.
-Currently S3A considers delete to be
-idempotent because it is convenient for many workflows, including the
-commit protocols. Just be aware that in the presence of transient failures,
-more things may be deleted than expected. (For anyone who considers this to
-be the wrong decision: rebuild the `hadoop-aws` module with the constant
-`S3AFileSystem.DELETE_CONSIDERED_IDEMPOTENT` set to `false`).
-
-
-
-
-
+S3A considers delete to be idempotent because it is convenient for many workflows,
+including the commit protocols. Just be aware that in the presence of transient failures,
+more things may be deleted than expected.
### Throttled requests from S3
@@ -1657,13 +1964,17 @@ the storage class you want.
```
Please note that S3A does not support reading from archive storage classes at the moment.
-`AccessDeniedException` with InvalidObjectState will be thrown if you're trying to do so.
+`AccessDeniedException` with `InvalidObjectState` will be thrown if you're trying to do so.
+
+When a file is "renamed" through the s3a connector it is copied then deleted.
+Storage Classes will normally be propagated.
+
-## <a name="upload"></a>Configuring S3A for S3 on Outposts
+## <a name="outposts"></a>Configuring S3A for S3 on Outposts
S3A now supports [S3 on Outposts](https://docs.aws.amazon.com/AmazonS3/latest/userguide/S3onOutposts.html).
Accessing data through an access point is done by using its Amazon Resource Name (ARN), as opposed to just the bucket name.
-The only supported storage class on Outposts is **OUTPOSTS**, and by default objects are encrypted with [SSE-S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-outposts-data-encryption.html).
+The only supported storage class on Outposts is `OUTPOSTS`, and by default objects are encrypted with [SSE-S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-outposts-data-encryption.html).
You can set the Access Point ARN property using the following per bucket configuration property:
```xml
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
index b3863879a320..148f9bab9551 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
@@ -377,6 +377,19 @@ This adds extra overhead to every operation, but helps verify that the connector
not keeping markers where it needs to be deleting them -and hence backwards compatibility
is maintained.
+## <a name="enabling-prefetch"></a> Enabling prefetch for all tests
+
+The tests are run with prefetch if the `prefetch` property is set in the
+maven build. This can be combined with the scale tests as well.
+
+```bash
+mvn verify -Dprefetch
+
+mvn verify -Dparallel-tests -Dprefetch -DtestsThreadCount=8
+
+mvn verify -Dparallel-tests -Dprefetch -Dscale -DtestsThreadCount=8
+```
+
## <a name="scale"></a> Scale Tests
There are a set of tests designed to measure the scalability and performance
@@ -519,7 +532,7 @@ Otherwise, set a large timeout in `fs.s3a.scale.test.timeout`
The tests are executed in an order to only clean up created files after
the end of all the tests. If the tests are interrupted, the test data will remain.
-## <a name="alternate_s3"></a> Load tests.
+## <a name="load"></a> Load tests.
Some are designed to overload AWS services with more
requests per second than an AWS account is permitted.
@@ -545,6 +558,32 @@ which address issues. In particular, we encourage testing of Hadoop release
candidates, as these third-party endpoints get even less testing than the
S3 endpoint itself.
+The core XML settings to turn off tests of features unavailable
+on third party stores.
+
+```xml
+ <property>
+ <name>test.fs.s3a.encryption.enabled</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>test.fs.s3a.create.storage.class.enabled</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>fs.s3a.select.enabled</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>test.fs.s3a.sts.enabled</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>test.fs.s3a.create.create.acl.enabled</name>
+ <value>false</value>
+ < /property>
+```
+
### Public datasets used in tests
Some tests rely on the presence of existing public datasets available on Amazon S3.
@@ -585,7 +624,7 @@ S3 storage class, these tests might fail. They can be disabled.
</property>
```
-### Configuring the CSV file read tests**
+### Configuring the CSV file read tests
To test on alternate infrastructures supporting
the same APIs, the option `fs.s3a.scale.test.csvfile` must either be
@@ -618,19 +657,20 @@ your `core-site.xml` file, so that trying to use S3 select fails fast with
a meaningful error ("S3 Select not supported") rather than a generic Bad Request
exception.
-### <a name="enabling-prefetch"></a> Enabling prefetch for all tests
+### Disabling V1 List API tests
-The tests are run with prefetch if the `prefetch` property is set in the
-maven build. This can be combined with the scale tests as well.
-```bash
-mvn verify -Dprefetch
-
-mvn verify -Dparallel-tests -Dprefetch -DtestsThreadCount=8
-
-mvn verify -Dparallel-tests -Dprefetch -Dscale -DtestsThreadCount=8
+If `ITestS3AContractGetFileStatusV1List` fails with any error about unsupported API.
+```xml
+ <property>
+ <name>test.fs.s3a.list.v1.enabled</name>
+ <value>false</value>
+ </property>
```
+Note: there's no equivalent for turning off v2 listing API, which all stores are now
+expected to support.
+
### Testing Requester Pays
@@ -697,6 +737,20 @@ The default is ""; meaning "use the amazon default endpoint" (`sts.amazonaws.com
Consult the [AWS documentation](https://docs.aws.amazon.com/general/latest/gr/rande.html#sts_region)
for the full list of locations.
+### Disabling Content Encoding tests
+
+Tests in `ITestS3AContentEncoding` may need disabling
+```xml
+ <property>
+ <name>test.fs.s3a.content.encoding.enabled</name>
+ <value>false</value>
+ </property>
+```
+### Tests which may fail (and which you can ignore)
+
+* `ITestS3AContractMultipartUploader` tests `testMultipartUploadAbort` and `testSingleUpload` raising `FileNotFoundException`
+* `ITestS3AMiscOperations.testEmptyFileChecksums`: if the FS encrypts data always.
+
## <a name="debugging"></a> Debugging Test failures
Logging at debug level is the standard way to provide more diagnostics output;
@@ -943,7 +997,7 @@ sequential one afterwards. The IO heavy ones must also be subclasses of
`S3AScaleTestBase` and so only run if the system/maven property
`fs.s3a.scale.test.enabled` is true.
-## Individual test cases can be run in an IDE
+### Individual test cases can be run in an IDE
This is invaluable for debugging test failures.
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/third_party_stores.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/third_party_stores.md
new file mode 100644
index 000000000000..a7ea7b2e5902
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/third_party_stores.md
@@ -0,0 +1,415 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Working with Third-party S3 Stores
+
+The S3A connector works well with third-party S3 stores if the following requirements are met:
+
+* It correctly implements the core S3 REST API, including support for uploads and the V2 listing API.
+* The store supports the AWS V4 signing API *or* a custom signer is switched to.
+ This release does not support the legacy v2 signing API.
+* Errors are reported with the same HTTPS status codes as the S3 store. Error messages do not
+ need to be consistent.
+* The store is consistent.
+
+There are also specific deployment requirements:
+* The clock on the store and the client are close enough that signing works.
+* The client is correctly configured to connect to the store *and not use unavailable features*
+* If HTTPS authentication is used, the client/JVM TLS configurations allows it to authenticate the endpoint.
+
+The features which may be unavailable include:
+
+* Checksum-based server-side change detection during copy/read (`fs.s3a.change.detection.mode=server`)
+* Object versioning and version-based change detection (`fs.s3a.change.detection.source=versionid` and `fs.s3a.versioned.store=true`)
+* Bulk delete (`fs.s3a.multiobjectdelete.enable=true`)
+* Encryption. (`fs.s3a.encryption.algorithm`)
+* Storage class set in `fs.s3a.create.storage.class`
+* Content encodings as set with `fs.s3a.object.content.encoding`.
+* Optional Bucket Probes at startup (`fs.s3a.bucket.probe = 0`).
+ This is now the default -do not change it.
+* List API to use (`fs.s3a.list.version = 1`)
+
+## Configuring s3a to connect to a third party store
+
+
+### Connecting to a third party object store over HTTPS
+
+The core setting for a third party store is to change the endpoint in `fs.s3a.endpoint`.
+
+This can be a URL or a hostname/hostname prefix
+For third-party stores without virtual hostname support, providing the URL is straightforward;
+path style access must also be enabled in `fs.s3a.path.style.access`.
+
+The v4 signing algorithm requires a region to be set in `fs.s3a.endpoint.region`.
+A non-empty value is generally sufficient, though some deployments may require
+a specific value.
+
+Finally, assuming the credential source is the normal access/secret key
+then these must be set, either in XML or (preferred) in a JCEKS file.
+
+```xml
+
+ <property>
+ <name>fs.s3a.endpoint</name>
+ <value>https://storeendpoint.example.com</value>
+ </property>
+
+ <property>
+ <name>fs.s3a.path.style.access</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.s3a.endpoint.region</name>
+ <value>anything</value>
+ </property>
+
+ <property>
+ <name>fs.s3a.access.key</name>
+ <value>13324445</value>
+ </property>
+
+ <property>
+ <name>fs.s3a.secret.key</name>
+ <value>4C6B906D-233E-4E56-BCEA-304CC73A14F8</value>
+ </property>
+
+```
+
+If per-bucket settings are used here, then third-party stores and credentials may be used alongside an AWS store.
+
+# Troubleshooting
+
+The most common problem when talking to third-party stores are
+
+1. The S3A client is still configured to talk to the AWS S3 endpoint. This leads to authentication failures and/or reports that the bucket is unknown.
+2. Path access has not been enabled, the client is generating a host name for the target bucket and it does not exist.
+3. Invalid authentication credentials.
+4. JVM HTTPS settings include the certificates needed to negotiate a TLS connection with the store.
+
+
+## How to improve troubleshooting
+
+### log more network info
+
+There are some very low level logs.
+```properties
+# Log all HTTP requests made; includes S3 interaction. This may
+# include sensitive information such as account IDs in HTTP headers.
+log4j.logger.software.amazon.awssdk.request=DEBUG
+
+# Turn on low level HTTP protocol debugging
+log4j.logger.org.apache.http.wire=DEBUG
+
+# async client
+log4j.logger.io.netty.handler.logging=DEBUG
+log4j.logger.io.netty.handler.codec.http2.Http2FrameLogger=DEBUG
+```
+
+### Cut back on retries, shorten timeouts
+
+By default, there's a lot of retries going on in the AWS connector (which even retries on DNS failures)
+and in the S3A code which invokes it.
+
+Normally this helps prevent long-lived jobs from failing due to a transient network problem, however
+it means that when trying to debug connectivity problems, the commands can hang for a long time
+as they keep trying to reconnect to ports which are never going to be available.
+
+```xml
+
+ <property>
+ <name>fs.iostatistics.logging.level</name>
+ <value>info</value>
+ </property>
+
+ <property>
+ <name>fs.s3a.bucket.nonexistent-bucket-example.attempts.maximum</name>
+ <value>0</value>
+ </property>
+
+ <property>
+ <name>fs.s3a.bucket.nonexistent-bucket-example.retry.limit</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>fs.s3a.bucket.nonexistent-bucket-example.connection.timeout</name>
+ <value>500</value>
+ </property>
+
+ <property>
+ <name>fs.s3a.bucket.nonexistent-bucket-example.connection.establish.timeout</name>
+ <value>500</value>
+ </property>
+```
+## Cloudstore's Storediag
+
+There's an external utility, [cloudstore](https://github.com/steveloughran/cloudstore) whose [storediag](https://github.com/steveloughran/cloudstore#command-storediag) exists to debug the connection settings to hadoop cloud storage.
+
+```bash
+hadoop jar cloudstore-1.0.jar storediag s3a://nonexistent-bucket-example/
+```
+
+The main reason it's not an ASF release is that it allows for a rapid release cycle, sometimes hours; if anyone doesn't trust
+third-party code then they can download and build it themselves.
+
+
+# Problems
+
+## S3A client still pointing at AWS endpoint
+
+This is the most common initial problem, as it happens by default.
+
+To fix, set `fs.s3a.endpoint` to the URL of the internal store.
+
+### `org.apache.hadoop.fs.s3a.UnknownStoreException: `s3a://nonexistent-bucket-example/': Bucket does not exist`
+
+Either the bucket doesn't exist, or the bucket does exist but the endpoint is still set to an AWS endpoint.
+
+```
+stat: `s3a://nonexistent-bucket-example/': Bucket does not exist
+```
+The hadoop filesystem commands don't log stack traces on failure -adding this adds too much risk
+of breaking scripts, and the output is very uninformative
+
+```
+stat: nonexistent-bucket-example: getS3Region on nonexistent-bucket-example:
+software.amazon.awssdk.services.s3.model.S3Exception: null
+(Service: S3, Status Code: 403, Request ID: X26NWV0RJ1697SXF, Extended Request ID: bqq0rRm5Bdwt1oHSfmWaDXTfSOXoYvNhQxkhjjNAOpxhRaDvWArKCFAdL2hDIzgec6nJk1BVpJE=):null
+```
+
+It is possible to turn on debugging
+
+```
+log4j.logger.org.apache.hadoop.fs.shell=DEBUG
+```
+
+After which useful stack traces are logged.
+
+```
+org.apache.hadoop.fs.s3a.UnknownStoreException: `s3a://nonexistent-bucket-example/': Bucket does not exist
+ at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$null$3(S3AFileSystem.java:1075)
+ at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:122)
+ at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$4(Invoker.java:376)
+ at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:468)
+ at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:372)
+ at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:347)
+ at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$getS3Region$4(S3AFileSystem.java:1039)
+ at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.invokeTrackingDuration(IOStatisticsBinding.java:543)
+ at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.lambda$trackDurationOfOperation$5(IOStatisticsBinding.java:524)
+ at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDuration(IOStatisticsBinding.java:445)
+ at org.apache.hadoop.fs.s3a.S3AFileSystem.trackDurationAndSpan(S3AFileSystem.java:2631)
+ at org.apache.hadoop.fs.s3a.S3AFileSystem.getS3Region(S3AFileSystem.java:1038)
+ at org.apache.hadoop.fs.s3a.S3AFileSystem.bindAWSClient(S3AFileSystem.java:982)
+ at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:622)
+ at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3452)
+```
+
+### `S3Exception: null (Service: S3, Status Code: 403...` or `AccessDeniedException`
+
+* Endpoint is default
+* Credentials were not issued by AWS .
+* `fs.s3a.endpoint.region` unset.
+
+If the client doesn't have any AWS credentials (from hadoop settings, environment variables or elsewhere) then
+the binding will fail even before the existence of the bucket can be probed for.
+
+```bash
+hadoop fs -stat s3a://nonexistent-bucket-example
+```
+
+
+```
+stat: nonexistent-bucket-example: getS3Region on nonexistent-bucket-example:
+software.amazon.awssdk.services.s3.model.S3Exception: null (Service: S3, Status Code: 403,
+ Request ID: X26NWV0RJ1697SXF, Extended Request ID: bqq0rRm5Bdwt1oHSfmWaDXTfSOXoYvNhQxkhjjNAOpxhRaDvWArKCFAdL2hDIzgec6nJk1BVpJE=):null
+```
+
+Or with a more detailed stack trace:
+
+```
+java.nio.file.AccessDeniedException: nonexistent-bucket-example: getS3Region on nonexistent-bucket-example: software.amazon.awssdk.services.s3.model.S3Exception: null (Service: S3, Status Code: 403, Request ID: X26NWV0RJ1697SXF, Extended Request ID: bqq0rRm5Bdwt1oHSfmWaDXTfSOXoYvNhQxkhjjNAOpxhRaDvWArKCFAdL2hDIzgec6nJk1BVpJE=):null
+ at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:235)
+ at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:124)
+ at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$4(Invoker.java:376)
+ at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:468)
+ at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:372)
+ at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:347)
+ at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$getS3Region$4(S3AFileSystem.java:1039)
+ at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.invokeTrackingDuration(IOStatisticsBinding.java:543)
+ at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.lambda$trackDurationOfOperation$5(IOStatisticsBinding.java:524)
+ at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDuration(IOStatisticsBinding.java:445)
+ at org.apache.hadoop.fs.s3a.S3AFileSystem.trackDurationAndSpan(S3AFileSystem.java:2631)
+ at org.apache.hadoop.fs.s3a.S3AFileSystem.getS3Region(S3AFileSystem.java:1038)
+ at org.apache.hadoop.fs.s3a.S3AFileSystem.bindAWSClient(S3AFileSystem.java:982)
+ at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:622)
+ at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3452)
+```
+
+## `Received an UnknownHostException when attempting to interact with a service`
+
+
+### Hypothesis 1: Region set, but not endpoint
+
+The bucket `fs.s3a.endpoint.region` region setting is valid internally, but as the endpoint
+is still AWS, this region is not recognised.
+The S3A client's creation of an endpoint URL generates an unknown host.
+
+```xml
+ <property>
+ <name>fs.s3a.bucket.nonexistent-bucket-example.endpoint.region</name>
+ <value>internal</value>
+ </property>
+```
+
+
+```
+ls: software.amazon.awssdk.core.exception.SdkClientException:
+ Received an UnknownHostException when attempting to interact with a service.
+ See cause for the exact endpoint that is failing to resolve.
+ If this is happening on an endpoint that previously worked, there may be
+ a network connectivity issue or your DNS cache could be storing endpoints for too long.:
+ nonexistent-bucket-example.s3.internal.amazonaws.com: nodename nor servname provided, or not known
+
+
+```
+
+### Hypothesis 2: region set, endpoint set, but `fs.s3a.path.style.access` is still set to `false`
+
+* The bucket `fs.s3a.endpoint.region` region setting is valid internally,
+* and `fs.s3a.endpoint` is set to a hostname (not a URL).
+* `fs.s3a.path.style.access` set to `false`
+
+```
+ls: software.amazon.awssdk.core.exception.SdkClientException:
+ Received an UnknownHostException when attempting to interact with a service.
+ See cause for the exact endpoint that is failing to resolve.
+ If this is happening on an endpoint that previously worked, there may be
+ a network connectivity issue or your DNS cache could be storing endpoints for too long.:
+ nonexistent-bucket-example.localhost: nodename nor servname provided, or not known
+```
+
+Fix: path style access
+
+```xml
+ <property>
+ <name>fs.s3a.bucket.nonexistent-bucket-example.path.style.access</name>
+ <value>true</value>
+ </property>
+```
+
+# Connecting to Google Cloud Storage through the S3A connector
+
+It *is* possible to connect to google cloud storage through the S3A connector.
+However, Google provide their own [Cloud Storage connector](https://cloud.google.com/dataproc/docs/concepts/connectors/cloud-storage).
+That is a well maintained Hadoop filesystem client which uses their XML API,
+And except for some very unusual cases, that is the connector to use.
+
+When interacting with a GCS container through the S3A connector may make sense
+* The installation doesn't have the gcs-connector JAR.
+* The different credential mechanism may be convenient.
+* There's a desired to use S3A Delegation Tokens to pass secrets with a job.
+* There's a desire to use an external S3A extension (delegation tokens etc.)
+
+The S3A connector binding works through the Google Cloud [S3 Storage API](https://cloud.google.com/distributed-cloud/hosted/docs/ga/gdch/apis/storage-s3-rest-api),
+which is a subset of the AWS API.
+
+
+To get a compatible access and secret key, follow the instructions of
+[Simple migration from Amazon S3 to Cloud Storage](https://cloud.google.com/storage/docs/aws-simple-migration#defaultproj).
+
+Here are the per-bucket setings for an example bucket "gcs-container"
+in Google Cloud Storage. Note the multiobject delete option must be disabled;
+this makes renaming and deleting significantly slower.
+
+
+```xml
+<configuration>
+
+ <property>
+ <name>fs.s3a.bucket.gcs-container.access.key</name>
+ <value>GOOG1EZ....</value>
+ </property>
+
+ <property>
+ <name>fs.s3a.bucket.gcs-container.secret.key</name>
+ <value>SECRETS</value>
+ </property>
+
+ <property>
+ <name>fs.s3a.bucket.gcs-container.endpoint</name>
+ <value>https://storage.googleapis.com</value>
+ </property>
+
+ <property>
+ <name>fs.s3a.bucket.gcs-container.bucket.probe</name>
+ <value>0</value>
+ </property>
+
+ <property>
+ <name>fs.s3a.bucket.gcs-container.list.version</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>fs.s3a.bucket.gcs-container.multiobjectdelete.enable</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.s3a.bucket.gcs-container.select.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.s3a.bucket.gcs-container.path.style.access</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.s3a.bucket.gcs-container.endpoint.region</name>
+ <value>dummy</value>
+ </property>
+
+</configuration>
+```
+
+This is a very rarely used configuration -however, it can be done, possibly as a way to interact with Google Cloud Storage in a deployment
+which lacks the GCS connector.
+
+It is also a way to regression test foundational S3A third-party store compatibility if you lack access to to any alternative.
+
+```xml
+<configuration>
+ <property>
+ <name>test.fs.s3a.encryption.enabled</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>fs.s3a.scale.test.csvfile</name>
+ <value></value>
+ </property>
+ <property>
+ <name>test.fs.s3a.sts.enabled</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>test.fs.s3a.content.encoding.enabled</name>
+ <value>false</value>
+ </property>
+</configuration>
+```
+
+_Note_ If anyone is set up to test this reguarly, please let the hadoop developer team know if regressions do surface,
+as it is not a common test configuration.
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
index 4444d7f16b42..8c57db1faf4c 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
@@ -71,28 +71,38 @@ An exception reporting this class as missing means that this JAR is not on
the classpath.
-### `NoClassDefFoundError: software/amazon/awssdk/crt/s3/S3MetaRequest`
-
-The library `aws-crt.jar` is not on the classpath. Its classes
-are not in the AWS `bundle.jar` file, yet may be needed by some uses made
-of the SDK.
-
-Fix: add.
-
-```
-java.lang.BootstrapMethodError: java.lang.NoClassDefFoundError: software/amazon/awssdk/crt/s3/S3MetaRequest
-at software.amazon.awssdk.services.s3.internal.crt.S3MetaRequestPauseObservable.<init>(S3MetaRequestPauseObservable.java:33)
-at software.amazon.awssdk.transfer.s3.internal.DefaultS3TransferManager.uploadFile(DefaultS3TransferManager.java:205)
-at org.apache.hadoop.fs.s3a.S3AFileSystem.putObject(S3AFileSystem.java:3064)
-at org.apache.hadoop.fs.s3a.S3AFileSystem.executePut(S3AFileSystem.java:4054)
-
-```
-### `ClassNotFoundException: software.amazon.awssdk.services.s3.S3Client`
-
-(or other `software.amazon` class.)
-
-This means that the AWS V2 SDK `bundle.jar` JAR is not on the classpath:
-add it.
+### `java.lang.NoClassDefFoundError: software/amazon/awssdk/services/s3/model/S3Exception`
+
+This is one of the first stack traces which can surface when trying to instantiate
+an S3A filesystem instance without having the AWS V2 SDK `bundle.jar` on the classpath
+
+```
+java.lang.NoClassDefFoundError: software/amazon/awssdk/services/s3/model/S3Exception
+ at java.lang.ClassLoader.defineClass1(Native Method)
+ at java.lang.ClassLoader.defineClass(ClassLoader.java:756)
+ at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
+ at java.net.URLClassLoader.defineClass(URLClassLoader.java:473)
+ at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
+ at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
+ at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
+ at java.security.AccessController.doPrivileged(Native Method)
+ at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
+ at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
+ at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
+ at java.lang.Class.forName0(Native Method)
+ at java.lang.Class.forName(Class.java:348)
+ at org.apache.hadoop.conf.Configuration.getClassByNameOrNull(Configuration.java:2639)
+ at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:2604)
+ at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2700)
+ at org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:3414)
+ at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3449)
+ at org.apache.hadoop.fs.FileSystem.access$300(FileSystem.java:162)
+```
+
+Fix: add it to classpath.
+
+Maven/Ivy/SBT/Gradle builds which import `hadoop-aws` or
+`hadoop-cloud-storage` artifacts should get the artifact automatically.
### `ClassNotFoundException: com.amazonaws.auth.AWSCredentials`
@@ -152,7 +162,6 @@ the client retries a number of times before eventually failing.
When it finally gives up, it will report a message about signature mismatch:
```
-com.amazonaws.services.s3.model.AmazonS3Exception:
The request signature we calculated does not match the signature you provided.
Check your key and signing method.
(Service: Amazon S3; Status Code: 403; Error Code: SignatureDoesNotMatch,
@@ -229,53 +238,18 @@ read requests are allowed, but operations which write to the bucket are denied.
Check the system clock.
-### <a name="bad_request"></a> "Bad Request" exception when working with AWS S3 Frankfurt, Seoul, or other "V4" endpoint
-
+### <a name="bad_request"></a> "Bad Request" exception when working with data stores in an AWS region other than us-eaast
-S3 Frankfurt and Seoul *only* support
-[the V4 authentication API](http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html).
-Requests using the V2 API will be rejected with 400 `Bad Request`
```
$ bin/hadoop fs -ls s3a://frankfurt/
WARN s3a.S3AFileSystem: Client: Amazon S3 error 400: 400 Bad Request; Bad Request (retryable)
-com.amazonaws.services.s3.model.AmazonS3Exception: Bad Request (Service: Amazon S3;
- Status Code: 400; Error Code: 400 Bad Request; Request ID: 923C5D9E75E44C06),
- S3 Extended Request ID: HDwje6k+ANEeDsM6aJ8+D5gUmNAMguOk2BvZ8PH3g9z0gpH+IuwT7N19oQOnIr5CIx7Vqb/uThE=
- at com.amazonaws.http.AmazonHttpClient.handleErrorResponse(AmazonHttpClient.java:1182)
- at com.amazonaws.http.AmazonHttpClient.executeOneRequest(AmazonHttpClient.java:770)
- at com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:489)
- at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:310)
- at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3785)
- at com.amazonaws.services.s3.AmazonS3Client.headBucket(AmazonS3Client.java:1107)
- at com.amazonaws.services.s3.AmazonS3Client.doesBucketExist(AmazonS3Client.java:1070)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.verifyBucketExists(S3AFileSystem.java:307)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:284)
- at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2793)
- at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:101)
- at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2830)
- at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2812)
- at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:389)
- at org.apache.hadoop.fs.Path.getFileSystem(Path.java:356)
- at org.apache.hadoop.fs.shell.PathData.expandAsGlob(PathData.java:325)
- at org.apache.hadoop.fs.shell.Command.expandArgument(Command.java:235)
- at org.apache.hadoop.fs.shell.Command.expandArguments(Command.java:218)
- at org.apache.hadoop.fs.shell.FsCommand.processRawArguments(FsCommand.java:103)
- at org.apache.hadoop.fs.shell.Command.run(Command.java:165)
- at org.apache.hadoop.fs.FsShell.run(FsShell.java:315)
- at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76)
- at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:90)
- at org.apache.hadoop.fs.FsShell.main(FsShell.java:373)
-ls: doesBucketExist on frankfurt-new: com.amazonaws.services.s3.model.AmazonS3Exception:
+ls: doesBucketExist on frankfurt-new: S3Exception:
Bad Request (Service: Amazon S3; Status Code: 400; Error Code: 400 Bad Request;
```
-This happens when trying to work with any S3 service which only supports the
-"V4" signing API —but the client is configured to use the default S3 service
-endpoint.
-
The S3A client needs to be given the endpoint to use via the `fs.s3a.endpoint`
property.
@@ -291,10 +265,10 @@ As an example, the endpoint for S3 Frankfurt is `s3.eu-central-1.amazonaws.com`:
When [PrivateLink](https://docs.aws.amazon.com/AmazonS3/latest/userguide/privatelink-interface-endpoints.html) URL
is used instead of standard s3a endpoint, it returns "authorization
-header is malformed" exception. So, if we set fs.s3a.endpoint=bucket.vpce
--<some_string>.s3.ca-central-1.vpce.amazonaws.com and make s3 calls we get:
+header is malformed" exception. So, if we set `fs.s3a.endpoint=bucket.vpce
+-<some_string>.s3.ca-central-1.vpce.amazonaws.com` and make s3 calls we get:
```
-com.amazonaws.services.s3.model.AmazonS3Exception: The authorization header is malformed; the region 'vpce' is wrong; expecting 'ca-central-1'
+S3Exception: The authorization header is malformed; the region 'vpce' is wrong; expecting 'ca-central-1'
(Service: Amazon S3; Status Code: 400; Error Code: AuthorizationHeaderMalformed; Request ID: req-id; S3 Extended Request ID: req-id-2), S3 Extended Request ID: req-id-2:AuthorizationHeaderMalformed: The authorization
header is malformed; the region 'vpce' is wrong; expecting 'ca-central-1' (Service: Amazon S3; Status Code: 400; Error Code: AuthorizationHeaderMalformed; Request ID: req-id;
```
@@ -314,35 +288,27 @@ S3 region as `ca-central-1`.
</property>
```
-### `Class does not implement AWSCredentialsProvider`
+### `Classdoes not implement software.amazon.awssdk.auth.credentials.AwsCredentialsProvider`
A credential provider listed in `fs.s3a.aws.credentials.provider` does not implement
-the interface `com.amazonaws.auth.AWSCredentialsProvider`.
-
-```
- Cause: java.lang.RuntimeException: java.io.IOException: Class class com.amazonaws.auth.EnvironmentVariableCredentialsProvider does not implement AWSCredentialsProvider
- at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:686)
- at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:621)
- at org.apache.spark.sql.hive.client.HiveClientImpl.newState(HiveClientImpl.scala:219)
- at org.apache.spark.sql.hive.client.HiveClientImpl.<init>(HiveClientImpl.scala:126)
- at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
- at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
- at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
- at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
- at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:306)
- at org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:433)
- ...
- Cause: java.io.IOException: Class class com.amazonaws.auth.EnvironmentVariableCredentialsProvider does not implement AWSCredentialsProvider
- at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:722)
- at org.apache.hadoop.fs.s3a.S3AUtils.buildAWSProviderList(S3AUtils.java:687)
- at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet(S3AUtils.java:620)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.bindAWSClient(S3AFileSystem.java:673)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:414)
- at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3462)
- at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:171)
- at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3522)
- at org.apache.hadoop.fs.FileSystem$Cache.getUnique(FileSystem.java:3496)
- at org.apache.hadoop.fs.FileSystem.newInstance(FileSystem.java:591)
+the interface `software.amazon.awssdk.auth.credentials.AwsCredentialsProvider`.
+
+```
+InstantiationIOException: `s3a://stevel-gcs/': Class org.apache.hadoop.fs.s3a.S3ARetryPolicy does not implement software.amazon.awssdk.auth.credentials.AwsCredentialsProvider (configuration key fs.s3a.aws.credentials.provider)
+ at org.apache.hadoop.fs.s3a.impl.InstantiationIOException.isNotInstanceOf(InstantiationIOException.java:128)
+ at org.apache.hadoop.fs.s3a.S3AUtils.getInstanceFromReflection(S3AUtils.java:604)
+ at org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.createAWSV2CredentialProvider(CredentialProviderListFactory.java:299)
+ at org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.buildAWSProviderList(CredentialProviderListFactory.java:245)
+ at org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.createAWSCredentialProviderList(CredentialProviderListFactory.java:144)
+ at org.apache.hadoop.fs.s3a.S3AFileSystem.bindAWSClient(S3AFileSystem.java:971)
+ at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:624)
+ at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3601)
+ at org.apache.hadoop.fs.FileSystem.access$300(FileSystem.java:171)
+ at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3702)
+ at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3653)
+ at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:555)
+ at org.apache.hadoop.fs.Path.getFileSystem(Path.java:366)
+
```
There's two main causes
@@ -357,12 +323,12 @@ There's two main causes
If you see this and you are trying to use the S3A connector with Spark, then the cause can
be that the isolated classloader used to load Hive classes is interfering with the S3A
-connector's dynamic loading of `com.amazonaws` classes. To fix this, declare that that
+connector's dynamic loading of `software.amazon.awssdk` classes. To fix this, declare that
the classes in the aws SDK are loaded from the same classloader which instantiated
the S3A FileSystem instance:
```
-spark.sql.hive.metastore.sharedPrefixes com.amazonaws.
+spark.sql.hive.metastore.sharedPrefixes software.amazon.awssdk.
```
## <a name="access_denied"></a> "The security token included in the request is invalid"
@@ -383,59 +349,8 @@ It may be mistyped, or the access key may have been deleted by one of the accoun
```
java.nio.file.AccessDeniedException: bucket: doesBucketExist on bucket:
- com.amazonaws.services.s3.model.AmazonS3Exception:
+ S3Exception:
The AWS Access Key Id you provided does not exist in our records.
- (Service: Amazon S3; Status Code: 403; Error Code: InvalidAccessKeyId;
- at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:214)
- at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:111)
- at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$3(Invoker.java:260)
- at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:314)
- at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:256)
- at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:231)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.verifyBucketExists(S3AFileSystem.java:366)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:302)
- at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3354)
- at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:124)
- at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3403)
- at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3371)
- at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:477)
- at org.apache.hadoop.fs.contract.AbstractBondedFSContract.init(AbstractBondedFSContract.java:72)
- at org.apache.hadoop.fs.contract.AbstractFSContractTestBase.setup(AbstractFSContractTestBase.java:177)
- at org.apache.hadoop.fs.s3a.commit.AbstractCommitITest.setup(AbstractCommitITest.java:163)
- at org.apache.hadoop.fs.s3a.commit.AbstractITCommitMRJob.setup(AbstractITCommitMRJob.java:129)
- at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
- at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47)
- at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
- at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44)
- at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24)
- at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
- at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48)
- at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55)
- at org.junit.internal.runners.statements.FailOnTimeout$StatementThread.run(FailOnTimeout.java:74)
-Caused by: com.amazonaws.services.s3.model.AmazonS3Exception:
- The AWS Access Key Id you provided does not exist in our records.
- (Service: Amazon S3; Status Code: 403; Error Code: InvalidAccessKeyId;
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1303)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1055)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:743)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:717)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:699)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:667)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:649)
- at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:513)
- at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4229)
- at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4176)
- at com.amazonaws.services.s3.AmazonS3Client.getAcl(AmazonS3Client.java:3381)
- at com.amazonaws.services.s3.AmazonS3Client.getBucketAcl(AmazonS3Client.java:1160)
- at com.amazonaws.services.s3.AmazonS3Client.getBucketAcl(AmazonS3Client.java:1150)
- at com.amazonaws.services.s3.AmazonS3Client.doesBucketExist(AmazonS3Client.java:1266)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$verifyBucketExists$1(S3AFileSystem.java:367)
- at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:109)
- ... 27 more
```
@@ -445,47 +360,12 @@ Caller has no permission to access the bucket at all.
```
doesBucketExist on fdsd: java.nio.file.AccessDeniedException: fdsd: doesBucketExist on fdsd:
- com.amazonaws.services.s3.model.AmazonS3Exception: All access to this object has been disabled
+ S3Exception: All access to this object has been disabled
(Service: Amazon S3; Status Code: 403; Error Code: AllAccessDisabled; Request ID: E6229D7F8134E64F;
S3 Extended Request ID: 6SzVz2t4qa8J2Wxo/oc8yBuB13Mgrn9uMKnxVY0hsBd2kU/YdHzW1IaujpJdDXRDCQRX3f1RYn0=),
S3 Extended Request ID: 6SzVz2t4qa8J2Wxo/oc8yBuB13Mgrn9uMKnxVY0hsBd2kU/YdHzW1IaujpJdDXRDCQRX3f1RYn0=:AllAccessDisabled
All access to this object has been disabled (Service: Amazon S3; Status Code: 403;
- at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:205)
- at org.apache.hadoop.fs.s3a.S3ALambda.once(S3ALambda.java:122)
- at org.apache.hadoop.fs.s3a.S3ALambda.lambda$retry$2(S3ALambda.java:233)
- at org.apache.hadoop.fs.s3a.S3ALambda.retryUntranslated(S3ALambda.java:288)
- at org.apache.hadoop.fs.s3a.S3ALambda.retry(S3ALambda.java:228)
- at org.apache.hadoop.fs.s3a.S3ALambda.retry(S3ALambda.java:203)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.verifyBucketExists(S3AFileSystem.java:357)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:293)
- at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3288)
- at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:123)
- at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3337)
- at org.apache.hadoop.fs.FileSystem$Cache.getUnique(FileSystem.java:3311)
- at org.apache.hadoop.fs.FileSystem.newInstance(FileSystem.java:529)
- at org.apache.hadoop.fs.s3a.s3guard.S3GuardTool$BucketInfo.run(S3GuardTool.java:997)
- at org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.run(S3GuardTool.java:309)
- at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76)
- at org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.run(S3GuardTool.java:1218)
- at org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.main(S3GuardTool.java:1227)
-Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: All access to this object has been disabled
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1303)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1055)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:743)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:717)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:699)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:667)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:649)
- at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:513)
- at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4229)
- at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4176)
- at com.amazonaws.services.s3.AmazonS3Client.getAcl(AmazonS3Client.java:3381)
- at com.amazonaws.services.s3.AmazonS3Client.getBucketAcl(AmazonS3Client.java:1160)
- at com.amazonaws.services.s3.AmazonS3Client.getBucketAcl(AmazonS3Client.java:1150)
- at com.amazonaws.services.s3.AmazonS3Client.doesBucketExist(AmazonS3Client.java:1266)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$verifyBucketExists$1(S3AFileSystem.java:360)
- at org.apache.hadoop.fs.s3a.S3ALambda.once(S3ALambda.java:120)
+
```
Check the name of the bucket is correct, and validate permissions for the active user/role.
@@ -500,19 +380,9 @@ or the caller does not have the right to access the data.
```
java.nio.file.AccessDeniedException: test/: PUT 0-byte object on test/:
- com.amazonaws.services.s3.model.AmazonS3Exception: Access Denied (Service: Amazon S3; Status Code: 403;
+ S3Exception: Access Denied (Service: Amazon S3; Status Code: 403;
Error Code: AccessDenied; Request ID: EDC662AD2EEEA33C;
- at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:210)
- at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:110)
- at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$3(Invoker.java:259)
- at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:313)
- at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:255)
- at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:230)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.createEmptyObject(S3AFileSystem.java:2691)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.createFakeDirectory(S3AFileSystem.java:2666)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.innerMkdirs(S3AFileSystem.java:2030)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.mkdirs(S3AFileSystem.java:1965)
- at org.apache.hadoop.fs.FileSystem.mkdirs(FileSystem.java:2305)
+
```
In the AWS S3 management console, select the "permissions" tab for the bucket, then "bucket policy".
@@ -539,7 +409,7 @@ _all_ the rights which the caller has.
```
mv: rename s3a://london/dest to s3a://london/src on
s3a://london/dest:
- com.amazonaws.services.s3.model.MultiObjectDeleteException: One or more objects
+ MultiObjectDeleteException: One or more objects
could not be deleted (Service: null; Status Code: 200; Error Code: null; Request
ID: 5C9018EF245F02C5; S3 Extended Request ID:
5fQ2RVCPF0rdvADRv2XY3U4yb2J0gHRID/4jm1eqCXp7RxpU0dH9DliChYsCUD1aVCFtbwfWJWY=),
@@ -561,7 +431,7 @@ directory will _only_ be in the destination. And files for which the rename oper
had yet to commence -they will only be in the source tree.
The user has to recover from this themselves. Be assured: no data will have been deleted, it
-is just that the data may now be scattered across two directories.
+is just that the data may now be scattered across two directories.
Note: this is one reason why any application which tries to atomically commit work
via rename (classic Hadoop output committers, distcp with the `-atomic` option) are
not safe to use with S3. It is not a file system.
@@ -589,7 +459,7 @@ If you don't enable this acknowledgement within S3A, then you will see a message
```
java.nio.file.AccessDeniedException: s3a://my-bucket/my-object: getFileStatus on s3a://my-bucket/my-object:
-com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403;
+S3Exception: Forbidden (Service: Amazon S3; Status Code: 403;
Error Code: 403 Forbidden; Request ID: myshortreqid; S3 Extended Request ID: mylongreqid):403 Forbidden
```
@@ -598,9 +468,9 @@ To enable requester pays, set `fs.s3a.requester.pays.enabled` property to `true`
### <a name="access_denied_archive_storage_class"></a>`AccessDeniedException` "InvalidObjectState" when trying to read files
```
-java.nio.file.AccessDeniedException: file1: copyFile(file1, file2) on file1: com.amazonaws.services.s3.model.AmazonS3Exception: Operation is not valid for the source object's storage class (Service: Amazon S3; Status Code: 403; Error Code: InvalidObjectState; Request ID: SK9EMPC1YRX75VZR; S3 Extended Request ID: /nhUfdwJ+y5DLz6B4YR2FdA0FnQWwhDAkSCakn42zs2JssK3qWTrfwdNDiy6bOyXHOvJY0VAlHw=; Proxy: null), S3 Extended Request ID: /nhUfdwJ+y5DLz6B4YR2FdA0FnQWwhDAkSCakn42zs2JssK3qWTrfwdNDiy6bO [...]
+java.nio.file.AccessDeniedException: file1: copyFile(file1, file2) on file1: S3Exception: Operation is not valid for the source object's storage class (Service: Amazon S3; Status Code: 403; Error Code: InvalidObjectState; Request ID: SK9EMPC1YRX75VZR; S3 Extended Request ID: /nhUfdwJ+y5DLz6B4YR2FdA0FnQWwhDAkSCakn42zs2JssK3qWTrfwdNDiy6bOyXHOvJY0VAlHw=; Proxy: null), S3 Extended Request ID: /nhUfdwJ+y5DLz6B4YR2FdA0FnQWwhDAkSCakn42zs2JssK3qWTrfwdNDiy6bOyXHOvJY0VAlHw=:InvalidObjectState
-Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Operation is not valid for the source object's storage class (Service: Amazon S3; Status Code: 403; Error Code: InvalidObjectState; Request ID: SK9EMPC1YRX75VZR; S3 Extended Request ID: /nhUfdwJ+y5DLz6B4YR2FdA0FnQWwhDAkSCakn42zs2JssK3qWTrfwdNDiy6bOyXHOvJY0VAlHw=; Proxy: null), S3 Extended Request ID: /nhUfdwJ+y5DLz6B4YR2FdA0FnQWwhDAkSCakn42zs2JssK3qWTrfwdNDiy6bOyXHOvJY0VAlHw=
+Caused by: S3Exception: Operation is not valid for the source object's storage class (Service: Amazon S3; Status Code: 403; Error Code: InvalidObjectState; Request ID: SK9EMPC1YRX75VZR; S3 Extended Request ID: /nhUfdwJ+y5DLz6B4YR2FdA0FnQWwhDAkSCakn42zs2JssK3qWTrfwdNDiy6bOyXHOvJY0VAlHw=; Proxy: null), S3 Extended Request ID: /nhUfdwJ+y5DLz6B4YR2FdA0FnQWwhDAkSCakn42zs2JssK3qWTrfwdNDiy6bOyXHOvJY0VAlHw=
```
This happens when you're trying to read or copy files that have archive storage class such as
@@ -614,7 +484,7 @@ Region must be provided when requesting session credentials, or an exception wil
message:
```
-com.amazonaws.SdkClientException: Unable to find a region via the region provider
+ Unable to find a region via the region provider
chain. Must provide an explicit region in the builder or setup environment to supply a region.
```
@@ -624,12 +494,13 @@ endpoint and region like the following:
```xml
<property>
- <name>fs.s3a.assumed.role.sts.endpoint</name>
- <value>${sts.endpoint}</value>
+ <name>fs.s3a.assumed.role.sts.endpoint</name>
+ <value>${sts.endpoint}</value>
</property>
+
<property>
-<name>fs.s3a.assumed.role.sts.endpoint.region</name>
-<value>${sts.region}</value>
+ <name>fs.s3a.assumed.role.sts.endpoint.region</name>
+ <value>${sts.region}</value>
</property>
```
@@ -672,8 +543,7 @@ can be used:
</property>
```
-Using the explicit endpoint for the region is recommended for speed and
-to use the V4 signing API.
+Using the explicit endpoint for the region is recommended for speed.
### <a name="NoRegion"></a> `Unable to find a region via the region provider chain`
@@ -690,15 +560,9 @@ This failure surfaces when _all_ the following conditions are met:
Stack trace (Hadoop 3.3.1):
```
-Caused by: com.amazonaws.SdkClientException: Unable to find a region via the region provider chain.
+Unable to find a region via the region provider chain.
Must provide an explicit region in the builder or setup environment to supply a region.
- at com.amazonaws.client.builder.AwsClientBuilder.setRegion(AwsClientBuilder.java:462)
- at com.amazonaws.client.builder.AwsClientBuilder.configureMutableProperties(AwsClientBuilder.java:424)
- at com.amazonaws.client.builder.AwsSyncClientBuilder.build(AwsSyncClientBuilder.java:46)
- at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.buildAmazonS3Client(DefaultS3ClientFactory.java:145)
- at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.createS3Client(DefaultS3ClientFactory.java:97)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.bindAWSClient(S3AFileSystem.java:788)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:478)
+
```
Log and stack trace on later releases, with
@@ -725,14 +589,7 @@ warning that the SDK resolution chain is in use:
at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3565)
at org.apache.hadoop.fs.FileSystem$Cache.getUnique(FileSystem.java:3518)
at org.apache.hadoop.fs.FileSystem.newInstance(FileSystem.java:592)
-Caused by: com.amazonaws.SdkClientException: Unable to find a region via the region provider chain.
- Must provide an explicit region in the builder or setup environment to supply a region.
- at com.amazonaws.client.builder.AwsClientBuilder.setRegion(AwsClientBuilder.java:462)
- at com.amazonaws.client.builder.AwsClientBuilder.configureMutableProperties(AwsClientBuilder.java:424)
- at com.amazonaws.client.builder.AwsSyncClientBuilder.build(AwsSyncClientBuilder.java:46)
- at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.buildAmazonS3Client(DefaultS3ClientFactory.java:185)
- at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.createS3Client(DefaultS3ClientFactory.java:117)
- ... 21 more
+
```
Due to changes in S3 client construction in Hadoop 3.3.1 this option surfaces in
@@ -778,33 +635,9 @@ This happens when using the output stream thread pool runs out of capacity.
```
[s3a-transfer-shared-pool1-t20] INFO http.AmazonHttpClient (AmazonHttpClient.java:executeHelper(496))
- Unable to execute HTTP request:
- Timeout waiting for connection from poolorg.apache.http.conn.ConnectionPoolTimeoutException:
+ org.apache.http.conn.ConnectionPoolTimeoutException:
Timeout waiting for connection from pool
- at org.apache.http.impl.conn.PoolingClientConnectionManager.leaseConnection(PoolingClientConnectionManager.java:230)
- at org.apache.http.impl.conn.PoolingClientConnectionManager$1.getConnection(PoolingClientConnectionManager.java:199)
- at sun.reflect.GeneratedMethodAccessor13.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at com.amazonaws.http.conn.ClientConnectionRequestFactory$Handler.invoke(ClientConnectionRequestFactory.java:70)
- at com.amazonaws.http.conn.$Proxy10.getConnection(Unknown Source)
- at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:424)
- at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:884)
- at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
- at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
- at com.amazonaws.http.AmazonHttpClient.executeOneRequest(AmazonHttpClient.java:728)
- at com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:489)
- at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:310)
- at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3785)
- at com.amazonaws.services.s3.AmazonS3Client.doUploadPart(AmazonS3Client.java:2921)
- at com.amazonaws.services.s3.AmazonS3Client.uploadPart(AmazonS3Client.java:2906)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.uploadPart(S3AFileSystem.java:1025)
- at org.apache.hadoop.fs.s3a.S3ABlockOutputStream$MultiPartUpload$1.call(S3ABlockOutputStream.java:360)
- at org.apache.hadoop.fs.s3a.S3ABlockOutputStream$MultiPartUpload$1.call(S3ABlockOutputStream.java:355)
- at org.apache.hadoop.fs.s3a.BlockingThreadPoolExecutorService$CallableWithPermitRelease.call(BlockingThreadPoolExecutorService.java:239)
- at java.util.concurrent.FutureTask.run(FutureTask.java:266)
- at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
- at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
- at java.lang.Thread.run(Thread.java:745)
+
```
Make sure that `fs.s3a.connection.maximum` is at least larger
@@ -813,12 +646,12 @@ than `fs.s3a.threads.max`.
```xml
<property>
<name>fs.s3a.threads.max</name>
- <value>20</value>
+ <value>64</value>
</property>
<property>
<name>fs.s3a.connection.maximum</name>
- <value>30</value>
+ <value>64</value>
</property>
```
@@ -836,39 +669,15 @@ Set `fs.s3a.connection.maximum` to a larger value (and at least as large as
The HTTP Server did not respond.
```
-2017-02-07 10:01:07,950 INFO [s3a-transfer-shared-pool1-t7] com.amazonaws.http.AmazonHttpClient:
- Unable to execute HTTP request: bucket.s3.amazonaws.com:443 failed to respond
+2017-02-07 10:01:07,950 INFO [s3a-transfer-shared-pool1-t7] Unable to execute HTTP request: bucket.s3.amazonaws.com:443 failed to respond
org.apache.http.NoHttpResponseException: bucket.s3.amazonaws.com:443 failed to respond
- at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
- at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
- at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
- at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
- at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:259)
- at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:209)
- at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
- at com.amazonaws.http.protocol.SdkHttpRequestExecutor.doReceiveResponse(SdkHttpRequestExecutor.java:66)
- at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
- at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:686)
- at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:488)
- at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:884)
- at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
- at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
- at com.amazonaws.http.AmazonHttpClient.executeOneRequest(AmazonHttpClient.java:728)
- at com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:489)
- at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:310)
- at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3785)
- at com.amazonaws.services.s3.AmazonS3Client.copyPart(AmazonS3Client.java:1731)
- at com.amazonaws.services.s3.transfer.internal.CopyPartCallable.call(CopyPartCallable.java:41)
- at com.amazonaws.services.s3.transfer.internal.CopyPartCallable.call(CopyPartCallable.java:28)
- at org.apache.hadoop.fs.s3a.SemaphoredDelegatingExecutor$CallableWithPermitRelease.call(SemaphoredDelegatingExecutor.java:222)
- at java.util.concurrent.FutureTask.run(FutureTask.java:266)
- at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
- at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
- at java.lang.Thread.run(Thread.java:745)
+
```
Probably network problems, unless it really is an outage of S3.
+If you are working with a third party store, check its network configuration.
+
### Out of heap memory when writing with via Fast Upload
@@ -922,19 +731,8 @@ for up to date advice.
```
org.apache.hadoop.fs.s3a.AWSClientIOException: getFileStatus on test/testname/streaming/:
- com.amazonaws.AmazonClientException: Failed to sanitize XML document
- destined for handler class com.amazonaws.services.s3.model.transform.XmlResponsesSaxParser$ListBucketHandler:
- Failed to sanitize XML document destined for handler class
- com.amazonaws.services.s3.model.transform.XmlResponsesSaxParser$ListBucketHandler
- at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:105)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.getFileStatus(S3AFileSystem.java:1462)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.innerListStatus(S3AFileSystem.java:1227)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.listStatus(S3AFileSystem.java:1203)
- at org.apache.hadoop.fs.s3a.S3AGlobber.listStatus(S3AGlobber.java:69)
- at org.apache.hadoop.fs.s3a.S3AGlobber.doGlob(S3AGlobber.java:210)
- at org.apache.hadoop.fs.s3a.S3AGlobber.glob(S3AGlobber.java:125)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.globStatus(S3AFileSystem.java:1853)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.globStatus(S3AFileSystem.java:1841)
+ Failed to sanitize XML document
+
```
We believe this is caused by the connection to S3 being broken.
@@ -944,7 +742,7 @@ It may go away if the operation is retried.
### JSON Parse Error from AWS SDK
-Sometimes a JSON Parse error is reported with the stack trace in the `com.amazonaws`,
+Sometimes a JSON Parse error is reported with the stack trace from `software.amazon.awssdk`,
Again, we believe this is caused by the connection to S3 being broken.
@@ -1031,18 +829,7 @@ from breaking.
org.apache.hadoop.fs.s3a.RemoteFileChangedException: re-open `s3a://my-bucket/test/file.txt':
Change reported by S3 while reading at position 1949.
ETag f9c186d787d4de9657e99f280ba26555 was unavailable
- at org.apache.hadoop.fs.s3a.impl.ChangeTracker.processResponse(ChangeTracker.java:137)
- at org.apache.hadoop.fs.s3a.S3AInputStream.reopen(S3AInputStream.java:200)
- at org.apache.hadoop.fs.s3a.S3AInputStream.lambda$lazySeek$1(S3AInputStream.java:346)
- at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$2(Invoker.java:195)
- at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:109)
- at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$3(Invoker.java:265)
- at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:322)
- at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:261)
- at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:193)
- at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:215)
- at org.apache.hadoop.fs.s3a.S3AInputStream.lazySeek(S3AInputStream.java:339)
- at org.apache.hadoop.fs.s3a.S3AInputStream.read(S3AInputStream.java:372)
+
```
If an S3 object is updated while an S3A filesystem reader has an open
@@ -1065,17 +852,7 @@ the following error.
org.apache.hadoop.fs.s3a.NoVersionAttributeException: `s3a://my-bucket/test/file.txt':
Change detection policy requires ETag
at org.apache.hadoop.fs.s3a.impl.ChangeTracker.processResponse(ChangeTracker.java:153)
- at org.apache.hadoop.fs.s3a.S3AInputStream.reopen(S3AInputStream.java:200)
- at org.apache.hadoop.fs.s3a.S3AInputStream.lambda$lazySeek$1(S3AInputStream.java:346)
- at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$2(Invoker.java:195)
- at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:109)
- at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$3(Invoker.java:265)
- at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:322)
- at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:261)
- at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:193)
- at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:215)
- at org.apache.hadoop.fs.s3a.S3AInputStream.lazySeek(S3AInputStream.java:339)
- at org.apache.hadoop.fs.s3a.S3AInputStream.read(S3AInputStream.java:372)
+
```
If the change policy is `versionid` there are a number of possible causes
@@ -1133,51 +910,11 @@ key arn is invalid.
```
org.apache.hadoop.fs.s3a.AWSS3IOException: innerMkdirs on /test:
- com.amazonaws.services.s3.model.AmazonS3Exception:
+ S3Exception:
Invalid arn (Service: Amazon S3; Status Code: 400; Error Code: KMS.NotFoundException;
Request ID: CA89F276B3394565),
S3 Extended Request ID: ncz0LWn8zor1cUO2fQ7gc5eyqOk3YfyQLDn2OQNoe5Zj/GqDLggUYz9QY7JhdZHdBaDTh+TL5ZQ=:
Invalid arn (Service: Amazon S3; Status Code: 400; Error Code: KMS.NotFoundException; Request ID: CA89F276B3394565)
- at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:194)
- at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:117)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.mkdirs(S3AFileSystem.java:1541)
- at org.apache.hadoop.fs.FileSystem.mkdirs(FileSystem.java:2230)
- at org.apache.hadoop.fs.contract.AbstractFSContractTestBase.mkdirs(AbstractFSContractTestBase.java:338)
- at org.apache.hadoop.fs.contract.AbstractFSContractTestBase.setup(AbstractFSContractTestBase.java:193)
- at org.apache.hadoop.fs.s3a.scale.S3AScaleTestBase.setup(S3AScaleTestBase.java:90)
- at org.apache.hadoop.fs.s3a.scale.AbstractSTestS3AHugeFiles.setup(AbstractSTestS3AHugeFiles.java:77)
- at sun.reflect.GeneratedMethodAccessor12.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47)
- at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
- at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44)
- at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24)
- at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
- at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55)
- at org.junit.internal.runners.statements.FailOnTimeout$StatementThread.run(FailOnTimeout.java:74)
-Caused by: com.amazonaws.services.s3.model.AmazonS3Exception:
- Invalid arn (Service: Amazon S3; Status Code: 400; Error Code: KMS.NotFoundException; Request ID: CA89F276B3394565)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1588)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1258)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1030)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:742)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:716)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:699)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:667)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:649)
- at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:513)
- at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4221)
- at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4168)
- at com.amazonaws.services.s3.AmazonS3Client.putObject(AmazonS3Client.java:1718)
- at com.amazonaws.services.s3.transfer.internal.UploadCallable.uploadInOneChunk(UploadCallable.java:133)
- at com.amazonaws.services.s3.transfer.internal.UploadCallable.call(UploadCallable.java:125)
- at com.amazonaws.services.s3.transfer.internal.UploadMonitor.call(UploadMonitor.java:143)
- at com.amazonaws.services.s3.transfer.internal.UploadMonitor.call(UploadMonitor.java:48)
- at java.util.concurrent.FutureTask.run(FutureTask.java:266)
- at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
- at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
- at java.lang.Thread.run(Thread.java:745)
```
Possible causes:
@@ -1186,14 +923,13 @@ Possible causes:
* the KMS key referenced by the ARN is in a different region than the S3 bucket
being used.
-
### Using SSE-C "Bad Request"
When performing file operations the user may run into an unexpected 400/403
error such as
```
org.apache.hadoop.fs.s3a.AWSS3IOException: getFileStatus on fork-4/:
- com.amazonaws.services.s3.model.AmazonS3Exception:
+ S3Exception:
Bad Request (Service: Amazon S3; Status Code: 400;
Error Code: 400 Bad Request; Request ID: 42F9A1987CB49A99),
S3 Extended Request ID: jU2kcwaXnWj5APB14Cgb1IKkc449gu2+dhIsW/+7x9J4D+VUkKvu78mBo03oh9jnOT2eoTLdECU=:
@@ -1214,40 +950,7 @@ file using configured SSE-C keyB into that structure.
Reading an unencrypted file would fail when read through CSE enabled client.
```
java.lang.SecurityException: Instruction file not found for S3 object with bucket name: ap-south-cse, key: unencryptedData.txt
- at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleAE.decipher(S3CryptoModuleAE.java:190)
- at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleAE.getObjectSecurely(S3CryptoModuleAE.java:136)
- at com.amazonaws.services.s3.AmazonS3EncryptionClientV2.getObject(AmazonS3EncryptionClientV2.java:241)
- at org.apache.hadoop.fs.s3a.S3AFileSystem$InputStreamCallbacksImpl.getObject(S3AFileSystem.java:1462)
- at org.apache.hadoop.fs.s3a.S3AInputStream.lambda$reopen$0(S3AInputStream.java:217)
- at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:117)
- at org.apache.hadoop.fs.s3a.S3AInputStream.reopen(S3AInputStream.java:216)
- at org.apache.hadoop.fs.s3a.S3AInputStream.lambda$lazySeek$1(S3AInputStream.java:382)
- at org.apache.hadoop.fs.s3a.Invoker.lambda$maybeRetry$3(Invoker.java:230)
- at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:117)
- at org.apache.hadoop.fs.s3a.Invoker.lambda$maybeRetry$5(Invoker.java:354)
- at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:414)
- at org.apache.hadoop.fs.s3a.Invoker.maybeRetry(Invoker.java:350)
- at org.apache.hadoop.fs.s3a.Invoker.maybeRetry(Invoker.java:228)
- at org.apache.hadoop.fs.s3a.Invoker.maybeRetry(Invoker.java:272)
- at org.apache.hadoop.fs.s3a.S3AInputStream.lazySeek(S3AInputStream.java:374)
- at org.apache.hadoop.fs.s3a.S3AInputStream.read(S3AInputStream.java:493)
- at java.io.DataInputStream.read(DataInputStream.java:100)
- at org.apache.hadoop.io.IOUtils.copyBytes(IOUtils.java:94)
- at org.apache.hadoop.io.IOUtils.copyBytes(IOUtils.java:68)
- at org.apache.hadoop.io.IOUtils.copyBytes(IOUtils.java:129)
- at org.apache.hadoop.fs.shell.Display$Cat.printToStdout(Display.java:101)
- at org.apache.hadoop.fs.shell.Display$Cat.processPath(Display.java:96)
- at org.apache.hadoop.fs.shell.Command.processPathInternal(Command.java:370)
- at org.apache.hadoop.fs.shell.Command.processPaths(Command.java:333)
- at org.apache.hadoop.fs.shell.Command.processPathArgument(Command.java:306)
- at org.apache.hadoop.fs.shell.Command.processArgument(Command.java:288)
- at org.apache.hadoop.fs.shell.Command.processArguments(Command.java:272)
- at org.apache.hadoop.fs.shell.FsCommand.processRawArguments(FsCommand.java:121)
- at org.apache.hadoop.fs.shell.Command.run(Command.java:179)
- at org.apache.hadoop.fs.FsShell.run(FsShell.java:327)
- at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:81)
- at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:95)
- at org.apache.hadoop.fs.FsShell.main(FsShell.java:390)
+
```
CSE enabled client should read encrypted data only.
@@ -1266,14 +969,14 @@ method requires KMS key ID. Use fs.s3a.encryption.key property to set it.
set `fs.s3a.encryption.key=<KMS_KEY_ID>` generated through AWS console.
-### `com.amazonaws.services.kms.model.IncorrectKeyException` The key ID in the request does not identify a CMK that can perform this operation.
+### `software.amazon.awssdk.services.kms.model.IncorrectKeyException` The key ID in the request does not identify a CMK that can perform this operation.
KMS key ID used to PUT(encrypt) the data, must be the one used to GET the
data.
```
cat: open s3a://ap-south-cse/encryptedData.txt at 0 on
s3a://ap-south-cse/encryptedData.txt:
-com.amazonaws.services.kms.model.IncorrectKeyException: The key ID in the
+software.amazon.awssdk.services.kms.model.IncorrectKeyException: The key ID in the
request does not identify a CMK that can perform this operation. (Service: AWSKMS;
Status Code: 400; ErrorCode: IncorrectKeyException;
Request ID: da21aa8a-f00d-467c-94a0-32b627d32bc0; Proxy: null):IncorrectKeyException:
@@ -1283,14 +986,14 @@ Request ID: da21aa8a-f00d-467c-94a0-32b627d32bc0; Proxy: null)
```
Use the same KMS key ID used to upload data to download and read it as well.
-### `com.amazonaws.services.kms.model.NotFoundException` key/<KMS_KEY_ID> does not exist
+### `software.amazon.awssdk.services.kms.model.NotFoundException` key/<KMS_KEY_ID> does not exist
Using a KMS key ID from a different region than the bucket used to store data
would lead to failure while uploading.
```
mkdir: PUT 0-byte object on testmkdir:
-com.amazonaws.services.kms.model.NotFoundException: Key
+software.amazon.awssdk.services.kms.model.NotFoundException: Key
'arn:aws:kms:ap-south-1:152813717728:key/<KMS_KEY_ID>'
does not exist (Service: AWSKMS; Status Code: 400; Error Code: NotFoundException;
Request ID: 279db85d-864d-4a38-9acd-d892adb504c0; Proxy: null):NotFoundException:
@@ -1307,24 +1010,6 @@ If Range get is not supported for a CSE algorithm or is disabled:
```
java.lang.SecurityException: Unable to perform range get request: Range get support has been disabled. See https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html
- at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleAE.assertCanGetPartialObject(S3CryptoModuleAE.java:446)
- at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleAE.getObjectSecurely(S3CryptoModuleAE.java:117)
- at com.amazonaws.services.s3.AmazonS3EncryptionClientV2.getObject(AmazonS3EncryptionClientV2.java:241)
- at org.apache.hadoop.fs.s3a.S3AFileSystem$InputStreamCallbacksImpl.getObject(S3AFileSystem.java:1462)
- at org.apache.hadoop.fs.s3a.S3AInputStream.lambda$reopen$0(S3AInputStream.java:217)
- at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:117)
- at org.apache.hadoop.fs.s3a.S3AInputStream.reopen(S3AInputStream.java:216)
- at org.apache.hadoop.fs.s3a.S3AInputStream.lambda$lazySeek$1(S3AInputStream.java:382)
- at org.apache.hadoop.fs.s3a.Invoker.lambda$maybeRetry$3(Invoker.java:230)
- at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:117)
- at org.apache.hadoop.fs.s3a.Invoker.lambda$maybeRetry$5(Invoker.java:354)
- at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:414)
- at org.apache.hadoop.fs.s3a.Invoker.maybeRetry(Invoker.java:350)
- at org.apache.hadoop.fs.s3a.Invoker.maybeRetry(Invoker.java:228)
- at org.apache.hadoop.fs.s3a.Invoker.maybeRetry(Invoker.java:272)
- at org.apache.hadoop.fs.s3a.S3AInputStream.lazySeek(S3AInputStream.java:374)
- at org.apache.hadoop.fs.s3a.S3AInputStream.read(S3AInputStream.java:408)
- at java.io.DataInputStream.readByte(DataInputStream.java:265)
```
Range gets must be enabled for CSE to work.
@@ -1360,124 +1045,43 @@ enhance security. See https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryp
We can ignore this, since this CryptoMode setting(CryptoMode.AuthenticatedEncryption)
is required for range gets to work.
-### com.amazonaws.services.kms.model.InvalidKeyUsageException: You cannot generate a data key with an asymmetric CMK
+### `software.amazon.awssdk.services.kms.mode.InvalidKeyUsageException: You cannot generate a data key with an asymmetric CMK`
If you generated an Asymmetric CMK from AWS console then CSE-KMS won't be
able to generate unique data key for encryption.
```
-Caused by: com.amazonaws.services.kms.model.InvalidKeyUsageException:
+Caused by: software.amazon.awssdk.services.kms.mode.InvalidKeyUsageException:
You cannot generate a data key with an asymmetric CMK
(Service: AWSKMS; Status Code: 400; Error Code: InvalidKeyUsageException; Request ID: 93609c15-e490-4035-8390-f4396f0d90bf; Proxy: null)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1819)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1403)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1372)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686)
- at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550)
- at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530)
- at com.amazonaws.services.kms.AWSKMSClient.doInvoke(AWSKMSClient.java:7223)
- at com.amazonaws.services.kms.AWSKMSClient.invoke(AWSKMSClient.java:7190)
- at com.amazonaws.services.kms.AWSKMSClient.invoke(AWSKMSClient.java:7179)
- at com.amazonaws.services.kms.AWSKMSClient.executeGenerateDataKey(AWSKMSClient.java:3482)
- at com.amazonaws.services.kms.AWSKMSClient.generateDataKey(AWSKMSClient.java:3451)
- at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.buildContentCryptoMaterial(S3CryptoModuleBase.java:533)
- at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.newContentCryptoMaterial(S3CryptoModuleBase.java:481)
- at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.createContentCryptoMaterial(S3CryptoModuleBase.java:447)
- at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.putObjectUsingMetadata(S3CryptoModuleBase.java:160)
- at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.putObjectSecurely(S3CryptoModuleBase.java:156)
- at com.amazonaws.services.s3.AmazonS3EncryptionClientV2.putObject(AmazonS3EncryptionClientV2.java:236)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$putObjectDirect$17(S3AFileSystem.java:2792)
- at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfSupplier(IOStatisticsBinding.java:604)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.putObjectDirect(S3AFileSystem.java:2789)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$createEmptyObject$33(S3AFileSystem.java:4440)
- at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:117)
- ... 49 more
```
Generate a Symmetric Key in the same region as your S3 storage for CSE-KMS to
work.
-### com.amazonaws.services.kms.model.NotFoundException: Invalid keyId
+### software.amazon.awssdk.services.kms.mode.NotFoundException: Invalid keyId
If the value in `fs.s3a.encryption.key` property, does not exist
/valid in AWS KMS CMK(Customer managed keys), then this error would be seen.
```
-Caused by: com.amazonaws.services.kms.model.NotFoundException: Invalid keyId abc
-(Service: AWSKMS; Status Code: 400; Error Code: NotFoundException; Request ID: 9d53552a-3d1b-47c8-984c-9a599d5c2391; Proxy: null)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1819)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1403)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1372)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686)
- at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550)
- at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530)
- at com.amazonaws.services.kms.AWSKMSClient.doInvoke(AWSKMSClient.java:7223)
- at com.amazonaws.services.kms.AWSKMSClient.invoke(AWSKMSClient.java:7190)
- at com.amazonaws.services.kms.AWSKMSClient.invoke(AWSKMSClient.java:7179)
- at com.amazonaws.services.kms.AWSKMSClient.executeGenerateDataKey(AWSKMSClient.java:3482)
- at com.amazonaws.services.kms.AWSKMSClient.generateDataKey(AWSKMSClient.java:3451)
- at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.buildContentCryptoMaterial(S3CryptoModuleBase.java:533)
- at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.newContentCryptoMaterial(S3CryptoModuleBase.java:481)
- at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.createContentCryptoMaterial(S3CryptoModuleBase.java:447)
- at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.putObjectUsingMetadata(S3CryptoModuleBase.java:160)
- at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.putObjectSecurely(S3CryptoModuleBase.java:156)
- at com.amazonaws.services.s3.AmazonS3EncryptionClientV2.putObject(AmazonS3EncryptionClientV2.java:236)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$putObjectDirect$17(S3AFileSystem.java:2792)
- at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfSupplier(IOStatisticsBinding.java:604)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.putObjectDirect(S3AFileSystem.java:2789)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$createEmptyObject$33(S3AFileSystem.java:4440)
- at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:117)
- ... 49 more
+Caused by: software.amazon.awssdk.services.kms.model.NotFoundException: Invalid keyId abc
+(Service: AWSKMS; Status Code: 400; Error Code: NotFoundException; Request ID:
+ 9d53552a-3d1b-47c8-984c-9a599d5c2391; Proxy: null)
```
Check if `fs.s3a.encryption.key` is set correctly and matches the
same on AWS console.
-### com.amazonaws.services.kms.model.AWSKMSException: User: <User_ARN> is not authorized to perform : kms :GenerateDataKey on resource: <KEY_ID>
+### software.amazon.awssdk.services.kms.model.KmsException: User: <User_ARN> is not authorized to perform : kms :GenerateDataKey on resource: <KEY_ID>
User doesn't have authorization to the specific AWS KMS Key ID.
```
-Caused by: com.amazonaws.services.kms.model.AWSKMSException:
-User: arn:aws:iam::152813717728:user/<user> is not authorized to perform: kms:GenerateDataKey on resource: <key_ID>
-(Service: AWSKMS; Status Code: 400; Error Code: AccessDeniedException; Request ID: 4ded9f1f-b245-4213-87fc-16cba7a1c4b9; Proxy: null)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1819)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1403)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1372)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686)
- at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550)
- at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530)
- at com.amazonaws.services.kms.AWSKMSClient.doInvoke(AWSKMSClient.java:7223)
- at com.amazonaws.services.kms.AWSKMSClient.invoke(AWSKMSClient.java:7190)
- at com.amazonaws.services.kms.AWSKMSClient.invoke(AWSKMSClient.java:7179)
- at com.amazonaws.services.kms.AWSKMSClient.executeGenerateDataKey(AWSKMSClient.java:3482)
- at com.amazonaws.services.kms.AWSKMSClient.generateDataKey(AWSKMSClient.java:3451)
- at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.buildContentCryptoMaterial(S3CryptoModuleBase.java:533)
- at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.newContentCryptoMaterial(S3CryptoModuleBase.java:481)
- at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.createContentCryptoMaterial(S3CryptoModuleBase.java:447)
- at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.putObjectUsingMetadata(S3CryptoModuleBase.java:160)
- at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.putObjectSecurely(S3CryptoModuleBase.java:156)
- at com.amazonaws.services.s3.AmazonS3EncryptionClientV2.putObject(AmazonS3EncryptionClientV2.java:236)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$putObjectDirect$17(S3AFileSystem.java:2792)
- at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfSupplier(IOStatisticsBinding.java:604)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.putObjectDirect(S3AFileSystem.java:2789)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$createEmptyObject$33(S3AFileSystem.java:4440)
- at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:117)
- ... 49 more
+Caused by: software.amazon.awssdk.services.kms.model.KmsException:
+User: arn:aws:iam::152813717728:user/<user> is not authorized to perform:
+ kms:GenerateDataKey on resource: <key_ID>
+(Service: AWSKMS; Status Code: 400; Error Code: AccessDeniedException;
+ Request ID: 4ded9f1f-b245-4213-87fc-16cba7a1c4b9; Proxy: null)
```
The user trying to use the KMS Key ID should have the right permissions to access
@@ -1528,13 +1132,13 @@ is more than 10000 (specified by aws SDK). You can configure
The bucket does not exist.
```
-org.apache.hadoop.fs.s3a.UnknownStoreException:
- Bucket random-bucket-33013fb8-f7f7-4edb-9c26-16a6ed019184 does not exist
- at org.apache.hadoop.fs.s3a.S3AFileSystem.verifyBucketExists(S3AFileSystem.java:537)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.doBucketProbing(S3AFileSystem.java:471)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:387)
- at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3422)
- at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:502)
+org.apache.hadoop.fs.s3a.UnknownStoreException: `s3a://random-bucket-7d9217b0-b426-4344-82ea-25d6cbb316f1/':
+ Bucket does not exist: software.amazon.awssdk.services.s3.model.NoSuchBucketException: null
+ (Service: S3, Status Code: 404, Request ID: RD254TC8EVDV98AK,
+ Extended Request ID: 49F5CO1IKavFsz+VBecf2uwZeNVar3InHkdIrONvAK5yQ73gqZ1hFoAEMo8/x5wRNe3OXO3aebvZkev2bS81kw==)
+ (Service: S3, Status Code: 404, Request ID: RD254TC8EVDV98AK): null
+ (Service: S3, Status Code: 404, Request ID: RD254TC8EVDV98AK, Extended Request ID: 49F5CO1IKavFsz+VBecf2uwZeNVar3InHkdIrONvAK5yQ73gqZ1hFoAEMo8/x5wRNe3OXO3aebvZkev2bS81kw==)
+ (Service: S3, Status Code: 404, Request ID: RD254TC8EVDV98AK)
```
Check the URI is correct, and that the bucket actually exists.
@@ -1547,20 +1151,6 @@ for a bucket is not an unusual occurrence.
This can surface during filesystem API calls if the bucket is deleted while you are using it,
-or the startup check for bucket existence has been disabled by setting `fs.s3a.bucket.probe` to 0.
-```
-org.apache.hadoop.fs.s3a.UnknownStoreException: s3a://random-bucket-7d9217b0-b426-4344-82ea-25d6cbb316f1/
-
- at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:254)
- at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:167)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.innerListFiles(S3AFileSystem.java:4149)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.listFiles(S3AFileSystem.java:3983)
-Caused by: com.amazonaws.services.s3.model.AmazonS3Exception:
-The specified bucket does not exist
- (Service: Amazon S3; Status Code: 404; Error Code: NoSuchBucket
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1712)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1367)
-```
-
## <a name="s3guard"></a> S3Guard Errors
@@ -1663,73 +1253,16 @@ Possible causes for this
This is a very, very rare occurrence.
-If the problem is a signing one, try changing the signature algorithm.
-
-```xml
-<property>
- <name>fs.s3a.signing-algorithm</name>
- <value>S3SignerType</value>
-</property>
-```
-
-We cannot make any promises that it will work, only that it has been known to
-make the problem go away "once"
### `AWSS3IOException` The Content-MD5 you specified did not match what we received
Reads work, but writes, even `mkdir`, fail:
-```
-org.apache.hadoop.fs.s3a.AWSS3IOException: copyFromLocalFile(file:/tmp/hello.txt, s3a://bucket/hello.txt)
- on file:/tmp/hello.txt:
- The Content-MD5 you specified did not match what we received.
- (Service: Amazon S3; Status Code: 400; Error Code: BadDigest; Request ID: 4018131225),
- S3 Extended Request ID: null
- at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:127)
- at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:69)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.copyFromLocalFile(S3AFileSystem.java:1494)
- at org.apache.hadoop.tools.cloudup.Cloudup.uploadOneFile(Cloudup.java:466)
- at org.apache.hadoop.tools.cloudup.Cloudup.access$000(Cloudup.java:63)
- at org.apache.hadoop.tools.cloudup.Cloudup$1.call(Cloudup.java:353)
- at org.apache.hadoop.tools.cloudup.Cloudup$1.call(Cloudup.java:350)
- at java.util.concurrent.FutureTask.run(FutureTask.java:266)
- at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
- at java.util.concurrent.FutureTask.run(FutureTask.java:266)
- at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
- at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
- at java.lang.Thread.run(Thread.java:748)
-Caused by: com.amazonaws.services.s3.model.AmazonS3Exception:
- The Content-MD5 you specified did not match what we received.
- (Service: Amazon S3; Status Code: 400; Error Code: BadDigest; Request ID: 4018131225),
- S3 Extended Request ID: null
- at com.amazonaws.http.AmazonHttpClient.handleErrorResponse(AmazonHttpClient.java:1307)
- at com.amazonaws.http.AmazonHttpClient.executeOneRequest(AmazonHttpClient.java:894)
- at com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:597)
- at com.amazonaws.http.AmazonHttpClient.doExecute(AmazonHttpClient.java:363)
- at com.amazonaws.http.AmazonHttpClient.executeWithTimer(AmazonHttpClient.java:329)
- at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:308)
- at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3659)
- at com.amazonaws.services.s3.AmazonS3Client.putObject(AmazonS3Client.java:1422)
- at com.amazonaws.services.s3.transfer.internal.UploadCallable.uploadInOneChunk(UploadCallable.java:131)
- at com.amazonaws.services.s3.transfer.internal.UploadCallable.call(UploadCallable.java:123)
- at com.amazonaws.services.s3.transfer.internal.UploadMonitor.call(UploadMonitor.java:139)
- at com.amazonaws.services.s3.transfer.internal.UploadMonitor.call(UploadMonitor.java:47)
- at org.apache.hadoop.fs.s3a.BlockingThreadPoolExecutorService$CallableWithPermitRelease.call(BlockingThreadPoolExecutorService.java:239)
- ... 4 more
-```
-
-This stack trace was seen when interacting with a third-party S3 store whose
-expectations of headers related to the AWS V4 signing mechanism was not
-compatible with that of the specific AWS SDK Hadoop was using.
+This has been seen with third party stores.
-Workaround: revert to V2 signing.
+If the store is configured to require content-MD5 headers with data uploaded: disable it.
-```xml
-<property>
- <name>fs.s3a.signing-algorithm</name>
- <value>S3SignerType</value>
-</property>
-```
+If the store requires the use of the v2 signing algorithm, know that it is unsupported on this release.
### When writing data: "java.io.FileNotFoundException: Completing multi-part upload"
@@ -1738,20 +1271,9 @@ with that ID.
```
java.io.FileNotFoundException: Completing multi-part upload on fork-5/test/multipart/1c397ca6-9dfb-4ac1-9cf7-db666673246b:
- com.amazonaws.services.s3.model.AmazonS3Exception: The specified upload does not exist.
+ S3Exception: The specified upload does not exist.
The upload ID may be invalid, or the upload may have been aborted or completed.
(Service: Amazon S3; Status Code: 404; Error Code: NoSuchUpload;
- at com.amazonaws.http.AmazonHttpClient.handleErrorResponse(AmazonHttpClient.java:1182)
- at com.amazonaws.http.AmazonHttpClient.executeOneRequest(AmazonHttpClient.java:770)
- at com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:489)
- at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:310)
- at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3785)
- at com.amazonaws.services.s3.AmazonS3Client.completeMultipartUpload(AmazonS3Client.java:2705)
- at org.apache.hadoop.fs.s3a.S3ABlockOutputStream$MultiPartUpload.complete(S3ABlockOutputStream.java:473)
- at org.apache.hadoop.fs.s3a.S3ABlockOutputStream$MultiPartUpload.access$200(S3ABlockOutputStream.java:382)
- at org.apache.hadoop.fs.s3a.S3ABlockOutputStream.close(S3ABlockOutputStream.java:272)
- at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72)
- at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106)
```
This can happen when all outstanding uploads have been aborted, including the
@@ -1839,37 +1361,16 @@ connections more frequently.
```
Cause: org.apache.hadoop.fs.s3a.AWSBadRequestException: put on :
- com.amazonaws.services.s3.model.AmazonS3Exception:
+ S3Exception:
The unspecified location constraint is incompatible for the region specific
endpoint this request was sent to.
(Service: Amazon S3; Status Code: 400; Error Code: IllegalLocationConstraintException;
-
- at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:178)
- at org.apache.hadoop.fs.s3a.S3ALambda.execute(S3ALambda.java:64)
- at org.apache.hadoop.fs.s3a.WriteOperationHelper.uploadObject(WriteOperationHelper.java:451)
- at org.apache.hadoop.fs.s3a.commit.magic.MagicCommitTracker.aboutToComplete(MagicCommitTracker.java:128)
- at org.apache.hadoop.fs.s3a.S3ABlockOutputStream.close(S3ABlockOutputStream.java:373)
- at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72)
- at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:101)
- at org.apache.hadoop.hive.ql.io.orc.WriterImpl.close(WriterImpl.java:2429)
- at org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat$OrcRecordWriter.close(OrcOutputFormat.java:106)
- at org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat$OrcRecordWriter.close(OrcOutputFormat.java:91)
...
- Cause: com.amazonaws.services.s3.model.AmazonS3Exception:
+ Cause: S3Exception:
The unspecified location constraint is incompatible for the region specific endpoint
this request was sent to. (Service: Amazon S3; Status Code: 400; Error Code: IllegalLocationConstraintException;
Request ID: EEBC5A08BCB3A645)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1588)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1258)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1030)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:742)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:716)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:699)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:667)
- at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:649)
- at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:513)
- at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4221)
- ...
+
```
Something has been trying to write data to "/".
@@ -1883,8 +1384,7 @@ more detail, as can S3A itself.
```properties
log4j.logger.org.apache.hadoop.fs.s3a=DEBUG
-log4j.logger.com.amazonaws.request=DEBUG
-log4j.logger.com.amazonaws.thirdparty.apache.http=DEBUG
+log4j.logger.software.amazon.awssdk.request=DEBUG
```
If using the "unshaded" JAR, then the Apache HttpClient can be directly configured:
@@ -1980,8 +1480,9 @@ The number of retries and interval between each retry can be configured:
</property>
```
-Not all failures are retried. Specifically excluded are those considered
-unrecoverable:
+Not all failures are retried *in the S3A Code* -though some may be retried within the
+AWS SDK.
+Specifically excluded are those considered unrecoverable:
* Low-level networking: `UnknownHostException`, `NoRouteToHostException`.
* 302 redirects.
@@ -2031,7 +1532,7 @@ If this value is configured too low, user may encounter `SdkClientException`s du
timing-out.
```
-com.amazonaws.SdkClientException: Unable to execute HTTP request:
+software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP request:
Request did not complete before the request timeout configuration.:
Unable to execute HTTP request: Request did not complete before the request timeout configuration.
at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:205)
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java
index 38c4685eb137..ded2f0b88507 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs.s3a;
import java.net.URI;
+import java.nio.file.AccessDeniedException;
import java.util.UUID;
import java.util.concurrent.Callable;
@@ -33,11 +34,13 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.LambdaTestUtils;
import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION;
import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_ACCESSPOINT_REQUIRED;
import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A;
+import static org.apache.hadoop.fs.s3a.Constants.PATH_STYLE_ACCESS;
import static org.apache.hadoop.fs.s3a.Constants.S3A_BUCKET_PROBE;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@@ -69,8 +72,14 @@ public class ITestS3ABucketExistence extends AbstractS3ATestBase {
assertTrue("getFileStatus on root should always return a directory",
fs.getFileStatus(root).isDirectory());
- expectUnknownStore(
- () -> fs.listStatus(root));
+ try {
+ expectUnknownStore(
+ () -> fs.listStatus(root));
+ } catch (AccessDeniedException e) {
+ // this is a sign that there's tests with a third-party bucket and
+ // interacting with aws is not going to authenticate
+ skip("no aws credentials");
+ }
Path src = new Path(root, "testfile");
Path dest = new Path(root, "dst");
@@ -129,7 +138,8 @@ public class ITestS3ABucketExistence extends AbstractS3ATestBase {
removeBaseAndBucketOverrides(conf,
S3A_BUCKET_PROBE,
ENDPOINT,
- AWS_REGION);
+ AWS_REGION,
+ PATH_STYLE_ACCESS);
conf.setInt(S3A_BUCKET_PROBE, probe);
conf.set(AWS_REGION, EU_WEST_1);
return conf;
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java
index 1c6fb9c3ab6a..e8dcca6df4ba 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java
@@ -40,7 +40,9 @@ import org.apache.hadoop.fs.s3a.impl.StoreContext;
import org.apache.hadoop.fs.store.audit.AuditSpan;
import static org.apache.hadoop.fs.s3a.Constants.CANNED_ACL;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfACLTestsDisabled;
/**
* Tests of ACL handling in the FS.
@@ -55,6 +57,8 @@ public class ITestS3ACannedACLs extends AbstractS3ATestBase {
@Override
protected Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
+ skipIfACLTestsDisabled(conf);
+ disableFilesystemCaching(conf);
removeBaseAndBucketOverrides(conf,
CANNED_ACL);
conf.set(CANNED_ACL, LOG_DELIVERY_WRITE);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
index 5570efe64100..60037e58cd61 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
@@ -19,8 +19,9 @@
package org.apache.hadoop.fs.s3a;
import java.io.File;
+import java.io.IOException;
+import java.net.ConnectException;
import java.net.URI;
-import java.nio.file.AccessDeniedException;
import java.security.PrivilegedExceptionAction;
import org.assertj.core.api.Assertions;
@@ -157,7 +158,7 @@ public class ITestS3AConfiguration {
conf.setInt(Constants.PROXY_PORT, 1);
String proxy =
conf.get(Constants.PROXY_HOST) + ":" + conf.get(Constants.PROXY_PORT);
- expectFSCreateFailure(AWSClientIOException.class,
+ expectFSCreateFailure(ConnectException.class,
conf, "when using proxy " + proxy);
}
@@ -211,10 +212,10 @@ public class ITestS3AConfiguration {
conf.unset(Constants.PROXY_PORT);
conf.set(Constants.PROXY_HOST, "127.0.0.1");
conf.set(Constants.SECURE_CONNECTIONS, "true");
- expectFSCreateFailure(AWSClientIOException.class,
+ expectFSCreateFailure(ConnectException.class,
conf, "Expected a connection error for proxy server");
conf.set(Constants.SECURE_CONNECTIONS, "false");
- expectFSCreateFailure(AWSClientIOException.class,
+ expectFSCreateFailure(ConnectException.class,
conf, "Expected a connection error for proxy server");
}
@@ -552,6 +553,7 @@ public class ITestS3AConfiguration {
config.set(SIGNING_ALGORITHM_STS, "CustomSTSSigner");
config.set(AWS_REGION, EU_WEST_1);
+ disableFilesystemCaching(config);
fs = S3ATestUtils.createTestFileSystem(config);
S3Client s3Client = getS3Client("testS3SpecificSignerOverride");
@@ -564,7 +566,7 @@ public class ITestS3AConfiguration {
intercept(StsException.class, "", () ->
stsClient.getSessionToken());
- intercept(AccessDeniedException.class, "", () ->
+ final IOException ioe = intercept(IOException.class, "", () ->
Invoker.once("head", bucket, () ->
s3Client.headBucket(HeadBucketRequest.builder().bucket(bucket).build())));
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AContentEncoding.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AContentEncoding.java
index 4a96bf5da91d..ed86143100ac 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AContentEncoding.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AContentEncoding.java
@@ -29,12 +29,16 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import static org.apache.hadoop.fs.s3a.Constants.CONTENT_ENCODING;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.raiseAsAssumption;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfNotEnabled;
import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.XA_CONTENT_ENCODING;
import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.decodeBytes;
/**
* Tests of content encoding object meta data.
+ * Some stores don't support gzip; rejection of the content encoding
+ * is downgraded to a skipped test.
*/
public class ITestS3AContentEncoding extends AbstractS3ATestBase {
@@ -43,6 +47,8 @@ public class ITestS3AContentEncoding extends AbstractS3ATestBase {
@Override
protected Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
+ skipIfNotEnabled(conf, KEY_CONTENT_ENCODING_ENABLED,
+ "Skipping storage class ACL tests");
removeBaseAndBucketOverrides(conf, CONTENT_ENCODING);
conf.set(CONTENT_ENCODING, GZIP);
@@ -51,20 +57,25 @@ public class ITestS3AContentEncoding extends AbstractS3ATestBase {
@Test
public void testCreatedObjectsHaveEncoding() throws Throwable {
- S3AFileSystem fs = getFileSystem();
- Path dir = methodPath();
- fs.mkdirs(dir);
- // even with content encoding enabled, directories do not have
- // encoding.
- Assertions.assertThat(getEncoding(dir))
- .describedAs("Encoding of object %s", dir)
- .isNull();
- Path path = new Path(dir, "1");
- ContractTestUtils.touch(fs, path);
- assertObjectHasEncoding(path);
- Path path2 = new Path(dir, "2");
- fs.rename(path, path2);
- assertObjectHasEncoding(path2);
+ try {
+ S3AFileSystem fs = getFileSystem();
+ Path dir = methodPath();
+ fs.mkdirs(dir);
+ // even with content encoding enabled, directories do not have
+ // encoding.
+ Assertions.assertThat(getEncoding(dir))
+ .describedAs("Encoding of object %s", dir)
+ .isNull();
+ Path path = new Path(dir, "1");
+ ContractTestUtils.touch(fs, path);
+ assertObjectHasEncoding(path);
+ Path path2 = new Path(dir, "2");
+ fs.rename(path, path2);
+ assertObjectHasEncoding(path2);
+ } catch (AWSUnsupportedFeatureException e) {
+ LOG.warn("Object store does not support {} content encoding", GZIP, e);
+ raiseAsAssumption(e);
+ }
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AContractGetFileStatusV1List.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AContractGetFileStatusV1List.java
index 6d950d9bfaaa..e07324300197 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AContractGetFileStatusV1List.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AContractGetFileStatusV1List.java
@@ -24,7 +24,9 @@ import org.apache.hadoop.fs.contract.AbstractFSContract;
import org.apache.hadoop.fs.contract.s3a.S3AContract;
import static org.apache.hadoop.fs.s3a.Constants.LIST_VERSION;
+import static org.apache.hadoop.fs.s3a.S3ATestConstants.KEY_LIST_V1_ENABLED;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfNotEnabled;
/**
* S3A contract tests for getFileStatus, using the v1 List Objects API.
@@ -48,6 +50,8 @@ public class ITestS3AContractGetFileStatusV1List
protected Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
disableFilesystemCaching(conf);
+ skipIfNotEnabled(conf, KEY_LIST_V1_ENABLED,
+ "Skipping V1 listing tests");
conf.setInt(Constants.MAX_PAGING_KEYS, 2);
// Use v1 List Objects API
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java
index e5e109ad91b5..eb010674beed 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.fs.s3a;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.net.UnknownHostException;
+import java.nio.file.AccessDeniedException;
import java.util.ArrayList;
import java.util.List;
@@ -38,7 +40,10 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext;
import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION;
+import static org.apache.hadoop.fs.s3a.Constants.PATH_STYLE_ACCESS;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.Statistic.STORE_REGION_PROBE;
+import static org.apache.hadoop.io.IOUtils.closeStream;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
/**
@@ -49,6 +54,26 @@ public class ITestS3AEndpointRegion extends AbstractS3ATestBase {
private static final String AWS_ENDPOINT_TEST = "test-endpoint";
+ private static final String USW_2_BUCKET = "landsat-pds";
+
+ public static final String USW_2_STORE = "s3a://" + USW_2_BUCKET;
+
+ /**
+ * If anyone were ever to create a bucket with this UUID pair it would break the tests.
+ */
+ public static final String UNKNOWN_BUCKET = "23FA76D4-5F17-48B8-9D7D-9050269D0E40"
+ + "-8281BAF2-DBCF-47AA-8A27-F2FA3589656A";
+
+ /**
+ * New FS instance which will be closed in teardown.
+ */
+ private S3AFileSystem newFS;
+
+ @Override
+ public void teardown() throws Exception {
+ closeStream(newFS);
+ super.teardown();
+ }
/**
* Test to verify that not setting the region config, will lead to the client factory making
@@ -57,50 +82,80 @@ public class ITestS3AEndpointRegion extends AbstractS3ATestBase {
*/
@Test
public void testWithoutRegionConfig() throws IOException {
+ describe("Verify that region lookup takes place");
+
Configuration conf = getConfiguration();
- String bucket = getFileSystem().getBucket();
- conf.unset(String.format("fs.s3a.bucket.%s.endpoint.region", bucket));
- conf.unset(AWS_REGION);
+ removeBaseAndBucketOverrides(conf, AWS_REGION, PATH_STYLE_ACCESS);
+ conf.setBoolean(PATH_STYLE_ACCESS, false);
+
+ newFS = new S3AFileSystem();
- S3AFileSystem fs = new S3AFileSystem();
- fs.initialize(getFileSystem().getUri(), conf);
+ try {
+ newFS.initialize(getFileSystem().getUri(), conf);
+ newFS.getS3AInternals().getBucketMetadata();
+ } catch (UnknownHostException | UnknownStoreException | AccessDeniedException allowed) {
+ // these are all valid failure modes from different test environments.
+ }
+ assertRegionProbeCount(1);
+ }
+
+ @Test
+ public void testUnknownBucket() throws Exception {
+ describe("Verify unknown bucket probe failure");
+ Configuration conf = getConfiguration();
+ removeBaseAndBucketOverrides(UNKNOWN_BUCKET, conf, AWS_REGION, PATH_STYLE_ACCESS);
- fs.getS3AInternals().getBucketMetadata();
+ newFS = new S3AFileSystem();
- Assertions.assertThat(fs.getInstrumentation().getCounterValue(STORE_REGION_PROBE))
- .describedAs("Region is not configured, region probe should have been made").isEqualTo(1);
+ try {
+ newFS.initialize(new URI("s3a://" + UNKNOWN_BUCKET), conf);
+ newFS.getS3AInternals().getBucketMetadata();
+ // expect a failure by here
+ fail("Expected failure, got " + newFS);
+ } catch (UnknownHostException | UnknownStoreException expected) {
+ // this is good.
+ }
+ assertRegionProbeCount(1);
}
@Test
public void testWithRegionConfig() throws IOException, URISyntaxException {
+ describe("Verify that region lookup is skipped if the region property is set");
Configuration conf = getConfiguration();
- conf.set(AWS_REGION, "us-east-2");
+ removeBaseAndBucketOverrides(conf, AWS_REGION, PATH_STYLE_ACCESS);
- S3AFileSystem fs = new S3AFileSystem();
- fs.initialize(new URI("s3a://landsat-pds"), conf);
+ conf.set(AWS_REGION, "us-east-2");
- Assertions.assertThat(fs.getInstrumentation().getCounterValue(STORE_REGION_PROBE))
- .describedAs("Region is configured, region probe should not have been made").isEqualTo(0);
+ newFS = new S3AFileSystem();
+ newFS.initialize(new URI(USW_2_STORE), conf);
+ assertRegionProbeCount(0);
}
@Test
public void testRegionCache() throws IOException, URISyntaxException {
+ describe("Verify that region lookup is cached on the second attempt");
Configuration conf = getConfiguration();
- conf.unset(AWS_REGION);
- conf.unset("fs.s3a.bucket.landsat-pds.endpoint.region");
- S3AFileSystem fs = new S3AFileSystem();
+ removeBaseAndBucketOverrides(USW_2_BUCKET, conf, AWS_REGION, PATH_STYLE_ACCESS);
+
+ newFS = new S3AFileSystem();
- fs.initialize(new URI("s3a://landsat-pds"), conf);
+ newFS.initialize(new URI(USW_2_STORE), conf);
- Assertions.assertThat(fs.getInstrumentation().getCounterValue(STORE_REGION_PROBE))
- .describedAs("Incorrect number of calls made to get bucket region").isEqualTo(1);
+ assertRegionProbeCount(1);
+ closeStream(newFS);
- fs.initialize(new URI("s3a://landsat-pds"), conf);
+ // create a new instance
+ newFS = new S3AFileSystem();
+ newFS.initialize(new URI(USW_2_STORE), conf);
// value should already be cached.
- Assertions.assertThat(fs.getInstrumentation().getCounterValue(STORE_REGION_PROBE))
- .describedAs("Incorrect number of calls made to get bucket region").isEqualTo(0);
+ assertRegionProbeCount(0);
+ }
+
+ private void assertRegionProbeCount(final int expected) {
+ Assertions.assertThat(newFS.getInstrumentation().getCounterValue(STORE_REGION_PROBE))
+ .describedAs("Incorrect number of calls made to get bucket region").isEqualTo(expected);
}
@Test
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
index a741b11b0ce4..7c53f9c32a66 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
@@ -45,6 +45,7 @@ import java.util.stream.Collectors;
import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.createFiles;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.isBulkDeleteEnabled;
import static org.apache.hadoop.fs.s3a.test.ExtraAssertions.failIf;
import static org.apache.hadoop.test.LambdaTestUtils.*;
import static org.apache.hadoop.util.functional.RemoteIterators.mappingRemoteIterator;
@@ -89,7 +90,22 @@ public class ITestS3AFailureHandling extends AbstractS3ATestBase {
S3AFileSystem fs = getFileSystem();
Path path = path("largeDir");
mkdirs(path);
- createFiles(fs, path, 1, 1005, 0);
+ final boolean bulkDeleteEnabled = isBulkDeleteEnabled(getFileSystem());
+
+ // with single object delete, only create a few files for a faster
+ // test run.
+ int filesToCreate;
+ int pages;
+ if (bulkDeleteEnabled) {
+ filesToCreate = 1005;
+ pages = 5;
+ } else {
+ filesToCreate = 250;
+ pages = 0;
+ }
+
+
+ createFiles(fs, path, 1, filesToCreate, 0);
RemoteIterator<LocatedFileStatus> locatedFileStatusRemoteIterator =
fs.listFiles(path, false);
List<String> keys = toList(mappingRemoteIterator(locatedFileStatusRemoteIterator,
@@ -102,9 +118,10 @@ public class ITestS3AFailureHandling extends AbstractS3ATestBase {
}
Long bulkDeleteReqAfter = getNumberOfBulkDeleteRequestsMadeTillNow(fs);
// number of delete requests is 5 as we have default page size of 250.
+
Assertions.assertThat(bulkDeleteReqAfter - bulkDeleteReqBefore)
.describedAs("Number of batched bulk delete requests")
- .isEqualTo(5);
+ .isEqualTo(pages);
}
private Long getNumberOfBulkDeleteRequestsMadeTillNow(S3AFileSystem fs) {
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java
index 4aa9c6a52eaa..66a211181bdd 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java
@@ -182,14 +182,8 @@ public class ITestS3AMiscOperations extends AbstractS3ATestBase {
* non-null.
*/
private void assumeNoDefaultEncryption() throws IOException {
- try {
- skipIfClientSideEncryption();
- Assume.assumeThat(getDefaultEncryption(), nullValue());
- } catch (AccessDeniedException e) {
- // if the user can't check the default encryption, assume that it is
- // null and keep going
- LOG.warn("User does not have permission to call getBucketEncryption()");
- }
+ skipIfClientSideEncryption();
+ Assume.assumeThat(getDefaultEncryption(), nullValue());
}
/**
@@ -418,7 +412,7 @@ public class ITestS3AMiscOperations extends AbstractS3ATestBase {
() -> s3.getBucketEncryption(GetBucketEncryptionRequest.builder()
.bucket(fs.getBucket())
.build()));
- } catch (FileNotFoundException e) {
+ } catch (FileNotFoundException | AccessDeniedException | AWSBadRequestException e) {
return null;
}
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMultipartUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMultipartUtils.java
index 197811f39fb9..263a857e0330 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMultipartUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMultipartUtils.java
@@ -18,18 +18,19 @@
package org.apache.hadoop.fs.s3a;
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
import software.amazon.awssdk.services.s3.model.MultipartUpload;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.store.audit.AuditSpan;
-import org.junit.Test;
-
-
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
+import static org.apache.hadoop.util.functional.RemoteIterators.foreach;
/**
* Tests for {@link MultipartUtils}.
@@ -101,17 +102,18 @@ public class ITestS3AMultipartUtils extends AbstractS3ATestBase {
// Don't modify passed-in set, use copy.
Set<MultipartTestUtils.IdKey> uploads = new HashSet<>(ourUploads);
- while (list.hasNext()) {
- MultipartTestUtils.IdKey listing = toIdKey(list.next());
- if (uploads.contains(listing)) {
+ foreach(list, (upload) -> {
+ MultipartTestUtils.IdKey listing = toIdKey(upload);
+ if (uploads.remove(listing)) {
LOG.debug("Matched: {},{}", listing.getKey(), listing.getUploadId());
- uploads.remove(listing);
} else {
LOG.debug("Not our upload {},{}", listing.getKey(),
listing.getUploadId());
}
- }
- assertTrue("Not all our uploads were listed", uploads.isEmpty());
+ });
+ Assertions.assertThat(uploads)
+ .describedAs("Uploads which we expected to be listed.")
+ .isEmpty();
}
private MultipartTestUtils.IdKey toIdKey(MultipartUpload mu) {
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AStorageClass.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AStorageClass.java
index 6ccb7ac26040..56cb541e8233 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AStorageClass.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AStorageClass.java
@@ -74,6 +74,7 @@ public class ITestS3AStorageClass extends AbstractS3ATestBase {
@Override
protected Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
+ skipIfStorageClassTestsDisabled(conf);
disableFilesystemCaching(conf);
removeBaseAndBucketOverrides(conf, STORAGE_CLASS, FAST_UPLOAD_BUFFER);
conf.set(FAST_UPLOAD_BUFFER, fastUploadBufferType);
@@ -81,12 +82,6 @@ public class ITestS3AStorageClass extends AbstractS3ATestBase {
return conf;
}
- @Override
- public void setup() throws Exception {
- super.setup();
- skipIfStorageClassTestsDisabled(getConfiguration());
- }
-
/*
* This test ensures the default storage class configuration (no config or null)
* works well with create and copy operations
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java
index 2b7620ddbddf..3e343a9ea85f 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java
@@ -199,5 +199,13 @@ public final class MultipartTestUtils {
public int hashCode() {
return Objects.hash(key, uploadId);
}
+
+ @Override
+ public String toString() {
+ return "IdKey{" +
+ "key='" + key + '\'' +
+ ", uploadId='" + uploadId + '\'' +
+ '}';
+ }
}
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
index 246d111d14b8..f2e0223adb1d 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
@@ -58,6 +58,21 @@ public interface S3ATestConstants {
*/
String KEY_STORAGE_CLASS_TESTS_ENABLED = TEST_FS_S3A + "create.storage.class.enabled";
+ /**
+ * A property set to true if ACL tests are enabled: {@value}.
+ */
+ String KEY_ACL_TESTS_ENABLED = TEST_FS_S3A + "create.acl.enabled";
+
+ /**
+ * A property set to true if V1 tests are enabled: {@value}.
+ */
+ String KEY_LIST_V1_ENABLED = TEST_FS_S3A + "list.v1.enabled";
+
+ /**
+ * A property set to true if content encoding tests are enabled: {@value}.
+ */
+ String KEY_CONTENT_ENCODING_ENABLED = TEST_FS_S3A + "content.encoding.enabled";
+
/**
* Tell tests that they are being executed in parallel: {@value}.
*/
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
index 239e52a7264a..3382c300b931 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
@@ -61,12 +61,13 @@ import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.functional.CallableRaisingIOE;
import org.apache.hadoop.util.functional.FutureIO;
-import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import org.assertj.core.api.Assertions;
import org.junit.Assert;
import org.junit.Assume;
+import org.junit.AssumptionViolatedException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import java.io.Closeable;
import java.io.File;
@@ -466,8 +467,20 @@ public final class S3ATestUtils {
*/
public static void skipIfEncryptionTestsDisabled(
Configuration configuration) {
- if (!configuration.getBoolean(KEY_ENCRYPTION_TESTS, true)) {
- skip("Skipping encryption tests");
+ skipIfNotEnabled(configuration, KEY_ENCRYPTION_TESTS, "Skipping encryption tests");
+ }
+
+ /**
+ * Skip a test suite/casee if a configuration has been explicitly disabled.
+ * @param configuration configuration to probe
+ * @param key key to resolve
+ * @param message assertion text
+ */
+ public static void skipIfNotEnabled(final Configuration configuration,
+ final String key,
+ final String message) {
+ if (!configuration.getBoolean(key, true)) {
+ skip(message);
}
}
@@ -477,9 +490,18 @@ public final class S3ATestUtils {
*/
public static void skipIfStorageClassTestsDisabled(
Configuration configuration) {
- if (!configuration.getBoolean(KEY_STORAGE_CLASS_TESTS_ENABLED, true)) {
- skip("Skipping storage class tests");
- }
+ skipIfNotEnabled(configuration, KEY_STORAGE_CLASS_TESTS_ENABLED,
+ "Skipping storage class tests");
+ }
+
+ /**
+ * Skip a test if ACL class tests are disabled.
+ * @param configuration configuration to probe
+ */
+ public static void skipIfACLTestsDisabled(
+ Configuration configuration) {
+ skipIfNotEnabled(configuration, KEY_ACL_TESTS_ENABLED,
+ "Skipping storage class ACL tests");
}
/**
@@ -635,9 +657,7 @@ public final class S3ATestUtils {
* @param conf configuration to examine
*/
public static void assumeSessionTestsEnabled(final Configuration conf) {
- if (!conf.getBoolean(TEST_STS_ENABLED, true)) {
- skip("STS functional tests disabled");
- }
+ skipIfNotEnabled(conf, TEST_STS_ENABLED, "STS functional tests disabled");
}
/**
@@ -1245,6 +1265,14 @@ public final class S3ATestUtils {
Assume.assumeTrue(message, condition);
}
+ /**
+ * Convert a throwable to an assumption failure.
+ * @param t thrown exception.
+ */
+ public static void raiseAsAssumption(Throwable t) {
+ throw new AssumptionViolatedException(t.toString(), t);
+ }
+
/**
* Get the statistics from a wrapped block output stream.
* @param out output stream
@@ -1515,4 +1543,15 @@ public final class S3ATestUtils {
public static void disablePrefetching(Configuration conf) {
removeBaseAndBucketOverrides(conf, PREFETCH_ENABLED_KEY);
}
+
+ /**
+ * Does this FS support multi object delete?
+ * @param fs filesystem
+ * @return true if multi-delete is enabled.
+ */
+
+ public static boolean isBulkDeleteEnabled(FileSystem fs) {
+ return fs.getConf().getBoolean(Constants.ENABLE_MULTI_DELETE,
+ true);
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java
index 0ac49812e4cb..f6e8f0c376d9 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java
@@ -43,6 +43,7 @@ import static org.apache.hadoop.fs.s3a.Constants.*;
import static org.apache.hadoop.fs.s3a.Invoker.*;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.verifyExceptionClass;
import static org.apache.hadoop.fs.s3a.S3AUtils.*;
+import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_400_BAD_REQUEST;
import static org.apache.hadoop.test.LambdaTestUtils.*;
/**
@@ -104,7 +105,7 @@ public class TestInvoker extends Assert {
.cause(new Local.ConnectTimeoutException("timeout"))
.build();
private static final AwsServiceException BAD_REQUEST = serviceException(
- AWSBadRequestException.STATUS_CODE,
+ SC_400_BAD_REQUEST,
"bad request");
@Before
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java
index ad7d59a7319c..fe48cd891157 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java
@@ -70,12 +70,15 @@ public class ITestCustomSigner extends AbstractS3ATestBase {
public void setup() throws Exception {
super.setup();
final S3AFileSystem fs = getFileSystem();
- regionName = determineRegion(fs.getBucket());
+ final Configuration conf = fs.getConf();
+ endpoint = conf.getTrimmed(Constants.ENDPOINT, Constants.CENTRAL_ENDPOINT);
+ LOG.info("Test endpoint is {}", endpoint);
+ regionName = conf.getTrimmed(Constants.AWS_REGION, "");
+ if (regionName.isEmpty()) {
+ regionName = determineRegion(fs.getBucket());
+ }
LOG.info("Determined region name to be [{}] for bucket [{}]", regionName,
fs.getBucket());
- endpoint = fs.getConf()
- .get(Constants.ENDPOINT, Constants.CENTRAL_ENDPOINT);
- LOG.info("Test endpoint is {}", endpoint);
}
@Test
@@ -118,11 +121,14 @@ public class ITestCustomSigner extends AbstractS3ATestBase {
.isGreaterThan(invocationCount);
Assertions.assertThat(CustomSigner.lastStoreValue)
- .as("Store value should not be null").isNotNull();
+ .as("Store value should not be null in %s", CustomSigner.description())
+ .isNotNull();
Assertions.assertThat(CustomSigner.lastStoreValue.conf)
- .as("Configuration should not be null").isNotNull();
+ .as("Configuration should not be null in %s", CustomSigner.description())
+ .isNotNull();
Assertions.assertThat(CustomSigner.lastStoreValue.conf.get(TEST_ID_KEY))
- .as("Configuration TEST_KEY mismatch").isEqualTo(identifier);
+ .as("Configuration TEST_KEY mismatch in %s", CustomSigner.description())
+ .isEqualTo(identifier);
return fs;
});
@@ -196,8 +202,9 @@ public class ITestCustomSigner extends AbstractS3ATestBase {
try {
lastStoreValue = CustomSignerInitializer
.getStoreValue(bucketName, UserGroupInformation.getCurrentUser());
+ LOG.info("Store value for bucket {} is {}", bucketName, lastStoreValue);
} catch (IOException e) {
- throw new RuntimeException("Failed to get current Ugi", e);
+ throw new RuntimeException("Failed to get current Ugi " + e, e);
}
if (bucketName.equals("kms")) {
Aws4Signer realKMSSigner = Aws4Signer.create();
@@ -247,6 +254,14 @@ public class ITestCustomSigner extends AbstractS3ATestBase {
public static int getInvocationCount() {
return INVOCATION_COUNT.get();
}
+
+ public static String description() {
+ return "CustomSigner{"
+ + "invocations=" + INVOCATION_COUNT.get()
+ + ", instantiations=" + INSTANTIATION_COUNT.get()
+ + ", lastStoreValue=" + lastStoreValue
+ + "}";
+ }
}
@Private
@@ -260,6 +275,7 @@ public class ITestCustomSigner extends AbstractS3ATestBase {
DelegationTokenProvider dtProvider, UserGroupInformation storeUgi) {
StoreKey storeKey = new StoreKey(bucketName, storeUgi);
StoreValue storeValue = new StoreValue(storeConf, dtProvider);
+ LOG.info("Registering store {} with value {}", storeKey, storeValue);
knownStores.put(storeKey, storeValue);
}
@@ -267,6 +283,7 @@ public class ITestCustomSigner extends AbstractS3ATestBase {
public void unregisterStore(String bucketName, Configuration storeConf,
DelegationTokenProvider dtProvider, UserGroupInformation storeUgi) {
StoreKey storeKey = new StoreKey(bucketName, storeUgi);
+ LOG.info("Unregistering store {}", storeKey);
knownStores.remove(storeKey);
}
@@ -302,6 +319,14 @@ public class ITestCustomSigner extends AbstractS3ATestBase {
public int hashCode() {
return Objects.hash(bucketName, ugi);
}
+
+ @Override
+ public String toString() {
+ return "StoreKey{" +
+ "bucketName='" + bucketName + '\'' +
+ ", ugi=" + ugi +
+ '}';
+ }
}
static class StoreValue {
@@ -313,6 +338,14 @@ public class ITestCustomSigner extends AbstractS3ATestBase {
this.conf = conf;
this.dtProvider = dtProvider;
}
+
+ @Override
+ public String toString() {
+ return "StoreValue{" +
+ "conf=" + conf +
+ ", dtProvider=" + dtProvider +
+ '}';
+ }
}
}
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestSignerManager.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestSignerManager.java
index 595e2687276b..b56b8c20bfe7 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestSignerManager.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestSignerManager.java
@@ -24,7 +24,6 @@ import java.security.PrivilegedExceptionAction;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
-import java.util.concurrent.TimeUnit;
import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
import software.amazon.awssdk.core.signer.Signer;
@@ -32,26 +31,27 @@ import software.amazon.awssdk.http.SdkHttpFullRequest;
import software.amazon.awssdk.http.SdkHttpMethod;
import org.assertj.core.api.Assertions;
import org.junit.Before;
-import org.junit.Rule;
import org.junit.Test;
-import org.junit.rules.Timeout;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.s3a.auth.TestSignerManager.SignerInitializerForTest.StoreValue;
import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider;
+import org.apache.hadoop.fs.s3a.impl.InstantiationIOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.test.AbstractHadoopTestBase;
import static org.apache.hadoop.fs.s3a.Constants.CUSTOM_SIGNERS;
+import static org.apache.hadoop.fs.s3a.auth.SignerFactory.S3_V2_SIGNER;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
/**
* Tests for the SignerManager.
*/
-public class TestSignerManager {
+public class TestSignerManager extends AbstractHadoopTestBase {
private static final Text TEST_TOKEN_KIND = new Text("TestTokenKind");
private static final Text TEST_TOKEN_SERVICE = new Text("TestTokenService");
@@ -61,9 +61,6 @@ public class TestSignerManager {
private static final String TESTUSER1 = "testuser1";
private static final String TESTUSER2 = "testuser2";
- @Rule public Timeout testTimeout = new Timeout(10_000L,
- TimeUnit.MILLISECONDS);
-
@Before
public void beforeTest() {
SignerForTest1.reset();
@@ -95,11 +92,8 @@ public class TestSignerManager {
// Make sure the config is respected.
signerManager.initCustomSigners();
// Simulate a call from the AWS SDK to create the signer.
- LambdaTestUtils.intercept(Exception.class,
+ intercept(InstantiationIOException.class,
() -> SignerFactory.createSigner("testsignerUnregistered", null));
- // Expecting generic Exception.class to handle future implementation
- // changes.
- // For now, this is an NPE
}
@Test
@@ -588,4 +582,17 @@ public class TestSignerManager {
return SdkHttpFullRequest.builder().host(host).protocol("https").method(SdkHttpMethod.GET)
.build();
}
+
+ @Test
+ public void testV2SignerRejected() throws Throwable {
+ intercept(InstantiationIOException.class, "no longer supported",
+ () -> SignerFactory.createSigner(S3_V2_SIGNER, "key"));
+ }
+
+ @Test
+ public void testUnknownSignerRejected() throws Throwable {
+ intercept(InstantiationIOException.class, "unknownSigner",
+ () -> SignerFactory.createSigner("unknownSigner", "key"));
+ }
+
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java
index 613fab192874..116d48e9de5f 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Map;
import org.assertj.core.api.Assertions;
+import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,8 +40,10 @@ import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
import org.apache.hadoop.fs.s3a.commit.impl.CommitContext;
import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations;
import org.apache.hadoop.fs.s3a.scale.AbstractSTestS3AHugeFiles;
+import org.apache.hadoop.fs.store.audit.AuditSpan;
import static org.apache.hadoop.fs.s3a.MultipartTestUtils.listMultipartUploads;
+import static org.apache.hadoop.fs.s3a.Statistic.MULTIPART_UPLOAD_ABORT_UNDER_PATH_INVOKED;
import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.extractXAttrLongValue;
@@ -118,6 +121,19 @@ public class ITestS3AHugeMagicCommits extends AbstractSTestS3AHugeFiles {
return magicOutputFile;
}
+ @Test
+ public void test_000_CleanupPendingUploads() throws IOException {
+ describe("Cleanup any existing pending uploads");
+ final S3AFileSystem fs = getFileSystem();
+ final String key = fs.pathToKey(finalDirectory);
+ final AuditSpan span = fs.getAuditSpanSource().createSpan(
+ MULTIPART_UPLOAD_ABORT_UNDER_PATH_INVOKED.getSymbol(),
+ key, null);
+ final int count = fs.createWriteOperationHelper(span)
+ .abortMultipartUploadsUnderPath(key + "/");
+ LOG.info("Aborted {} uploads under {}", count, key);
+ }
+
@Override
public void test_030_postCreationAssertions() throws Throwable {
describe("Committing file");
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestXAttrCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestXAttrCost.java
index 3a390e34ecad..f6fe68b8e6d3 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestXAttrCost.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestXAttrCost.java
@@ -37,7 +37,6 @@ import static org.apache.hadoop.fs.s3a.Statistic.INVOCATION_OP_XATTR_LIST;
import static org.apache.hadoop.fs.s3a.Statistic.INVOCATION_XATTR_GET_MAP;
import static org.apache.hadoop.fs.s3a.Statistic.INVOCATION_XATTR_GET_NAMED;
import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.CONTENT_TYPE_OCTET_STREAM;
-import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.CONTENT_TYPE_APPLICATION_XML;
import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.CONTENT_TYPE_X_DIRECTORY;
import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.XA_CONTENT_LENGTH;
import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.XA_CONTENT_TYPE;
@@ -74,15 +73,12 @@ public class ITestXAttrCost extends AbstractS3ACostTest {
fs.listXAttrs(root),
with(INVOCATION_OP_XATTR_LIST, GET_METADATA_ON_OBJECT));
- // verify this contains all the standard markers,
- // but not the magic marker header
+ // don't make any assertions on the headers entries
+ // as different S3 providers may have different headers
+ // and they may even change over time.
Assertions.assertThat(headerList)
.describedAs("Headers on root object")
- .containsOnly(
- XA_CONTENT_LENGTH,
- XA_CONTENT_TYPE);
- assertHeaderEntry(xAttrs, XA_CONTENT_TYPE)
- .isEqualTo(CONTENT_TYPE_APPLICATION_XML);
+ .hasSize(xAttrs.size());
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java
new file mode 100644
index 000000000000..71536880dd5c
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+import java.io.IOException;
+import java.net.ConnectException;
+import java.net.NoRouteToHostException;
+import java.net.UnknownHostException;
+
+import org.junit.Test;
+import software.amazon.awssdk.core.exception.SdkClientException;
+
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractNetworkException;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Unit tests related to the {@link ErrorTranslation} class.
+ */
+public class TestErrorTranslation extends AbstractHadoopTestBase {
+
+ /**
+ * Create an sdk exception with the given cause.
+ * @param message error message
+ * @param cause cause
+ * @return a new exception
+ */
+ private static SdkClientException sdkException(
+ String message,
+ Throwable cause) {
+ return SdkClientException.builder()
+ .message(message)
+ .cause(cause)
+ .build();
+ }
+
+ @Test
+ public void testUnknownHostExceptionExtraction() throws Throwable {
+ final SdkClientException thrown = sdkException("top",
+ sdkException("middle",
+ new UnknownHostException("bottom")));
+ final IOException ioe = intercept(UnknownHostException.class, "top",
+ () -> {
+ throw maybeExtractNetworkException("", thrown);
+ });
+
+ // the wrapped exception is the top level one: no stack traces have
+ // been lost
+ if (ioe.getCause() != thrown) {
+ throw new AssertionError("Cause of " + ioe + " is not " + thrown, thrown);
+ }
+
+ }
+
+ @Test
+ public void testNoRouteToHostExceptionExtraction() throws Throwable {
+ intercept(NoRouteToHostException.class, "top",
+ () -> {
+ throw maybeExtractNetworkException("p2",
+ sdkException("top",
+ sdkException("middle",
+ new NoRouteToHostException("bottom"))));
+ });
+ }
+
+ @Test
+ public void testConnectExceptionExtraction() throws Throwable {
+ intercept(ConnectException.class, "top",
+ () -> {
+ throw maybeExtractNetworkException("p1",
+ sdkException("top",
+ sdkException("middle",
+ new ConnectException("bottom"))));
+ });
+ }
+ @Test
+ public void testNoConstructorExtraction() throws Throwable {
+ intercept(PathIOException.class, NoConstructorIOE.MESSAGE,
+ () -> {
+ throw maybeExtractNetworkException("p1",
+ sdkException("top",
+ sdkException("middle",
+ new NoConstructorIOE())));
+ });
+ }
+
+
+ public static final class NoConstructorIOE extends IOException {
+
+ public static final String MESSAGE = "no-arg constructor";
+
+ public NoConstructorIOE() {
+ super(MESSAGE);
+ }
+ }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java
index 8bbf52b578e1..48378ce75dc9 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.fs.FSDataOutputStreamBuilder;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
-import org.apache.hadoop.fs.s3a.Constants;
import org.apache.hadoop.fs.s3a.S3AFileStatus;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.Statistic;
@@ -135,9 +134,7 @@ public class AbstractS3ACostTest extends AbstractS3ATestBase {
setupCostValidator();
// determine bulk delete settings
- final Configuration fsConf = getFileSystem().getConf();
- isBulkDelete = fsConf.getBoolean(Constants.ENABLE_MULTI_DELETE,
- true);
+ isBulkDelete = isBulkDeleteEnabled(getFileSystem());
deleteMarkerStatistic = isBulkDelete()
? OBJECT_BULK_DELETE_REQUEST
: OBJECT_DELETE_REQUEST;
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java
index 48791f557c00..cd4ee4440667 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java
@@ -455,6 +455,7 @@ public class ITestDirectoryMarkerListing extends AbstractS3ATestBase {
Path src = basePath;
Path dest = new Path(methodPath(), "dest");
+ getFileSystem().delete(dest, true);
assertRenamed(src, dest);
assertPathDoesNotExist("source", src);
@@ -610,15 +611,18 @@ public class ITestDirectoryMarkerListing extends AbstractS3ATestBase {
RequestBody.fromString(content)));
}
/**
- * Delete an object.
+ * Delete an object; exceptions are swallowed.
* @param key key
- * @param content string
*/
private void deleteObject(final String key) throws Exception {
- exec("DELETE " + key, () -> {
- s3client.deleteObject(b -> b.bucket(bucket).key(key));
- return "deleted " + key;
- });
+ try {
+ exec("DELETE " + key, () -> {
+ s3client.deleteObject(b -> b.bucket(bucket).key(key));
+ return "deleted " + key;
+ });
+ } catch (IOException ignored) {
+
+ }
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java
index be4de7942f7b..fae2df973af7 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java
@@ -99,16 +99,17 @@ public class ITestS3ADeleteCost extends AbstractS3ACostTest {
// still be there
Path simpleFile = file(new Path(dir, "simple.txt"));
- boolean rawAndKeeping = !isDeleting();
- boolean rawAndDeleting = isDeleting();
+ boolean keeping = !isDeleting();
+ boolean deleting = isDeleting();
+ boolean bulkDelete = isBulkDelete();
verifyMetrics(() -> {
fs.delete(simpleFile, false);
return "after fs.delete(simpleFile) " + getMetricSummary();
},
- probe(rawAndKeeping, OBJECT_METADATA_REQUESTS,
+ probe(keeping, OBJECT_METADATA_REQUESTS,
FILESTATUS_FILE_PROBE_H),
// if deleting markers, look for the parent too
- probe(rawAndDeleting, OBJECT_METADATA_REQUESTS,
+ probe(deleting, OBJECT_METADATA_REQUESTS,
FILESTATUS_FILE_PROBE_H + FILESTATUS_DIR_PROBE_H),
with(OBJECT_LIST_REQUEST,
FILESTATUS_FILE_PROBE_L + FILESTATUS_DIR_PROBE_L),
@@ -116,7 +117,9 @@ public class ITestS3ADeleteCost extends AbstractS3ACostTest {
with(FILES_DELETED, 1),
// a single DELETE call is made to delete the object
- with(OBJECT_DELETE_REQUEST, DELETE_OBJECT_REQUEST),
+ probe(bulkDelete, OBJECT_DELETE_REQUEST, DELETE_OBJECT_REQUEST),
+ probe(!bulkDelete, OBJECT_DELETE_REQUEST,
+ DELETE_OBJECT_REQUEST + DELETE_MARKER_REQUEST),
// keeping: create no parent dirs or delete parents
withWhenKeeping(DIRECTORIES_CREATED, 0),
@@ -127,7 +130,8 @@ public class ITestS3ADeleteCost extends AbstractS3ACostTest {
// a bulk delete for all parents is issued.
// the number of objects in it depends on the depth of the tree;
// don't worry about that
- withWhenDeleting(OBJECT_BULK_DELETE_REQUEST, DELETE_MARKER_REQUEST)
+ probe(deleting && bulkDelete, OBJECT_BULK_DELETE_REQUEST,
+ DELETE_MARKER_REQUEST)
);
// there is an empty dir for a parent
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java
index 3dc64ea3b45f..6f19ba15c1c9 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java
@@ -19,9 +19,9 @@
package org.apache.hadoop.fs.s3a.scale;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.s3a.AWSUnsupportedFeatureException;
import org.apache.hadoop.fs.s3a.Constants;
import org.apache.hadoop.fs.s3a.S3AEncryptionMethods;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
import java.nio.file.AccessDeniedException;
@@ -54,8 +54,7 @@ public class ITestS3AHugeFilesSSECDiskBlocks
public void setup() throws Exception {
try {
super.setup();
- skipIfEncryptionTestsDisabled(getConfiguration());
- } catch (AccessDeniedException e) {
+ } catch (AccessDeniedException | AWSUnsupportedFeatureException e) {
skip("Bucket does not allow " + S3AEncryptionMethods.SSE_C + " encryption method");
}
}
@@ -67,7 +66,7 @@ public class ITestS3AHugeFilesSSECDiskBlocks
removeBaseAndBucketOverrides(conf, S3_ENCRYPTION_KEY,
S3_ENCRYPTION_ALGORITHM, SERVER_SIDE_ENCRYPTION_ALGORITHM,
SERVER_SIDE_ENCRYPTION_KEY);
- S3ATestUtils.disableFilesystemCaching(conf);
+ skipIfEncryptionTestsDisabled(conf);
conf.set(Constants.S3_ENCRYPTION_ALGORITHM,
getSSEAlgorithm().getMethod());
conf.set(Constants.S3_ENCRYPTION_KEY, KEY_1);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesStorageClass.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesStorageClass.java
index ccc71c58644f..58988c9c41bf 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesStorageClass.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesStorageClass.java
@@ -46,15 +46,10 @@ public class ITestS3AHugeFilesStorageClass extends AbstractSTestS3AHugeFiles {
private static final Logger LOG = LoggerFactory.getLogger(ITestS3AHugeFilesStorageClass.class);
- @Override
- public void setup() throws Exception {
- super.setup();
- skipIfStorageClassTestsDisabled(getConfiguration());
- }
-
@Override
protected Configuration createScaleConfiguration() {
Configuration conf = super.createScaleConfiguration();
+ skipIfStorageClassTestsDisabled(conf);
disableFilesystemCaching(conf);
removeBaseAndBucketOverrides(conf, STORAGE_CLASS);
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
index 153347722964..12379fe6c8a6 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
+++ b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
@@ -189,6 +189,18 @@
<value>10s</value>
</property>
+ <property>
+ <name>fs.s3a.attempts.maximum</name>
+ <value>1</value>
+ <description>How many times should the SDK retry commands on (probably) transient errors.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.retry.limit</name>
+ <value>3</value>
+ <description>Fail fairly fast</description>
+ </property>
+
<!--
To run these tests.
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties b/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties
index 306a79a20a20..15c3b571b126 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties
+++ b/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties
@@ -64,10 +64,14 @@ log4j.logger.org.apache.hadoop.fs.s3a.SDKV2Upgrade=WARN
# Log all HTTP requests made; includes S3 interaction. This may
# include sensitive information such as account IDs in HTTP headers.
-#log4j.logger.com.amazonaws.request=DEBUG
+# log4j.logger.software.amazon.awssdk.request=DEBUG
# Turn on low level HTTP protocol debugging
-#log4j.logger.com.amazonaws.thirdparty.apache.http=DEBUG
+#log4j.logger.org.apache.http.wire=DEBUG
+
+# async client
+#log4j.logger.io.netty.handler.logging=DEBUG
+#log4j.logger.io.netty.handler.codec.http2.Http2FrameLogger=DEBUG
log4j.logger.org.apache.hadoop.mapreduce.lib.output=DEBUG
log4j.logger.org.apache.hadoop.fs.s3a.S3AStorageStatistics=INFO
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 07/17: HADOOP-18908. Improve S3A region handling. (#6187)
Posted by ah...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ahmar pushed a commit to branch feature-HADOOP-18073-sdk-v2-upgrade-3.6-aws
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 1808e92b0a358b5143d92a9be27714cb963bc600
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Tue Oct 17 15:37:36 2023 +0100
HADOOP-18908. Improve S3A region handling. (#6187)
S3A region logic improved for better inference and
to be compatible with previous releases
1. If you are using an AWS S3 AccessPoint, its region is determined
from the ARN itself.
2. If fs.s3a.endpoint.region is set and non-empty, it is used.
3. If fs.s3a.endpoint is an s3.*.amazonaws.com url,
the region is determined by by parsing the URL
Note: vpce endpoints are not handled by this.
4. If fs.s3a.endpoint.region==null, and none could be determined
from the endpoint, use us-east-2 as default.
5. If fs.s3a.endpoint.region=="" then it is handed off to
The default AWS SDK resolution process.
Consult the AWS SDK documentation for the details on its resolution
process, knowing that it is complicated and may use environment variables,
entries in ~/.aws/config, IAM instance information within
EC2 deployments and possibly even JSON resources on the classpath.
Put differently: it is somewhat brittle across deployments.
Contributed by Ahmar Suhail
---
.../hadoop/fs/statistics/StoreStatisticNames.java | 4 -
.../java/org/apache/hadoop/fs/s3a/Constants.java | 6 +
.../hadoop/fs/s3a/DefaultS3ClientFactory.java | 115 ++++++++++++++--
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 85 +-----------
.../org/apache/hadoop/fs/s3a/S3ClientFactory.java | 33 +++--
.../java/org/apache/hadoop/fs/s3a/Statistic.java | 5 -
.../apache/hadoop/fs/s3a/s3guard/S3GuardTool.java | 7 +
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 17 +++
.../hadoop/fs/s3a/ITestS3AEndpointRegion.java | 153 ++++++++++++++-------
.../ITestSessionDelegationInFilesystem.java | 5 +-
.../hadoop-aws/src/test/resources/core-site.xml | 18 ++-
11 files changed, 269 insertions(+), 179 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java
index 3a8927aba493..c04c1bb47fce 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java
@@ -407,10 +407,6 @@ public final class StoreStatisticNames {
public static final String MULTIPART_UPLOAD_LIST
= "multipart_upload_list";
- /** Probe for store region: {@value}. */
- public static final String STORE_REGION_PROBE
- = "store_region_probe";
-
private StoreStatisticNames() {
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index aa836ffb6fd1..67d21ddfaa71 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -1179,6 +1179,12 @@ public final class Constants {
*/
public static final String AWS_S3_CENTRAL_REGION = "us-east-1";
+ /**
+ * The default S3 region when using cross region client.
+ * Value {@value}.
+ */
+ public static final String AWS_S3_DEFAULT_REGION = "us-east-2";
+
/**
* Require that all S3 access is made through Access Points.
*/
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
index c85263f1903a..6aaa0e2d0660 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.s3a.impl.AWSClientConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.awscore.util.AwsHostNameUtils;
import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption;
import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
@@ -48,6 +49,9 @@ import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.s3a.statistics.impl.AwsStatisticsCollector;
import org.apache.hadoop.fs.store.LogExactlyOnce;
+import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION;
+import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_DEFAULT_REGION;
+import static org.apache.hadoop.fs.s3a.Constants.CENTRAL_ENDPOINT;
import static org.apache.hadoop.fs.s3a.impl.AWSHeaders.REQUESTER_PAYS_HEADER;
import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_SECURE_CONNECTIONS;
import static org.apache.hadoop.fs.s3a.Constants.SECURE_CONNECTIONS;
@@ -66,12 +70,27 @@ public class DefaultS3ClientFactory extends Configured
private static final String REQUESTER_PAYS_HEADER_VALUE = "requester";
+ private static final String S3_SERVICE_NAME = "s3";
+
/**
* Subclasses refer to this.
*/
protected static final Logger LOG =
LoggerFactory.getLogger(DefaultS3ClientFactory.class);
+ /**
+ * A one-off warning of default region chains in use.
+ */
+ private static final LogExactlyOnce WARN_OF_DEFAULT_REGION_CHAIN =
+ new LogExactlyOnce(LOG);
+
+ /**
+ * Warning message printed when the SDK Region chain is in use.
+ */
+ private static final String SDK_REGION_CHAIN_IN_USE =
+ "S3A filesystem client is using"
+ + " the SDK region resolution chain.";
+
/** Exactly once log to inform about ignoring the AWS-SDK Warnings for CSE. */
private static final LogExactlyOnce IGNORE_CSE_WARN = new LogExactlyOnce(LOG);
@@ -138,15 +157,7 @@ public class DefaultS3ClientFactory extends Configured
BuilderT builder, S3ClientCreationParameters parameters, Configuration conf, String bucket)
throws IOException {
- Region region = parameters.getRegion();
- LOG.debug("Using region {}", region);
-
- URI endpoint = getS3Endpoint(parameters.getEndpoint(), conf);
-
- if (endpoint != null) {
- builder.endpointOverride(endpoint);
- LOG.debug("Using endpoint {}", endpoint);
- }
+ configureEndpointAndRegion(builder, parameters, conf);
S3Configuration serviceConfiguration = S3Configuration.builder()
.pathStyleAccessEnabled(parameters.isPathStyleAccess())
@@ -155,7 +166,6 @@ public class DefaultS3ClientFactory extends Configured
return builder
.overrideConfiguration(createClientOverrideConfiguration(parameters, conf))
.credentialsProvider(parameters.getCredentialSet())
- .region(region)
.serviceConfiguration(serviceConfiguration);
}
@@ -201,6 +211,72 @@ public class DefaultS3ClientFactory extends Configured
return clientOverrideConfigBuilder.build();
}
+ /**
+ * This method configures the endpoint and region for a S3 client.
+ * The order of configuration is:
+ *
+ * <ol>
+ * <li>If region is configured via fs.s3a.endpoint.region, use it.</li>
+ * <li>If endpoint is configured via via fs.s3a.endpoint, set it.
+ * If no region is configured, try to parse region from endpoint. </li>
+ * <li> If no region is configured, and it could not be parsed from the endpoint,
+ * set the default region as US_EAST_2 and enable cross region access. </li>
+ * <li> If configured region is empty, fallback to SDK resolution chain. </li>
+ * </ol>
+ *
+ * @param builder S3 client builder.
+ * @param parameters parameter object
+ * @param conf conf configuration object
+ * @param <BuilderT> S3 client builder type
+ * @param <ClientT> S3 client type
+ */
+ private <BuilderT extends S3BaseClientBuilder<BuilderT, ClientT>, ClientT> void configureEndpointAndRegion(
+ BuilderT builder, S3ClientCreationParameters parameters, Configuration conf) {
+ URI endpoint = getS3Endpoint(parameters.getEndpoint(), conf);
+
+ String configuredRegion = parameters.getRegion();
+ Region region = null;
+ String origin = "";
+
+ // If the region was configured, set it.
+ if (configuredRegion != null && !configuredRegion.isEmpty()) {
+ origin = AWS_REGION;
+ region = Region.of(configuredRegion);
+ }
+
+ if (endpoint != null) {
+ builder.endpointOverride(endpoint);
+ // No region was configured, try to determine it from the endpoint.
+ if (region == null) {
+ region = getS3RegionFromEndpoint(parameters.getEndpoint());
+ if (region != null) {
+ origin = "endpoint";
+ }
+ }
+ LOG.debug("Setting endpoint to {}", endpoint);
+ }
+
+ if (region != null) {
+ builder.region(region);
+ } else if (configuredRegion == null) {
+ // no region is configured, and none could be determined from the endpoint.
+ // Use US_EAST_2 as default.
+ region = Region.of(AWS_S3_DEFAULT_REGION);
+ builder.crossRegionAccessEnabled(true);
+ builder.region(region);
+ origin = "cross region access fallback";
+ } else if (configuredRegion.isEmpty()) {
+ // region configuration was set to empty string.
+ // allow this if people really want it; it is OK to rely on this
+ // when deployed in EC2.
+ WARN_OF_DEFAULT_REGION_CHAIN.warn(SDK_REGION_CHAIN_IN_USE);
+ LOG.debug(SDK_REGION_CHAIN_IN_USE);
+ origin = "SDK region chain";
+ }
+
+ LOG.debug("Setting region to {} from {}", region, origin);
+ }
+
/**
* Given a endpoint string, create the endpoint URI.
*
@@ -229,4 +305,23 @@ public class DefaultS3ClientFactory extends Configured
throw new IllegalArgumentException(e);
}
}
+
+ /**
+ * Parses the endpoint to get the region.
+ * If endpoint is the central one, use US_EAST_1.
+ *
+ * @param endpoint the configure endpoint.
+ * @return the S3 region, null if unable to resolve from endpoint.
+ */
+ private static Region getS3RegionFromEndpoint(String endpoint) {
+
+ if(!endpoint.endsWith(CENTRAL_ENDPOINT)) {
+ LOG.debug("Endpoint {} is not the default; parsing", endpoint);
+ return AwsHostNameUtils.parseSigningRegion(endpoint, S3_SERVICE_NAME).orElse(null);
+ }
+
+ // endpoint is for US_EAST_1;
+ return Region.US_EAST_1;
+ }
+
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index f6a9df4cdb27..ff2a35a6315a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -34,7 +34,6 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.EnumSet;
-import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
@@ -54,7 +53,6 @@ import javax.annotation.Nullable;
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.exception.SdkException;
-import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
@@ -83,7 +81,6 @@ import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
import software.amazon.awssdk.services.s3.model.S3Error;
-import software.amazon.awssdk.services.s3.model.S3Exception;
import software.amazon.awssdk.services.s3.model.S3Object;
import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest;
import software.amazon.awssdk.services.s3.model.SelectObjectContentResponseHandler;
@@ -98,7 +95,6 @@ import software.amazon.awssdk.transfer.s3.model.CopyRequest;
import software.amazon.awssdk.transfer.s3.model.FileUpload;
import software.amazon.awssdk.transfer.s3.model.UploadFileRequest;
-import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.impl.prefetch.ExecutorServiceFuturePool;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -246,7 +242,6 @@ import static org.apache.hadoop.fs.s3a.impl.InternalConstants.ARN_BUCKET_OPTION;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DEFAULT_UPLOAD_PART_COUNT_LIMIT;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DELETE_CONSIDERED_IDEMPOTENT;
-import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_301_MOVED_PERMANENTLY;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_403_FORBIDDEN;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_404_NOT_FOUND;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.UPLOAD_PART_COUNT_LIMIT;
@@ -332,8 +327,6 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
private int executorCapacity;
private long multiPartThreshold;
public static final Logger LOG = LoggerFactory.getLogger(S3AFileSystem.class);
- /** Exactly once log to warn about setting the region in config to avoid probe. */
- private static final LogExactlyOnce SET_REGION_WARNING = new LogExactlyOnce(LOG);
/** Log to warn of storage class configuration problems. */
private static final LogExactlyOnce STORAGE_CLASS_WARNING = new LogExactlyOnce(LOG);
@@ -461,8 +454,6 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
*/
private String scheme = FS_S3A;
- private final static Map<String, Region> BUCKET_REGIONS = new HashMap<>();
-
/** Add any deprecated keys. */
@SuppressWarnings("deprecation")
private static void addDeprecatedKeys() {
@@ -872,9 +863,6 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
STORE_EXISTS_PROBE, bucket, null, () ->
invoker.retry("doesBucketExist", bucket, true, () -> {
try {
- if (BUCKET_REGIONS.containsKey(bucket)) {
- return true;
- }
s3Client.headBucket(HeadBucketRequest.builder().bucket(bucket).build());
return true;
} catch (AwsServiceException ex) {
@@ -984,8 +972,6 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
? conf.getTrimmed(AWS_REGION)
: accessPoint.getRegion();
- Region region = getS3Region(configuredRegion);
-
S3ClientFactory.S3ClientCreationParameters parameters =
new S3ClientFactory.S3ClientCreationParameters()
.withCredentialSet(credentials)
@@ -1000,7 +986,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
.withMultipartCopyEnabled(isMultipartCopyEnabled)
.withMultipartThreshold(multiPartThreshold)
.withTransferManagerExecutor(unboundedThreadPool)
- .withRegion(region);
+ .withRegion(configuredRegion);
S3ClientFactory clientFactory = ReflectionUtils.newInstance(s3ClientFactoryClass, conf);
s3Client = clientFactory.createS3Client(getUri(), parameters);
@@ -1021,75 +1007,6 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
s3AsyncClient = clientFactory.createS3AsyncClient(getUri(), parameters);
}
- /**
- * Get the bucket region.
- *
- * @param region AWS S3 Region set in the config. This property may not be set, in which case
- * ask S3 for the region.
- * @return region of the bucket.
- */
- private Region getS3Region(String region) throws IOException {
-
- if (!StringUtils.isBlank(region)) {
- return Region.of(region);
- }
-
- Region cachedRegion = BUCKET_REGIONS.get(bucket);
-
- if (cachedRegion != null) {
- LOG.debug("Got region {} for bucket {} from cache", cachedRegion, bucket);
- return cachedRegion;
- }
-
- Region s3Region = trackDurationAndSpan(STORE_REGION_PROBE, bucket, null,
- () -> invoker.retry("getS3Region", bucket, true, () -> {
- try {
-
- SET_REGION_WARNING.warn(
- "Getting region for bucket {} from S3, this will slow down FS initialisation. "
- + "To avoid this, set the region using property {}", bucket,
- FS_S3A_BUCKET_PREFIX + bucket + ".endpoint.region");
-
- // build a s3 client with region eu-west-1 that can be used to get the region of the
- // bucket. Using eu-west-1, as headBucket() doesn't work with us-east-1. This is because
- // us-east-1 uses the endpoint s3.amazonaws.com, which resolves bucket.s3.amazonaws.com
- // to the actual region the bucket is in. As the request is signed with us-east-1 and
- // not the bucket's region, it fails.
- S3Client getRegionS3Client =
- S3Client.builder().region(Region.EU_WEST_1).credentialsProvider(credentials)
- .build();
-
- HeadBucketResponse headBucketResponse =
- getRegionS3Client.headBucket(HeadBucketRequest.builder().bucket(bucket).build());
-
- Region bucketRegion = Region.of(
- headBucketResponse.sdkHttpResponse().headers().get(BUCKET_REGION_HEADER).get(0));
- BUCKET_REGIONS.put(bucket, bucketRegion);
-
- return bucketRegion;
- } catch (S3Exception exception) {
- if (exception.statusCode() == SC_301_MOVED_PERMANENTLY) {
- Region bucketRegion = Region.of(
- exception.awsErrorDetails().sdkHttpResponse().headers().get(BUCKET_REGION_HEADER)
- .get(0));
- BUCKET_REGIONS.put(bucket, bucketRegion);
-
- return bucketRegion;
- }
-
- if (exception.statusCode() == SC_404_NOT_FOUND) {
- throw new UnknownStoreException("s3a://" + bucket + "/",
- " Bucket does not exist: " + exception,
- exception);
- }
-
- throw exception;
- }
- }));
-
- return s3Region;
- }
-
/**
* Initialize and launch the audit manager and service.
* As this takes the FS IOStatistics store, it must be invoked
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
index e2e792ebfb66..451eab8e9ad5 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
@@ -28,7 +28,6 @@ import java.util.concurrent.Executor;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
-import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.transfer.s3.S3TransferManager;
@@ -169,7 +168,7 @@ public interface S3ClientFactory {
/**
* Region of the S3 bucket.
*/
- private Region region;
+ private String region;
/**
@@ -386,42 +385,42 @@ public interface S3ClientFactory {
}
/**
- * Set region.
+ * Set the multipart flag..
*
* @param value new value
* @return the builder
*/
- public S3ClientCreationParameters withRegion(
- final Region value) {
- region = value;
+ public S3ClientCreationParameters withMultipartCopyEnabled(final boolean value) {
+ this.multipartCopy = value;
return this;
}
/**
- * Get the region.
- * @return invoker
+ * Get the multipart flag.
+ * @return multipart flag
*/
- public Region getRegion() {
- return region;
+ public boolean isMultipartCopy() {
+ return multipartCopy;
}
/**
- * Set the multipart flag..
+ * Set region.
*
* @param value new value
* @return the builder
*/
- public S3ClientCreationParameters withMultipartCopyEnabled(final boolean value) {
- this.multipartCopy = value;
+ public S3ClientCreationParameters withRegion(
+ final String value) {
+ region = value;
return this;
}
/**
- * Get the multipart flag.
- * @return multipart flag
+ * Get the region.
+ * @return invoker
*/
- public boolean isMultipartCopy() {
- return multipartCopy;
+ public String getRegion() {
+ return region;
}
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
index f54113af6c5c..f4e28aa62783 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
@@ -548,11 +548,6 @@ public enum Statistic {
StoreStatisticNames.STORE_IO_THROTTLE_RATE,
"Rate of S3 request throttling",
TYPE_QUANTILE),
- STORE_REGION_PROBE(
- StoreStatisticNames.STORE_REGION_PROBE,
- "Store Region Probe",
- TYPE_DURATION
- ),
/*
* Delegation Token Operations.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
index 330b92186dde..22fc630dad1f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
@@ -406,6 +406,13 @@ public abstract class S3GuardTool extends Configured implements Tool,
// Note and continue.
LOG.debug("failed to get bucket location", e);
println(out, LOCATION_UNKNOWN);
+
+ // it may be the bucket is not found; we can't differentiate
+ // that and handle third party store issues where the API may
+ // not work.
+ // Fallback to looking for bucket root attributes.
+ println(out, "Probing for bucket existence");
+ fs.listXAttrs(new Path("/"));
}
// print any auth paths for directory marker info
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
index 60037e58cd61..d9c847d4b2b0 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.util.VersionInfo;
import org.apache.http.HttpStatus;
import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
import static org.apache.hadoop.fs.s3a.Constants.*;
import static org.apache.hadoop.fs.s3a.S3ATestConstants.EU_WEST_1;
import static org.apache.hadoop.fs.s3a.S3AUtils.*;
@@ -366,6 +367,7 @@ public class ITestS3AConfiguration {
throws Exception {
conf = new Configuration();
+ skipIfCrossRegionClient(conf);
conf.set(Constants.PATH_STYLE_ACCESS, Boolean.toString(true));
assertTrue(conf.getBoolean(Constants.PATH_STYLE_ACCESS, false));
@@ -404,6 +406,7 @@ public class ITestS3AConfiguration {
@Test
public void testDefaultUserAgent() throws Exception {
conf = new Configuration();
+ skipIfCrossRegionClient(conf);
fs = S3ATestUtils.createTestFileSystem(conf);
assertNotNull(fs);
S3Client s3 = getS3Client("User Agent");
@@ -417,6 +420,7 @@ public class ITestS3AConfiguration {
@Test
public void testCustomUserAgent() throws Exception {
conf = new Configuration();
+ skipIfCrossRegionClient(conf);
conf.set(Constants.USER_AGENT_PREFIX, "MyApp");
fs = S3ATestUtils.createTestFileSystem(conf);
assertNotNull(fs);
@@ -431,6 +435,7 @@ public class ITestS3AConfiguration {
@Test
public void testRequestTimeout() throws Exception {
conf = new Configuration();
+ skipIfCrossRegionClient(conf);
conf.set(REQUEST_TIMEOUT, "120");
fs = S3ATestUtils.createTestFileSystem(conf);
S3Client s3 = getS3Client("Request timeout (ms)");
@@ -610,4 +615,16 @@ public class ITestS3AConfiguration {
return stsSignerCalled;
}
}
+
+ /**
+ * Skip a test if client created is cross region.
+ * @param configuration configuration to probe
+ */
+ private static void skipIfCrossRegionClient(
+ Configuration configuration) {
+ if (configuration.get(ENDPOINT, null) == null
+ && configuration.get(AWS_REGION, null) == null) {
+ skip("Skipping test as cross region client is in use ");
+ }
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java
index eb010674beed..5d10590dfe30 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java
@@ -27,6 +27,7 @@ import java.util.ArrayList;
import java.util.List;
import org.assertj.core.api.Assertions;
+import org.junit.Ignore;
import org.junit.Test;
import software.amazon.awssdk.awscore.AwsExecutionAttribute;
import software.amazon.awssdk.awscore.exception.AwsServiceException;
@@ -42,8 +43,9 @@ import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext;
import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION;
import static org.apache.hadoop.fs.s3a.Constants.PATH_STYLE_ACCESS;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
-import static org.apache.hadoop.fs.s3a.Statistic.STORE_REGION_PROBE;
import static org.apache.hadoop.io.IOUtils.closeStream;
+import static org.apache.hadoop.fs.s3a.Constants.CENTRAL_ENDPOINT;
+
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
/**
@@ -54,9 +56,17 @@ public class ITestS3AEndpointRegion extends AbstractS3ATestBase {
private static final String AWS_ENDPOINT_TEST = "test-endpoint";
- private static final String USW_2_BUCKET = "landsat-pds";
+ private static final String US_EAST_1 = "us-east-1";
+
+ private static final String US_EAST_2 = "us-east-2";
+
+ private static final String US_WEST_2 = "us-west-2";
+
+ private static final String EU_WEST_2 = "eu-west-2";
- public static final String USW_2_STORE = "s3a://" + USW_2_BUCKET;
+ private static final String CN_NORTHWEST_1 = "cn-northwest-1";
+
+ private static final String US_GOV_EAST_1 = "us-gov-east-1";
/**
* If anyone were ever to create a bucket with this UUID pair it would break the tests.
@@ -64,6 +74,14 @@ public class ITestS3AEndpointRegion extends AbstractS3ATestBase {
public static final String UNKNOWN_BUCKET = "23FA76D4-5F17-48B8-9D7D-9050269D0E40"
+ "-8281BAF2-DBCF-47AA-8A27-F2FA3589656A";
+ private static final String EU_WEST_2_ENDPOINT = "s3.eu-west-2.amazonaws.com";
+
+ private static final String CN_ENDPOINT = "s3.cn-northwest-1.amazonaws.com.cn";
+
+ private static final String GOV_ENDPOINT = "s3-fips.us-gov-east-1.amazonaws.com";
+
+ private static final String VPC_ENDPOINT = "vpce-1a2b3c4d-5e6f.s3.us-west-2.vpce.amazonaws.com";
+
/**
* New FS instance which will be closed in teardown.
*/
@@ -75,11 +93,6 @@ public class ITestS3AEndpointRegion extends AbstractS3ATestBase {
super.teardown();
}
- /**
- * Test to verify that not setting the region config, will lead to the client factory making
- * a HEAD bucket call to configure the correct region. If an incorrect region is set, the HEAD
- * bucket call in this test will raise an exception.
- */
@Test
public void testWithoutRegionConfig() throws IOException {
describe("Verify that region lookup takes place");
@@ -96,7 +109,6 @@ public class ITestS3AEndpointRegion extends AbstractS3ATestBase {
} catch (UnknownHostException | UnknownStoreException | AccessDeniedException allowed) {
// these are all valid failure modes from different test environments.
}
- assertRegionProbeCount(1);
}
@Test
@@ -115,82 +127,128 @@ public class ITestS3AEndpointRegion extends AbstractS3ATestBase {
} catch (UnknownHostException | UnknownStoreException expected) {
// this is good.
}
- assertRegionProbeCount(1);
}
+ @Test
+ public void testEndpointOverride() throws Throwable {
+ describe("Create a client with a configured endpoint");
+ Configuration conf = getConfiguration();
+
+ S3Client client = createS3Client(conf, AWS_ENDPOINT_TEST, null, US_EAST_2);
+
+ intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket(
+ HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build()));
+ }
@Test
- public void testWithRegionConfig() throws IOException, URISyntaxException {
- describe("Verify that region lookup is skipped if the region property is set");
+ public void testCentralEndpoint() throws Throwable {
+ describe("Create a client with the central endpoint");
Configuration conf = getConfiguration();
- removeBaseAndBucketOverrides(conf, AWS_REGION, PATH_STYLE_ACCESS);
- conf.set(AWS_REGION, "us-east-2");
+ S3Client client = createS3Client(conf, CENTRAL_ENDPOINT, null, US_EAST_1);
- newFS = new S3AFileSystem();
- newFS.initialize(new URI(USW_2_STORE), conf);
- assertRegionProbeCount(0);
+ intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket(
+ HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build()));
}
@Test
- public void testRegionCache() throws IOException, URISyntaxException {
- describe("Verify that region lookup is cached on the second attempt");
+ public void testWithRegionConfig() throws Throwable {
+ describe("Create a client with a configured region");
Configuration conf = getConfiguration();
- removeBaseAndBucketOverrides(USW_2_BUCKET, conf, AWS_REGION, PATH_STYLE_ACCESS);
- newFS = new S3AFileSystem();
+ S3Client client = createS3Client(conf, null, EU_WEST_2, EU_WEST_2);
- newFS.initialize(new URI(USW_2_STORE), conf);
+ intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket(
+ HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build()));
+ }
- assertRegionProbeCount(1);
- closeStream(newFS);
- // create a new instance
- newFS = new S3AFileSystem();
- newFS.initialize(new URI(USW_2_STORE), conf);
+ public void testEUWest2Endpoint() throws Throwable {
+ describe("Create a client with the eu west 2 endpoint");
+ Configuration conf = getConfiguration();
+
+ S3Client client = createS3Client(conf, EU_WEST_2_ENDPOINT, null, EU_WEST_2);
- // value should already be cached.
- assertRegionProbeCount(0);
+ intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket(
+ HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build()));
}
- private void assertRegionProbeCount(final int expected) {
- Assertions.assertThat(newFS.getInstrumentation().getCounterValue(STORE_REGION_PROBE))
- .describedAs("Incorrect number of calls made to get bucket region").isEqualTo(expected);
+ @Test
+ public void testWithRegionAndEndpointConfig() throws Throwable {
+ describe("Test that when both region and endpoint are configured, region takes precedence");
+ Configuration conf = getConfiguration();
+
+ S3Client client = createS3Client(conf, EU_WEST_2_ENDPOINT, US_WEST_2, US_WEST_2);
+
+ intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket(
+ HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build()));
}
@Test
- public void testEndpointOverride() throws Throwable {
- describe("Create a client with no region and the default endpoint");
+ public void testWithChinaEndpoint() throws Throwable {
+ describe("Test with a china endpoint");
Configuration conf = getConfiguration();
- S3Client client = createS3Client(conf, AWS_ENDPOINT_TEST);
+ S3Client client = createS3Client(conf, CN_ENDPOINT, null, CN_NORTHWEST_1);
intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket(
HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build()));
}
+ @Test
+ public void testWithGovCloudEndpoint() throws Throwable {
+ describe("Test with a gov cloud endpoint");
+ Configuration conf = getConfiguration();
+
+ S3Client client = createS3Client(conf, GOV_ENDPOINT, null, US_GOV_EAST_1);
+
+ intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket(
+ HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build()));
+ }
+
+ @Test
+ @Ignore("Pending HADOOP-18938. S3A region logic to handle vpce and non standard endpoints")
+ public void testWithVPCE() throws Throwable {
+ describe("Test with vpc endpoint");
+ Configuration conf = getConfiguration();
+
+ S3Client client = createS3Client(conf, VPC_ENDPOINT, null, US_WEST_2);
+
+ intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket(
+ HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build()));
+ }
class RegionInterceptor implements ExecutionInterceptor {
- private boolean endpointOverridden;
+ private String endpoint;
+ private String region;
- RegionInterceptor(boolean endpointOverridden) {
- this.endpointOverridden = endpointOverridden;
+ RegionInterceptor(String endpoint, String region) {
+ this.endpoint = endpoint;
+ this.region = region;
}
@Override
public void beforeExecution(Context.BeforeExecution context,
ExecutionAttributes executionAttributes) {
- if (endpointOverridden) {
+ if (endpoint != null) {
Assertions.assertThat(
executionAttributes.getAttribute(AwsExecutionAttribute.ENDPOINT_OVERRIDDEN))
.describedAs("Endpoint not overridden").isTrue();
Assertions.assertThat(
executionAttributes.getAttribute(AwsExecutionAttribute.CLIENT_ENDPOINT).toString())
- .describedAs("There is an endpoint mismatch").isEqualTo("https://" + AWS_ENDPOINT_TEST);
+ .describedAs("There is an endpoint mismatch").isEqualTo("https://" + endpoint);
+ } else {
+ Assertions.assertThat(
+ executionAttributes.getAttribute(AwsExecutionAttribute.ENDPOINT_OVERRIDDEN))
+ .describedAs("Endpoint is overridden").isEqualTo(null);
}
+ Assertions.assertThat(
+ executionAttributes.getAttribute(AwsExecutionAttribute.AWS_REGION).toString())
+ .describedAs("Incorrect region set").isEqualTo(region);
+
// We don't actually want to make a request, so exit early.
throw AwsServiceException.builder().message("Exception thrown by interceptor").build();
}
@@ -202,23 +260,18 @@ public class ITestS3AEndpointRegion extends AbstractS3ATestBase {
* value.
* @param conf configuration to use.
* @param endpoint endpoint.
+ * @param expectedRegion the region that should be set in the client.
* @return the client.
* @throws URISyntaxException parse problems.
* @throws IOException IO problems
*/
@SuppressWarnings("deprecation")
private S3Client createS3Client(Configuration conf,
- String endpoint)
+ String endpoint, String configuredRegion, String expectedRegion)
throws IOException {
- boolean endpointOverridden = false;
-
- if (endpoint != null && !endpoint.isEmpty()) {
- endpointOverridden = true;
- }
-
List<ExecutionInterceptor> interceptors = new ArrayList<>();
- interceptors.add(new RegionInterceptor(endpointOverridden));
+ interceptors.add(new RegionInterceptor(endpoint, expectedRegion));
DefaultS3ClientFactory factory
= new DefaultS3ClientFactory();
@@ -229,7 +282,9 @@ public class ITestS3AEndpointRegion extends AbstractS3ATestBase {
.withEndpoint(endpoint)
.withMetrics(new EmptyS3AStatisticsContext()
.newStatisticsFromAwsSdk())
- .withExecutionInterceptors(interceptors);
+ .withExecutionInterceptors(interceptors)
+ .withRegion(configuredRegion);
+
S3Client client = factory.createS3Client(
getFileSystem().getUri(),
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFilesystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFilesystem.java
index ebad90336f7d..e85e33c844d5 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFilesystem.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFilesystem.java
@@ -26,7 +26,6 @@ import java.io.UncheckedIOException;
import java.net.URI;
import java.nio.file.AccessDeniedException;
-import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.HeadBucketResponse;
import org.junit.AfterClass;
@@ -596,8 +595,8 @@ public class ITestSessionDelegationInFilesystem extends AbstractDelegationIT {
.withPathUri(new URI("s3a://localhost/"))
.withMetrics(new EmptyS3AStatisticsContext()
.newStatisticsFromAwsSdk())
- .withUserAgentSuffix("ITestSessionDelegationInFilesystem")
- .withRegion(Region.US_WEST_2);
+ .withUserAgentSuffix("ITestSessionDelegationInFilesystem");
+
S3Client s3 = factory.createS3Client(landsat, parameters);
return Invoker.once("HEAD", host,
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
index 12379fe6c8a6..e6c3eeed3d7e 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
+++ b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
@@ -31,11 +31,10 @@
</property>
<!-- Per-bucket configurations: landsat-pds -->
-
<property>
- <name>fs.s3a.bucket.landsat-pds.endpoint</name>
- <value>${central.endpoint}</value>
- <description>The endpoint for s3a://landsat-pds URLs</description>
+ <name>fs.s3a.bucket.landsat-pds.endpoint.region</name>
+ <value>us-west-2</value>
+ <description>The region for s3a://landsat-pds</description>
</property>
<property>
@@ -58,10 +57,9 @@
</property>
<!-- Per-bucket configurations: usgs-landsat -->
-
<property>
- <name>fs.s3a.bucket.usgs-landsat.endpoint</name>
- <value>${central.endpoint}</value>
+ <name>fs.s3a.bucket.usgs-landsat.endpoint.region</name>
+ <value>us-west-2</value>
</property>
<property>
@@ -82,6 +80,12 @@
<value>false</value>
</property>
+ <!-- Per-bucket configurations: osm-pds -->
+ <property>
+ <name>fs.s3a.bucket.osm-pds.endpoint.region</name>
+ <value>us-east-1</value>
+ <description>The region for s3a://osm-pds</description>
+ </property>
<!--
This is the default endpoint, which can be used to interact
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org