You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ah...@apache.org on 2023/12/05 09:15:56 UTC
(hadoop) 02/17: HADOOP-18073. S3A: Upgrade AWS SDK to V2 (#5995)
This is an automated email from the ASF dual-hosted git repository.
ahmar pushed a commit to branch feature-HADOOP-18073-sdk-v2-upgrade-3.6-aws
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 1fbe185f9ba86977495d69e7c80a931628cb7ba0
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Mon Sep 11 14:30:25 2023 +0100
HADOOP-18073. S3A: Upgrade AWS SDK to V2 (#5995)
This patch migrates the S3A connector to use the V2 AWS SDK.
This is a significant change at the source code level.
Any applications using the internal extension/override points in
the filesystem connector are likely to break.
This includes but is not limited to:
- Code invoking methods on the S3AFileSystem class
which used classes from the V1 SDK.
- The ability to define the factory for the `AmazonS3` client, and
to retrieve it from the S3AFileSystem. There is a new factory
API and a special interface S3AInternals to access a limited
set of internal classes and operations.
- Delegation token and auditing extensions.
- Classes trying to integrate with the AWS SDK.
All standard V1 credential providers listed in the option
fs.s3a.aws.credentials.provider will be automatically remapped to their
V2 equivalent.
Other V1 Credential Providers are supported, but only if the V1 SDK is
added back to the classpath.
The SDK Signing plugin has changed; all v1 signers are incompatible.
There is no support for the S3 "v2" signing algorithm.
Finally, the aws-sdk-bundle JAR has been replaced by the shaded V2
equivalent, "bundle.jar", which is now exported by the hadoop-aws module.
Consult the document aws_sdk_upgrade for the full details.
Contributed by Ahmar Suhail + some bits by Steve Loughran
---
LICENSE-binary | 1 +
.../hadoop/fs/statistics/StoreStatisticNames.java | 4 +
.../src/main/resources/core-default.xml | 52 +-
hadoop-project/pom.xml | 22 +-
.../hadoop-aws/dev-support/findbugs-exclude.xml | 5 +
hadoop-tools/hadoop-aws/pom.xml | 16 +-
.../hadoop/fs/s3a/AWSBadRequestException.java | 4 +-
.../apache/hadoop/fs/s3a/AWSClientIOException.java | 18 +-
.../hadoop/fs/s3a/AWSCredentialProviderList.java | 110 +-
.../hadoop/fs/s3a/AWSNoResponseException.java | 9 +-
.../apache/hadoop/fs/s3a/AWSRedirectException.java | 4 +-
.../org/apache/hadoop/fs/s3a/AWSS3IOException.java | 25 +-
.../hadoop/fs/s3a/AWSServiceIOException.java | 37 +-
.../fs/s3a/AWSServiceThrottledException.java | 9 +-
.../hadoop/fs/s3a/AWSStatus500Exception.java | 9 +-
.../fs/s3a/AnonymousAWSCredentialsProvider.java | 18 +-
.../java/org/apache/hadoop/fs/s3a/ArnResource.java | 12 +-
.../java/org/apache/hadoop/fs/s3a/Constants.java | 16 +-
.../fs/s3a/CredentialInitializationException.java | 18 +-
.../hadoop/fs/s3a/DefaultS3ClientFactory.java | 430 +++-----
.../hadoop/fs/s3a/FailureInjectionPolicy.java | 2 +-
.../hadoop/fs/s3a/InconsistentAmazonS3Client.java | 345 ------
.../hadoop/fs/s3a/InconsistentS3ClientFactory.java | 54 -
.../java/org/apache/hadoop/fs/s3a/Invoker.java | 21 +-
.../java/org/apache/hadoop/fs/s3a/Listing.java | 53 +-
.../org/apache/hadoop/fs/s3a/MultipartUtils.java | 49 +-
.../fs/s3a/ProgressableProgressListener.java | 46 +-
.../apache/hadoop/fs/s3a/S3ABlockOutputStream.java | 124 ++-
.../org/apache/hadoop/fs/s3a/S3ADataBlocks.java | 8 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 1099 +++++++++++---------
.../org/apache/hadoop/fs/s3a/S3AInputStream.java | 116 +--
.../org/apache/hadoop/fs/s3a/S3AInternals.java | 118 +++
.../org/apache/hadoop/fs/s3a/S3ARetryPolicy.java | 56 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 678 ++++--------
.../org/apache/hadoop/fs/s3a/S3ClientFactory.java | 187 +++-
.../org/apache/hadoop/fs/s3a/S3ListRequest.java | 16 +-
.../org/apache/hadoop/fs/s3a/S3ListResult.java | 61 +-
.../apache/hadoop/fs/s3a/S3ObjectAttributes.java | 27 -
.../fs/s3a/SharedInstanceCredentialProvider.java | 1 -
.../fs/s3a/SimpleAWSCredentialsProvider.java | 31 +-
.../java/org/apache/hadoop/fs/s3a/Statistic.java | 5 +
.../fs/s3a/TemporaryAWSCredentialsProvider.java | 7 +-
.../java/org/apache/hadoop/fs/s3a/Tristate.java | 45 +-
.../java/org/apache/hadoop/fs/s3a/UploadInfo.java | 12 +-
.../apache/hadoop/fs/s3a/WriteOperationHelper.java | 202 ++--
.../org/apache/hadoop/fs/s3a/WriteOperations.java | 100 +-
.../hadoop/fs/s3a/adapter/AwsV1BindingSupport.java | 118 +++
.../V1ToV2AwsCredentialProviderAdapter.java | 165 +++
.../{UploadInfo.java => adapter/package-info.java} | 32 +-
.../apache/hadoop/fs/s3a/api/RequestFactory.java | 224 ++--
.../fs/s3a/audit/AWSAuditEventCallbacks.java | 105 +-
.../hadoop/fs/s3a/audit/AWSRequestAnalyzer.java | 135 +--
.../hadoop/fs/s3a/audit/AuditIntegration.java | 54 +-
.../hadoop/fs/s3a/audit/AuditManagerS3A.java | 21 +-
.../hadoop/fs/s3a/audit/S3AAuditConstants.java | 13 +-
.../fs/s3a/audit/impl/ActiveAuditManagerS3A.java | 419 +++++---
.../hadoop/fs/s3a/audit/impl/LoggingAuditor.java | 125 ++-
.../fs/s3a/audit/impl/NoopAuditManagerS3A.java | 16 +-
.../s3a/audit/impl/S3AInternalAuditConstants.java | 16 +-
.../fs/s3a/auth/AbstractAWSCredentialProvider.java | 13 +-
.../auth/AbstractSessionCredentialsProvider.java | 27 +-
.../fs/s3a/auth/AssumedRoleCredentialProvider.java | 95 +-
.../fs/s3a/auth/CredentialProviderListFactory.java | 303 ++++++
.../s3a/auth/IAMInstanceCredentialsProvider.java | 49 +-
.../fs/s3a/auth/MarshalledCredentialBinding.java | 58 +-
.../fs/s3a/auth/MarshalledCredentialProvider.java | 5 +-
.../hadoop/fs/s3a/auth/NoAuthWithAWSException.java | 2 +-
.../hadoop/fs/s3a/auth/STSClientFactory.java | 141 ++-
.../apache/hadoop/fs/s3a/auth/SignerFactory.java | 114 ++
.../apache/hadoop/fs/s3a/auth/SignerManager.java | 8 +-
.../delegation/EncryptionSecretOperations.java | 38 +-
.../fs/s3a/auth/delegation/RoleTokenBinding.java | 2 +-
.../s3a/auth/delegation/S3ADelegationTokens.java | 5 -
.../s3a/auth/delegation/SessionTokenBinding.java | 61 +-
.../hadoop/fs/s3a/commit/AbstractS3ACommitter.java | 9 +-
.../apache/hadoop/fs/s3a/commit/PutTracker.java | 4 +-
.../fs/s3a/commit/files/SinglePendingCommit.java | 13 +-
.../fs/s3a/commit/impl/CommitOperations.java | 54 +-
.../fs/s3a/commit/magic/MagicCommitTracker.java | 28 +-
.../AWSCannedACL.java} | 28 +-
.../apache/hadoop/fs/s3a/impl/AWSClientConfig.java | 394 +++++++
.../org/apache/hadoop/fs/s3a/impl/AWSHeaders.java | 98 ++
.../hadoop/fs/s3a/impl/BulkDeleteRetryHandler.java | 12 +-
.../hadoop/fs/s3a/impl/ChangeDetectionPolicy.java | 125 ++-
.../apache/hadoop/fs/s3a/impl/ChangeTracker.java | 71 +-
.../s3a/impl/ConfigureShadedAWSSocketFactory.java | 13 +-
.../org/apache/hadoop/fs/s3a/impl/CopyOutcome.java | 80 --
.../apache/hadoop/fs/s3a/impl/DeleteOperation.java | 17 +-
.../hadoop/fs/s3a/impl/ErrorTranslation.java | 14 +-
.../hadoop/fs/s3a/impl/HeaderProcessing.java | 206 ++--
.../fs/s3a/impl/InstantiationIOException.java | 180 ++++
.../hadoop/fs/s3a/impl/InternalConstants.java | 51 +-
...upport.java => MultiObjectDeleteException.java} | 67 +-
.../apache/hadoop/fs/s3a/impl/NetworkBinding.java | 11 +-
.../hadoop/fs/s3a/impl/OperationCallbacks.java | 15 +-
.../{Tristate.java => impl/ProgressListener.java} | 14 +-
.../ProgressListenerEvent.java} | 17 +-
.../apache/hadoop/fs/s3a/impl/RenameOperation.java | 23 +-
.../hadoop/fs/s3a/impl/RequestFactoryImpl.java | 663 ++++++------
.../hadoop/fs/s3a/impl/S3AMultipartUploader.java | 31 +-
.../hadoop/fs/s3a/impl/SDKStreamDrainer.java | 43 +-
.../org/apache/hadoop/fs/s3a/impl/V2Migration.java | 72 +-
.../hadoop/fs/s3a/prefetch/S3ARemoteObject.java | 54 +-
.../fs/s3a/prefetch/S3ARemoteObjectReader.java | 7 +-
.../apache/hadoop/fs/s3a/s3guard/S3GuardTool.java | 8 +-
.../hadoop/fs/s3a/select/BlockingEnumeration.java | 156 +++
.../apache/hadoop/fs/s3a/select/SelectBinding.java | 126 ++-
.../fs/s3a/select/SelectEventStreamPublisher.java | 124 +++
.../hadoop/fs/s3a/select/SelectInputStream.java | 31 +-
.../fs/s3a/select/SelectObjectContentHelper.java | 114 ++
.../statistics/impl/AwsStatisticsCollector.java | 166 ++-
.../org/apache/hadoop/fs/s3a/tools/MarkerTool.java | 17 +-
.../hadoop/fs/s3a/tools/MarkerToolOperations.java | 13 +-
.../fs/s3a/tools/MarkerToolOperationsImpl.java | 11 +-
.../markdown/tools/hadoop-aws/assumed_roles.md | 2 +-
.../src/site/markdown/tools/hadoop-aws/auditing.md | 48 +-
.../markdown/tools/hadoop-aws/aws_sdk_upgrade.md | 340 +++++-
.../tools/hadoop-aws/aws_sdk_v2_changelog.md | 340 ++++++
.../markdown/tools/hadoop-aws/delegation_tokens.md | 2 +-
.../src/site/markdown/tools/hadoop-aws/index.md | 142 +--
.../site/markdown/tools/hadoop-aws/s3_select.md | 43 +-
.../src/site/markdown/tools/hadoop-aws/testing.md | 69 +-
.../tools/hadoop-aws/troubleshooting_s3a.md | 95 +-
.../apache/hadoop/fs/s3a/AbstractS3AMockTest.java | 30 +-
.../apache/hadoop/fs/s3a/AbstractS3ATestBase.java | 8 +
.../apache/hadoop/fs/s3a/EncryptionTestUtils.java | 20 +-
.../fs/s3a/ITestS3AAWSCredentialsProvider.java | 164 +--
.../hadoop/fs/s3a/ITestS3ABucketExistence.java | 41 +-
.../apache/hadoop/fs/s3a/ITestS3ACannedACLs.java | 31 +-
.../fs/s3a/ITestS3AClientSideEncryptionKms.java | 6 +-
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 236 +++--
.../hadoop/fs/s3a/ITestS3AEncryptionSSEC.java | 2 +-
.../fs/s3a/ITestS3AEncryptionSSEKMSDefaultKey.java | 8 +-
.../ITestS3AEncryptionWithDefaultS3Settings.java | 3 +-
.../hadoop/fs/s3a/ITestS3AEndpointRegion.java | 200 ++--
.../hadoop/fs/s3a/ITestS3AFailureHandling.java | 28 +-
.../hadoop/fs/s3a/ITestS3AMiscOperations.java | 35 +-
.../hadoop/fs/s3a/ITestS3AMultipartUtils.java | 5 +-
.../hadoop/fs/s3a/ITestS3ARequesterPays.java | 2 +-
.../apache/hadoop/fs/s3a/ITestS3AStorageClass.java | 1 -
.../fs/s3a/ITestS3ATemporaryCredentials.java | 30 +-
.../apache/hadoop/fs/s3a/MockS3AFileSystem.java | 17 +-
.../apache/hadoop/fs/s3a/MockS3ClientFactory.java | 49 +-
.../apache/hadoop/fs/s3a/MultipartTestUtils.java | 35 +-
.../org/apache/hadoop/fs/s3a/S3ATestConstants.java | 5 +
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 17 +-
.../org/apache/hadoop/fs/s3a/TestArnResource.java | 9 +-
.../java/org/apache/hadoop/fs/s3a/TestInvoker.java | 105 +-
.../fs/s3a/TestS3AAWSCredentialsProvider.java | 266 ++---
.../hadoop/fs/s3a/TestS3ABlockOutputStream.java | 11 +-
.../apache/hadoop/fs/s3a/TestS3ADeleteOnExit.java | 25 +-
.../hadoop/fs/s3a/TestS3AExceptionTranslation.java | 208 +++-
.../apache/hadoop/fs/s3a/TestS3AGetFileStatus.java | 95 +-
.../hadoop/fs/s3a/TestS3AInputStreamRetry.java | 130 ++-
.../org/apache/hadoop/fs/s3a/TestS3AProxy.java | 18 +-
.../org/apache/hadoop/fs/s3a/TestS3AUnbuffer.java | 50 +-
.../hadoop/fs/s3a/TestStreamChangeTracker.java | 107 +-
.../fs/s3a/TestWildflyAndOpenSSLBinding.java | 9 +-
.../fs/s3a/adapter/TestV1CredentialsProvider.java | 222 ++++
.../hadoop/fs/s3a/audit/AbstractAuditingTest.java | 98 +-
.../hadoop/fs/s3a/audit/AuditTestSupport.java | 4 +-
.../hadoop/fs/s3a/audit/ITestAuditManager.java | 18 +-
...ler.java => SimpleAWSExecutionInterceptor.java} | 37 +-
.../hadoop/fs/s3a/audit/TestAuditIntegration.java | 82 +-
.../fs/s3a/audit/TestAuditSpanLifecycle.java | 11 +-
.../fs/s3a/audit/TestHttpReferrerAuditHeader.java | 55 +-
.../hadoop/fs/s3a/audit/TestLoggingAuditor.java | 41 +-
.../apache/hadoop/fs/s3a/auth/ITestAssumeRole.java | 36 +-
.../hadoop/fs/s3a/auth/ITestCustomSigner.java | 47 +-
.../apache/hadoop/fs/s3a/auth/RoleTestUtils.java | 7 +-
.../fs/s3a/auth/TestMarshalledCredentials.java | 10 +-
.../hadoop/fs/s3a/auth/TestSignerManager.java | 41 +-
.../auth/delegation/CountInvocationsProvider.java | 34 +-
.../ITestSessionDelegationInFilesystem.java | 30 +-
.../delegation/ITestSessionDelegationTokens.java | 28 +-
.../delegation/TestS3ADelegationTokenSupport.java | 1 -
.../fs/s3a/commit/AbstractITCommitProtocol.java | 4 +-
.../fs/s3a/commit/staging/StagingTestBase.java | 178 ++--
.../staging/TestDirectoryCommitterScale.java | 7 +-
.../s3a/commit/staging/TestStagingCommitter.java | 32 +-
.../staging/TestStagingPartitionedTaskCommit.java | 9 +-
.../fileContext/ITestS3AFileContextStatistics.java | 3 +-
.../fs/s3a/impl/ITestPartialRenamesDeletes.java | 1 -
.../hadoop/fs/s3a/impl/ITestRenameDeleteRace.java | 5 +-
.../hadoop/fs/s3a/impl/TestHeaderProcessing.java | 32 +-
.../hadoop/fs/s3a/impl/TestNetworkBinding.java | 43 -
.../hadoop/fs/s3a/impl/TestRequestFactory.java | 109 +-
.../hadoop/fs/s3a/impl/TestSDKStreamDrainer.java | 16 +-
.../performance/ITestDirectoryMarkerListing.java | 25 +-
.../fs/s3a/prefetch/MockS3ARemoteObject.java | 27 +-
.../hadoop/fs/s3a/prefetch/S3APrefetchFakes.java | 48 +-
.../fs/s3a/scale/AbstractSTestS3AHugeFiles.java | 25 +-
.../scale/ILoadTestS3ABulkDeleteThrottling.java | 13 +-
.../fs/s3a/scale/ITestS3ADirectoryPerformance.java | 22 +-
.../s3a/scale/ITestS3AHugeFilesStorageClass.java | 4 +-
.../hadoop/fs/s3a/select/AbstractS3SelectTest.java | 4 +-
.../hadoop/fs/s3a/select/ITestS3SelectLandsat.java | 2 +-
.../hadoop/fs/s3a/select/StreamPublisher.java | 89 ++
.../fs/s3a/select/TestBlockingEnumeration.java | 200 ++++
.../s3a/select/TestSelectEventStreamPublisher.java | 190 ++++
.../statistics/ITestAWSStatisticCollection.java | 4 +-
.../apache/hadoop/fs/s3a/test/ExtraAssertions.java | 2 +-
.../fs/s3a/test/MinimalOperationCallbacks.java | 14 +-
.../test/MinimalWriteOperationHelperCallbacks.java | 16 +-
.../hadoop/fs/s3a/tools/ITestMarkerTool.java | 7 -
205 files changed, 8900 insertions(+), 5946 deletions(-)
diff --git a/LICENSE-binary b/LICENSE-binary
index d84f311ce190..f2e7d9b28b26 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -364,6 +364,7 @@ org.objenesis:objenesis:2.6
org.xerial.snappy:snappy-java:1.1.10.4
org.yaml:snakeyaml:2.0
org.wildfly.openssl:wildfly-openssl:1.1.3.Final
+software.amazon.awssdk:bundle:jar:2.20.128
--------------------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java
index c04c1bb47fce..3a8927aba493 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java
@@ -407,6 +407,10 @@ public final class StoreStatisticNames {
public static final String MULTIPART_UPLOAD_LIST
= "multipart_upload_list";
+ /** Probe for store region: {@value}. */
+ public static final String STORE_REGION_PROBE
+ = "store_region_probe";
+
private StoreStatisticNames() {
}
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 1501ee4bc268..6af4f793732a 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1201,17 +1201,24 @@
<description>AWS secret key used by S3A file system. Omit for IAM role-based or provider-based authentication.</description>
</property>
+<property>
+ <name>fs.s3a.session.token</name>
+ <description>Session token, when using org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider
+ as one of the providers.
+ </description>
+</property>
+
<property>
<name>fs.s3a.aws.credentials.provider</name>
<value>
org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider,
org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,
- com.amazonaws.auth.EnvironmentVariableCredentialsProvider,
+ software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider,
org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider
</value>
<description>
Comma-separated class names of credential provider classes which implement
- com.amazonaws.auth.AWSCredentialsProvider.
+ software.amazon.awssdk.auth.credentials.AwsCredentialsProvider.
When S3A delegation tokens are not enabled, this list will be used
to directly authenticate with S3 and other AWS services.
@@ -1219,43 +1226,6 @@
token binding it may be used
to communicate wih the STS endpoint to request session/role
credentials.
-
- These are loaded and queried in sequence for a valid set of credentials.
- Each listed class must implement one of the following means of
- construction, which are attempted in order:
- * a public constructor accepting java.net.URI and
- org.apache.hadoop.conf.Configuration,
- * a public constructor accepting org.apache.hadoop.conf.Configuration,
- * a public static method named getInstance that accepts no
- arguments and returns an instance of
- com.amazonaws.auth.AWSCredentialsProvider, or
- * a public default constructor.
-
- Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows
- anonymous access to a publicly accessible S3 bucket without any credentials.
- Please note that allowing anonymous access to an S3 bucket compromises
- security and therefore is unsuitable for most use cases. It can be useful
- for accessing public data sets without requiring AWS credentials.
-
- If unspecified, then the default list of credential provider classes,
- queried in sequence, is:
- * org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider: looks
- for session login secrets in the Hadoop configuration.
- * org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider:
- Uses the values of fs.s3a.access.key and fs.s3a.secret.key.
- * com.amazonaws.auth.EnvironmentVariableCredentialsProvider: supports
- configuration of AWS access key ID and secret access key in
- environment variables named AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,
- and AWS_SESSION_TOKEN as documented in the AWS SDK.
- * org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider: picks up
- IAM credentials of any EC2 VM or AWS container in which the process is running.
- </description>
-</property>
-
-<property>
- <name>fs.s3a.session.token</name>
- <description>Session token, when using org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider
- as one of the providers.
</description>
</property>
@@ -1353,10 +1323,10 @@
Note: for job submission to actually collect these tokens,
Kerberos must be enabled.
- Options are:
+ Bindings available in hadoop-aws are:
org.apache.hadoop.fs.s3a.auth.delegation.SessionTokenBinding
org.apache.hadoop.fs.s3a.auth.delegation.FullCredentialsTokenBinding
- and org.apache.hadoop.fs.s3a.auth.delegation.RoleTokenBinding
+ org.apache.hadoop.fs.s3a.auth.delegation.RoleTokenBinding
</description>
</property>
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index c43f846331bb..77e12835e8bf 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -193,6 +193,8 @@
<surefire.fork.timeout>900</surefire.fork.timeout>
<aws-java-sdk.version>1.12.499</aws-java-sdk.version>
<hsqldb.version>2.7.1</hsqldb.version>
+ <aws-java-sdk-v2.version>2.20.128</aws-java-sdk-v2.version>
+ <aws.eventstream.version>1.0.1</aws.eventstream.version>
<frontend-maven-plugin.version>1.11.2</frontend-maven-plugin.version>
<jasmine-maven-plugin.version>2.1</jasmine-maven-plugin.version>
<phantomjs-maven-plugin.version>0.7</phantomjs-maven-plugin.version>
@@ -1103,15 +1105,31 @@
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-bundle</artifactId>
+ <artifactId>aws-java-sdk-core</artifactId>
<version>${aws-java-sdk.version}</version>
<exclusions>
<exclusion>
- <groupId>io.netty</groupId>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>bundle</artifactId>
+ <version>${aws-java-sdk-v2.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>software.amazon.eventstream</groupId>
+ <artifactId>eventstream</artifactId>
+ <version>${aws.eventstream.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.mina</groupId>
<artifactId>mina-core</artifactId>
diff --git a/hadoop-tools/hadoop-aws/dev-support/findbugs-exclude.xml b/hadoop-tools/hadoop-aws/dev-support/findbugs-exclude.xml
index 4bc00a3e5f16..17f7db631c3e 100644
--- a/hadoop-tools/hadoop-aws/dev-support/findbugs-exclude.xml
+++ b/hadoop-tools/hadoop-aws/dev-support/findbugs-exclude.xml
@@ -64,6 +64,11 @@
<Field name="futurePool"/>
<Bug pattern="IS2_INCONSISTENT_SYNC"/>
</Match>
+ <Match>
+ <Class name="org.apache.hadoop.fs.s3a.S3AFileSystem"/>
+ <Field name="s3AsyncClient"/>
+ <Bug pattern="IS2_INCONSISTENT_SYNC"/>
+ </Match>
<Match>
<Class name="org.apache.hadoop.fs.s3a.s3guard.S3GuardTool$BucketInfo"/>
<Method name="run"/>
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 207b1c0f9413..85880b8482f1 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -483,11 +483,25 @@
<scope>test</scope>
<type>test-jar</type>
</dependency>
+
+ <!-- The v1 SDK is used at compilation time for adapter classes in
+ org.apache.hadoop.fs.s3a.adapter. It is not needed at runtime
+ unless a non-standard v1 credential provider is declared. -->
<dependency>
<groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-bundle</artifactId>
+ <artifactId>aws-java-sdk-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>bundle</artifactId>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>software.amazon.eventstream</groupId>
+ <artifactId>eventstream</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java
index 482c5a1db7a1..c5867eeab4f4 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.AmazonServiceException;
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
/**
* A 400 "Bad Request" exception was received.
@@ -36,7 +36,7 @@ public class AWSBadRequestException extends AWSServiceIOException {
* @param cause the underlying cause
*/
public AWSBadRequestException(String operation,
- AmazonServiceException cause) {
+ AwsServiceException cause) {
super(operation, cause);
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSClientIOException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSClientIOException.java
index f31f3c79b355..43b636c6d576 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSClientIOException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSClientIOException.java
@@ -18,29 +18,28 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.SdkBaseException;
+import software.amazon.awssdk.core.exception.SdkException;
import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
import java.io.IOException;
/**
- * IOException equivalent of an {@link AmazonClientException}.
+ * IOException equivalent of an {@link SdkException}.
*/
public class AWSClientIOException extends IOException {
private final String operation;
public AWSClientIOException(String operation,
- SdkBaseException cause) {
+ SdkException cause) {
super(cause);
Preconditions.checkArgument(operation != null, "Null 'operation' argument");
Preconditions.checkArgument(cause != null, "Null 'cause' argument");
this.operation = operation;
}
- public AmazonClientException getCause() {
- return (AmazonClientException) super.getCause();
+ public SdkException getCause() {
+ return (SdkException) super.getCause();
}
@Override
@@ -48,4 +47,11 @@ public class AWSClientIOException extends IOException {
return operation + ": " + getCause().getMessage();
}
+ /**
+ * Query inner cause for retryability.
+ * @return what the cause says.
+ */
+ public boolean retryable() {
+ return getCause().retryable();
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java
index 75b09a4ed746..1a084cd2357d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.fs.s3a;
import java.io.Closeable;
+import java.io.IOException;
+import java.nio.file.AccessDeniedException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -27,21 +29,22 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.AnonymousAWSCredentials;
-import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.s3a.auth.NoAuthWithAWSException;
import org.apache.hadoop.fs.s3a.auth.NoAwsCredentialsException;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.core.exception.SdkException;
/**
* A list of providers.
@@ -51,17 +54,17 @@ import org.apache.hadoop.io.IOUtils;
* <ol>
* <li>Allows extra providers to be added dynamically.</li>
* <li>If any provider in the chain throws an exception other than
- * an {@link AmazonClientException}, that is rethrown, rather than
+ * an {@link SdkException}, that is rethrown, rather than
* swallowed.</li>
* <li>Has some more diagnostics.</li>
- * <li>On failure, the last "relevant" AmazonClientException raised is
+ * <li>On failure, the last "relevant" {@link SdkException} raised is
* rethrown; exceptions other than 'no credentials' have priority.</li>
- * <li>Special handling of {@link AnonymousAWSCredentials}.</li>
+ * <li>Special handling of {@link AnonymousCredentialsProvider}.</li>
* </ol>
*/
-@InterfaceAudience.Private
+@InterfaceAudience.LimitedPrivate("extensions")
@InterfaceStability.Evolving
-public final class AWSCredentialProviderList implements AWSCredentialsProvider,
+public final class AWSCredentialProviderList implements AwsCredentialsProvider,
AutoCloseable {
private static final Logger LOG = LoggerFactory.getLogger(
@@ -73,9 +76,9 @@ public final class AWSCredentialProviderList implements AWSCredentialsProvider,
CREDENTIALS_REQUESTED_WHEN_CLOSED
= "Credentials requested after provider list was closed";
- private final List<AWSCredentialsProvider> providers = new ArrayList<>(1);
+ private final List<AwsCredentialsProvider> providers = new ArrayList<>(1);
private boolean reuseLastProvider = true;
- private AWSCredentialsProvider lastProvider;
+ private AwsCredentialsProvider lastProvider;
private final AtomicInteger refCount = new AtomicInteger(1);
@@ -98,17 +101,17 @@ public final class AWSCredentialProviderList implements AWSCredentialsProvider,
* @param providers provider list.
*/
public AWSCredentialProviderList(
- Collection<AWSCredentialsProvider> providers) {
+ Collection<AwsCredentialsProvider> providers) {
this.providers.addAll(providers);
}
/**
- * Create with an initial list of providers.
+ * Create with an initial list of SDK V2 credential providers.
* @param name name for error messages, may be ""
* @param providerArgs provider list.
*/
public AWSCredentialProviderList(final String name,
- final AWSCredentialsProvider... providerArgs) {
+ final AwsCredentialsProvider... providerArgs) {
setName(name);
Collections.addAll(providers, providerArgs);
}
@@ -126,11 +129,11 @@ public final class AWSCredentialProviderList implements AWSCredentialsProvider,
}
/**
- * Add a new provider.
- * @param p provider
+ * Add a new SDK V2 provider.
+ * @param provider provider
*/
- public void add(AWSCredentialsProvider p) {
- providers.add(p);
+ public void add(AwsCredentialsProvider provider) {
+ providers.add(provider);
}
/**
@@ -142,16 +145,11 @@ public final class AWSCredentialProviderList implements AWSCredentialsProvider,
}
/**
- * Refresh all child entries.
+ * Was an implementation of the v1 refresh; now just
+ * a no-op.
*/
- @Override
+ @Deprecated
public void refresh() {
- if (isClosed()) {
- return;
- }
- for (AWSCredentialsProvider provider : providers) {
- provider.refresh();
- }
}
/**
@@ -160,7 +158,7 @@ public final class AWSCredentialProviderList implements AWSCredentialsProvider,
* @return a set of credentials (possibly anonymous), for authenticating.
*/
@Override
- public AWSCredentials getCredentials() {
+ public AwsCredentials resolveCredentials() {
if (isClosed()) {
LOG.warn(CREDENTIALS_REQUESTED_WHEN_CLOSED);
throw new NoAuthWithAWSException(name +
@@ -168,18 +166,18 @@ public final class AWSCredentialProviderList implements AWSCredentialsProvider,
}
checkNotEmpty();
if (reuseLastProvider && lastProvider != null) {
- return lastProvider.getCredentials();
+ return lastProvider.resolveCredentials();
}
- AmazonClientException lastException = null;
- for (AWSCredentialsProvider provider : providers) {
+ SdkException lastException = null;
+ for (AwsCredentialsProvider provider : providers) {
try {
- AWSCredentials credentials = provider.getCredentials();
+ AwsCredentials credentials = provider.resolveCredentials();
Preconditions.checkNotNull(credentials,
"Null credentials returned by %s", provider);
- if ((credentials.getAWSAccessKeyId() != null &&
- credentials.getAWSSecretKey() != null)
- || (credentials instanceof AnonymousAWSCredentials)) {
+ if ((credentials.accessKeyId() != null && credentials.secretAccessKey() != null) || (
+ provider instanceof AnonymousCredentialsProvider
+ || provider instanceof AnonymousAWSCredentialsProvider)) {
lastProvider = provider;
LOG.debug("Using credentials from {}", provider);
return credentials;
@@ -196,7 +194,7 @@ public final class AWSCredentialProviderList implements AWSCredentialsProvider,
}
LOG.debug("No credentials from {}: {}",
provider, e.toString());
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
lastException = e;
LOG.debug("No credentials provided by {}: {}",
provider, e.toString(), e);
@@ -222,14 +220,13 @@ public final class AWSCredentialProviderList implements AWSCredentialsProvider,
*
* @return providers
*/
- @VisibleForTesting
- List<AWSCredentialsProvider> getProviders() {
+ public List<AwsCredentialsProvider> getProviders() {
return providers;
}
/**
* Verify that the provider list is not empty.
- * @throws AmazonClientException if there are no providers.
+ * @throws SdkException if there are no providers.
*/
public void checkNotEmpty() {
if (providers.isEmpty()) {
@@ -255,9 +252,11 @@ public final class AWSCredentialProviderList implements AWSCredentialsProvider,
*/
@Override
public String toString() {
- return "AWSCredentialProviderList[" +
- name +
- "refcount= " + refCount.get() + ": [" +
+ return "AWSCredentialProviderList"
+ + " name=" + name
+ + "; refcount= " + refCount.get()
+ + "; size="+ providers.size()
+ + ": [" +
StringUtils.join(providers, ", ") + ']'
+ (lastProvider != null ? (" last provider: " + lastProvider) : "");
}
@@ -317,7 +316,7 @@ public final class AWSCredentialProviderList implements AWSCredentialsProvider,
}
// do this outside the synchronized block.
- for (AWSCredentialsProvider p : providers) {
+ for (AwsCredentialsProvider p : providers) {
if (p instanceof Closeable) {
IOUtils.closeStream((Closeable) p);
} else if (p instanceof AutoCloseable) {
@@ -333,4 +332,27 @@ public final class AWSCredentialProviderList implements AWSCredentialsProvider,
public int size() {
return providers.size();
}
+
+
+ /**
+ * Translate an exception if it or its inner exception is an
+ * {@link CredentialInitializationException}.
+ * If this condition is not met, null is returned.
+ * @param path path of operation.
+ * @param throwable exception
+ * @return a translated exception or null.
+ */
+ public static IOException maybeTranslateCredentialException(String path,
+ Throwable throwable) {
+ if (throwable instanceof CredentialInitializationException) {
+ // the exception raised by AWSCredentialProvider list if the
+ // credentials were not accepted,
+ return (AccessDeniedException)new AccessDeniedException(path, null,
+ throwable.toString()).initCause(throwable);
+ } else if (throwable.getCause() instanceof CredentialInitializationException) {
+ return maybeTranslateCredentialException(path, throwable.getCause());
+ } else {
+ return null;
+ }
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSNoResponseException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSNoResponseException.java
index e6a23b2361da..b8562714b1aa 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSNoResponseException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSNoResponseException.java
@@ -18,14 +18,19 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.AmazonServiceException;
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
/**
* Status code 443, no response from server. This is considered idempotent.
*/
public class AWSNoResponseException extends AWSServiceIOException {
public AWSNoResponseException(String operation,
- AmazonServiceException cause) {
+ AwsServiceException cause) {
super(operation, cause);
}
+
+ @Override
+ public boolean retryable() {
+ return true;
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSRedirectException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSRedirectException.java
index bb337ee5eebd..cb478482a8ed 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSRedirectException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSRedirectException.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.AmazonServiceException;
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
/**
* Request is redirected.
@@ -32,7 +32,7 @@ public class AWSRedirectException extends AWSServiceIOException {
* @param cause the underlying cause
*/
public AWSRedirectException(String operation,
- AmazonServiceException cause) {
+ AwsServiceException cause) {
super(operation, cause);
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSS3IOException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSS3IOException.java
index 014d217b6a4f..de1dd8b4a7a1 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSS3IOException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSS3IOException.java
@@ -18,14 +18,13 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.services.s3.model.AmazonS3Exception;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import java.util.Map;
-
/**
- * Wrap a {@link AmazonS3Exception} as an IOE, relaying all
+ * Wrap a {@link S3Exception} as an IOE, relaying all
* getters.
*/
@InterfaceAudience.Public
@@ -38,24 +37,12 @@ public class AWSS3IOException extends AWSServiceIOException {
* @param cause the underlying cause
*/
public AWSS3IOException(String operation,
- AmazonS3Exception cause) {
+ S3Exception cause) {
super(operation, cause);
}
- public AmazonS3Exception getCause() {
- return (AmazonS3Exception) super.getCause();
- }
-
- public String getErrorResponseXml() {
- return getCause().getErrorResponseXml();
- }
-
- public Map<String, String> getAdditionalDetails() {
- return getCause().getAdditionalDetails();
- }
-
- public String getExtendedRequestId() {
- return getCause().getExtendedRequestId();
+ public S3Exception getCause() {
+ return (S3Exception) super.getCause();
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceIOException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceIOException.java
index a9c2c9840203..434ec8df292a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceIOException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceIOException.java
@@ -18,13 +18,15 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.AmazonServiceException;
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
+import software.amazon.awssdk.awscore.exception.AwsErrorDetails;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+
/**
* A specific exception from AWS operations.
- * The exception must always be created with an {@link AmazonServiceException}.
+ * The exception must always be created with an {@link AwsServiceException}.
* The attributes of this exception can all be directly accessed.
*/
@InterfaceAudience.Public
@@ -37,36 +39,27 @@ public class AWSServiceIOException extends AWSClientIOException {
* @param cause the underlying cause
*/
public AWSServiceIOException(String operation,
- AmazonServiceException cause) {
+ AwsServiceException cause) {
super(operation, cause);
}
- public AmazonServiceException getCause() {
- return (AmazonServiceException) super.getCause();
- }
-
- public String getRequestId() {
- return getCause().getRequestId();
+ public AwsServiceException getCause() {
+ return (AwsServiceException) super.getCause();
}
- public String getServiceName() {
- return getCause().getServiceName();
+ public String requestId() {
+ return getCause().requestId();
}
- public String getErrorCode() {
- return getCause().getErrorCode();
+ public AwsErrorDetails awsErrorDetails() {
+ return getCause().awsErrorDetails();
}
- public int getStatusCode() {
- return getCause().getStatusCode();
+ public int statusCode() {
+ return getCause().statusCode();
}
- public String getRawResponseContent() {
- return getCause().getRawResponseContent();
+ public String extendedRequestId() {
+ return getCause().extendedRequestId();
}
-
- public boolean isRetryable() {
- return getCause().isRetryable();
- }
-
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceThrottledException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceThrottledException.java
index 131cea756224..5cd2eb9d320f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceThrottledException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceThrottledException.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.AmazonServiceException;
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
/**
* Exception raised when a service was throttled.
@@ -36,7 +36,12 @@ public class AWSServiceThrottledException extends AWSServiceIOException {
* @param cause the underlying cause
*/
public AWSServiceThrottledException(String operation,
- AmazonServiceException cause) {
+ AwsServiceException cause) {
super(operation, cause);
}
+
+ @Override
+ public boolean retryable() {
+ return true;
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java
index 83be294fac7c..ecfe5da1455c 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.AmazonServiceException;
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
/**
* A 500 response came back from a service.
@@ -31,7 +31,12 @@ import com.amazonaws.AmazonServiceException;
*/
public class AWSStatus500Exception extends AWSServiceIOException {
public AWSStatus500Exception(String operation,
- AmazonServiceException cause) {
+ AwsServiceException cause) {
super(operation, cause);
}
+
+ @Override
+ public boolean retryable() {
+ return true;
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java
index 564c03bf731d..dcfc2a03b123 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java
@@ -18,9 +18,10 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.AnonymousAWSCredentials;
-import com.amazonaws.auth.AWSCredentials;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -35,23 +36,18 @@ import org.apache.hadoop.classification.InterfaceStability;
* property fs.s3a.aws.credentials.provider. Therefore, changing the class name
* would be a backward-incompatible change.
*
- * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
- * as part of upgrading S3A to SDK V2. See HADOOP-18073.
*/
@InterfaceAudience.Private
@InterfaceStability.Stable
-@Deprecated
-public class AnonymousAWSCredentialsProvider implements AWSCredentialsProvider {
+public class AnonymousAWSCredentialsProvider implements AwsCredentialsProvider {
public static final String NAME
= "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider";
- public AWSCredentials getCredentials() {
- return new AnonymousAWSCredentials();
+ public AwsCredentials resolveCredentials() {
+ return AnonymousCredentialsProvider.create().resolveCredentials();
}
- public void refresh() {}
-
@Override
public String toString() {
return getClass().getSimpleName();
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java
index a85f26223ffc..98745b295b5d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.fs.s3a;
import javax.annotation.Nonnull;
-import com.amazonaws.arn.Arn;
+import software.amazon.awssdk.arns.Arn;
/**
* Represents an Arn Resource, this can be an accesspoint or bucket.
@@ -126,14 +126,14 @@ public final class ArnResource {
public static ArnResource accessPointFromArn(String arn) throws IllegalArgumentException {
Arn parsed = Arn.fromString(arn);
- if (parsed.getRegion().isEmpty() || parsed.getAccountId().isEmpty() ||
- parsed.getResourceAsString().isEmpty()) {
+ if (!parsed.region().isPresent() || !parsed.accountId().isPresent() ||
+ parsed.resourceAsString().isEmpty()) {
throw new IllegalArgumentException(
String.format("Access Point Arn %s has an invalid format or missing properties", arn));
}
- String resourceName = parsed.getResource().getResource();
- return new ArnResource(resourceName, parsed.getAccountId(), parsed.getRegion(),
- parsed.getPartition(), arn);
+ String resourceName = parsed.resource().resource();
+ return new ArnResource(resourceName, parsed.accountId().get(), parsed.region().get(),
+ parsed.partition(), arn);
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index edd63b5f263c..e52e4a711a6d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -140,7 +140,6 @@ public final class Constants {
public static final String ASSUMED_ROLE_POLICY =
"fs.s3a.assumed.role.policy";
- @SuppressWarnings("deprecation")
public static final String ASSUMED_ROLE_CREDENTIALS_DEFAULT =
SimpleAWSCredentialsProvider.NAME;
@@ -597,7 +596,7 @@ public final class Constants {
public static final String SIGNING_ALGORITHM_STS =
"fs.s3a." + Constants.AWS_SERVICE_IDENTIFIER_STS.toLowerCase()
- + "signing-algorithm";
+ + ".signing-algorithm";
public static final String S3N_FOLDER_SUFFIX = "_$folder$";
public static final String FS_S3A_BLOCK_SIZE = "fs.s3a.block.size";
@@ -739,14 +738,21 @@ public final class Constants {
public static final String STREAM_READ_GAUGE_INPUT_POLICY =
"stream_read_gauge_input_policy";
+ /**
+ * S3 Client Factory implementation class: {@value}.
+ * Unstable and incompatible between v1 and v2 SDK versions.
+ */
@InterfaceAudience.Private
@InterfaceStability.Unstable
public static final String S3_CLIENT_FACTORY_IMPL =
"fs.s3a.s3.client.factory.impl";
+ /**
+ * Default factory:
+ * {@code org.apache.hadoop.fs.s3a.DefaultS3ClientFactory}.
+ */
@InterfaceAudience.Private
@InterfaceStability.Unstable
- @SuppressWarnings("deprecation")
public static final Class<? extends S3ClientFactory>
DEFAULT_S3_CLIENT_FACTORY_IMPL =
DefaultS3ClientFactory.class;
@@ -1299,4 +1305,8 @@ public final class Constants {
*/
public static final int DEFAULT_PREFETCH_MAX_BLOCKS_COUNT = 4;
+ /**
+ * The bucket region header.
+ */
+ public static final String BUCKET_REGION_HEADER = "x-amz-bucket-region";
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CredentialInitializationException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CredentialInitializationException.java
index 2f0cfd37ad37..c6a1b8fcc713 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CredentialInitializationException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CredentialInitializationException.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.AmazonClientException;
+import software.amazon.awssdk.core.exception.SdkClientException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -26,22 +26,22 @@ import org.apache.hadoop.classification.InterfaceStability;
/**
* Exception which Hadoop's AWSCredentialsProvider implementations should
* throw when there is a problem with the credential setup. This
- * is a subclass of {@link AmazonClientException} which sets
- * {@link #isRetryable()} to false, so as to fail fast.
+ * is a subclass of {@link SdkClientException} which sets
+ * {@link #retryable()} to false, so as to fail fast.
* This is used in credential providers and elsewhere.
* When passed through {@code S3AUtils.translateException()} it
- * is mapped to an AccessDeniedException. As a result, the Invoker
- * code will automatically translate
+ * is mapped to an AccessDeniedException.
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
-public class CredentialInitializationException extends AmazonClientException {
+public class CredentialInitializationException extends SdkClientException {
+
public CredentialInitializationException(String message, Throwable t) {
- super(message, t);
+ super(builder().message(message).cause(t));
}
public CredentialInitializationException(String message) {
- super(message);
+ super(builder().message(message));
}
/**
@@ -49,7 +49,7 @@ public class CredentialInitializationException extends AmazonClientException {
* @return false, always.
*/
@Override
- public boolean isRetryable() {
+ public boolean retryable() {
return false;
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
index bab4ec896ff7..98c72d276628 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
@@ -20,32 +20,26 @@ package org.apache.hadoop.fs.s3a;
import java.io.IOException;
import java.net.URI;
+import java.net.URISyntaxException;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.SdkClientException;
-import com.amazonaws.client.builder.AwsClientBuilder;
-import com.amazonaws.handlers.RequestHandler2;
-import com.amazonaws.regions.RegionUtils;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3Builder;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.AmazonS3ClientBuilder;
-import com.amazonaws.services.s3.AmazonS3EncryptionClientV2Builder;
-import com.amazonaws.services.s3.AmazonS3EncryptionV2;
-import com.amazonaws.services.s3.S3ClientOptions;
-import com.amazonaws.services.s3.internal.ServiceUtils;
-import com.amazonaws.services.s3.model.CryptoConfigurationV2;
-import com.amazonaws.services.s3.model.CryptoMode;
-import com.amazonaws.services.s3.model.CryptoRangeGetMode;
-import com.amazonaws.services.s3.model.EncryptionMaterialsProvider;
-import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider;
-import com.amazonaws.util.AwsHostNameUtils;
-import com.amazonaws.util.RuntimeHttpUtils;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.fs.s3a.impl.AWSClientConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
+import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
+import software.amazon.awssdk.core.retry.RetryPolicy;
+import software.amazon.awssdk.http.apache.ApacheHttpClient;
+import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.S3BaseClientBuilder;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3Configuration;
+import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration;
+import software.amazon.awssdk.transfer.s3.S3TransferManager;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -54,15 +48,11 @@ import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.s3a.statistics.impl.AwsStatisticsCollector;
import org.apache.hadoop.fs.store.LogExactlyOnce;
-import static com.amazonaws.services.s3.Headers.REQUESTER_PAYS_HEADER;
-import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION;
-import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_CENTRAL_REGION;
-import static org.apache.hadoop.fs.s3a.Constants.EXPERIMENTAL_AWS_INTERNAL_THROTTLING;
-import static org.apache.hadoop.fs.s3a.Constants.EXPERIMENTAL_AWS_INTERNAL_THROTTLING_DEFAULT;
-import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY;
-import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm;
-import static org.apache.hadoop.fs.s3a.S3AUtils.getS3EncryptionKey;
-import static org.apache.hadoop.fs.s3a.S3AUtils.translateException;
+import static org.apache.hadoop.fs.s3a.impl.AWSHeaders.REQUESTER_PAYS_HEADER;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_SECURE_CONNECTIONS;
+import static org.apache.hadoop.fs.s3a.Constants.SECURE_CONNECTIONS;
+import static org.apache.hadoop.fs.s3a.Constants.AWS_SERVICE_IDENTIFIER_S3;
+
/**
* The default {@link S3ClientFactory} implementation.
@@ -71,12 +61,9 @@ import static org.apache.hadoop.fs.s3a.S3AUtils.translateException;
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-@SuppressWarnings("deprecation")
public class DefaultS3ClientFactory extends Configured
implements S3ClientFactory {
- private static final String S3_SERVICE_NAME = "s3";
-
private static final String REQUESTER_PAYS_HEADER_VALUE = "requester";
/**
@@ -85,310 +72,161 @@ public class DefaultS3ClientFactory extends Configured
protected static final Logger LOG =
LoggerFactory.getLogger(DefaultS3ClientFactory.class);
- /**
- * A one-off warning of default region chains in use.
- */
- private static final LogExactlyOnce WARN_OF_DEFAULT_REGION_CHAIN =
- new LogExactlyOnce(LOG);
-
- /**
- * Warning message printed when the SDK Region chain is in use.
- */
- private static final String SDK_REGION_CHAIN_IN_USE =
- "S3A filesystem client is using"
- + " the SDK region resolution chain.";
/** Exactly once log to inform about ignoring the AWS-SDK Warnings for CSE. */
private static final LogExactlyOnce IGNORE_CSE_WARN = new LogExactlyOnce(LOG);
- /** Bucket name. */
- private String bucket;
-
- /**
- * Create the client by preparing the AwsConf configuration
- * and then invoking {@code buildAmazonS3Client()}.
- */
@Override
- public AmazonS3 createS3Client(
+ public S3Client createS3Client(
final URI uri,
final S3ClientCreationParameters parameters) throws IOException {
- Configuration conf = getConf();
- bucket = uri.getHost();
- final ClientConfiguration awsConf = S3AUtils
- .createAwsConf(conf,
- bucket,
- Constants.AWS_SERVICE_IDENTIFIER_S3);
- // add any headers
- parameters.getHeaders().forEach((h, v) ->
- awsConf.addHeader(h, v));
- if (parameters.isRequesterPays()) {
- // All calls must acknowledge requester will pay via header.
- awsConf.addHeader(REQUESTER_PAYS_HEADER, REQUESTER_PAYS_HEADER_VALUE);
- }
+ Configuration conf = getConf();
+ String bucket = uri.getHost();
+
+ ApacheHttpClient.Builder httpClientBuilder = AWSClientConfig
+ .createHttpClientBuilder(conf)
+ .proxyConfiguration(AWSClientConfig.createProxyConfiguration(conf, bucket));
+ return configureClientBuilder(S3Client.builder(), parameters, conf, bucket)
+ .httpClientBuilder(httpClientBuilder)
+ .build();
+ }
- // When EXPERIMENTAL_AWS_INTERNAL_THROTTLING is false
- // throttling is explicitly disabled on the S3 client so that
- // all failures are collected in S3A instrumentation, and its
- // retry policy is the only one used.
- // This may cause problems in copy/rename.
- awsConf.setUseThrottleRetries(
- conf.getBoolean(EXPERIMENTAL_AWS_INTERNAL_THROTTLING,
- EXPERIMENTAL_AWS_INTERNAL_THROTTLING_DEFAULT));
+ @Override
+ public S3AsyncClient createS3AsyncClient(
+ final URI uri,
+ final S3ClientCreationParameters parameters) throws IOException {
- if (!StringUtils.isEmpty(parameters.getUserAgentSuffix())) {
- awsConf.setUserAgentSuffix(parameters.getUserAgentSuffix());
- }
+ Configuration conf = getConf();
+ String bucket = uri.getHost();
+
+ NettyNioAsyncHttpClient.Builder httpClientBuilder = AWSClientConfig
+ .createAsyncHttpClientBuilder(conf)
+ .proxyConfiguration(AWSClientConfig.createAsyncProxyConfiguration(conf, bucket));
+
+ MultipartConfiguration multipartConfiguration = MultipartConfiguration.builder()
+ .minimumPartSizeInBytes(parameters.getMinimumPartSize())
+ .thresholdInBytes(parameters.getMultiPartThreshold())
+ .build();
+
+ return configureClientBuilder(S3AsyncClient.builder(), parameters, conf, bucket)
+ .httpClientBuilder(httpClientBuilder)
+ .multipartConfiguration(multipartConfiguration)
+ .multipartEnabled(true)
+ .build();
+ }
- // Get the encryption method for this bucket.
- S3AEncryptionMethods encryptionMethods =
- getEncryptionAlgorithm(bucket, conf);
- try {
- // If CSE is enabled then build a S3EncryptionClient.
- if (S3AEncryptionMethods.CSE_KMS.getMethod()
- .equals(encryptionMethods.getMethod())) {
- return buildAmazonS3EncryptionClient(
- awsConf,
- parameters);
- } else {
- return buildAmazonS3Client(
- awsConf,
- parameters);
- }
- } catch (SdkClientException e) {
- // SDK refused to build.
- throw translateException("creating AWS S3 client", uri.toString(), e);
- }
+ @Override
+ public S3TransferManager createS3TransferManager(final S3AsyncClient s3AsyncClient) {
+ return S3TransferManager.builder()
+ .s3Client(s3AsyncClient)
+ .build();
}
/**
- * Create an {@link AmazonS3} client of type
- * {@link AmazonS3EncryptionV2} if CSE is enabled.
- *
- * @param awsConf AWS configuration.
- * @param parameters parameters.
- *
- * @return new AmazonS3 client.
- * @throws IOException if lookupPassword() has any problem.
+ * Configure a sync or async S3 client builder.
+ * This method handles all shared configuration.
+ * @param builder S3 client builder
+ * @param parameters parameter object
+ * @param conf configuration object
+ * @param bucket bucket name
+ * @return the builder object
+ * @param <BuilderT> S3 client builder type
+ * @param <ClientT> S3 client type
*/
- protected AmazonS3 buildAmazonS3EncryptionClient(
- final ClientConfiguration awsConf,
- final S3ClientCreationParameters parameters) throws IOException {
+ private <BuilderT extends S3BaseClientBuilder<BuilderT, ClientT>, ClientT> BuilderT configureClientBuilder(
+ BuilderT builder, S3ClientCreationParameters parameters, Configuration conf, String bucket)
+ throws IOException {
- AmazonS3 client;
- AmazonS3EncryptionClientV2Builder builder =
- new AmazonS3EncryptionClientV2Builder();
- Configuration conf = getConf();
+ Region region = parameters.getRegion();
+ LOG.debug("Using region {}", region);
- // CSE-KMS Method
- String kmsKeyId = getS3EncryptionKey(bucket, conf, true);
- // Check if kmsKeyID is not null
- Preconditions.checkArgument(!StringUtils.isBlank(kmsKeyId), "CSE-KMS "
- + "method requires KMS key ID. Use " + S3_ENCRYPTION_KEY
- + " property to set it. ");
-
- EncryptionMaterialsProvider materialsProvider =
- new KMSEncryptionMaterialsProvider(kmsKeyId);
- builder.withEncryptionMaterialsProvider(materialsProvider);
- //Configure basic params of a S3 builder.
- configureBasicParams(builder, awsConf, parameters);
-
- // Configuring endpoint.
- AmazonS3EncryptionClientV2Builder.EndpointConfiguration epr
- = createEndpointConfiguration(parameters.getEndpoint(),
- awsConf, getConf().getTrimmed(AWS_REGION));
- configureEndpoint(builder, epr);
-
- // Create cryptoConfig.
- CryptoConfigurationV2 cryptoConfigurationV2 =
- new CryptoConfigurationV2(CryptoMode.AuthenticatedEncryption)
- .withRangeGetMode(CryptoRangeGetMode.ALL);
- if (epr != null) {
- cryptoConfigurationV2
- .withAwsKmsRegion(RegionUtils.getRegion(epr.getSigningRegion()));
- LOG.debug("KMS region used: {}", cryptoConfigurationV2.getAwsKmsRegion());
+ URI endpoint = getS3Endpoint(parameters.getEndpoint(), conf);
+
+ if (endpoint != null) {
+ builder.endpointOverride(endpoint);
+ LOG.debug("Using endpoint {}", endpoint);
}
- builder.withCryptoConfiguration(cryptoConfigurationV2);
- client = builder.build();
- IGNORE_CSE_WARN.info("S3 client-side encryption enabled: Ignore S3-CSE "
- + "Warnings.");
- return client;
- }
+ S3Configuration serviceConfiguration = S3Configuration.builder()
+ .pathStyleAccessEnabled(parameters.isPathStyleAccess())
+ .build();
- /**
- * Use the Builder API to create an AWS S3 client.
- * <p>
- * This has a more complex endpoint configuration mechanism
- * which initially caused problems; the
- * {@code withForceGlobalBucketAccessEnabled(true)}
- * command is critical here.
- * @param awsConf AWS configuration
- * @param parameters parameters
- * @return new AmazonS3 client
- * @throws SdkClientException if the configuration is invalid.
- */
- protected AmazonS3 buildAmazonS3Client(
- final ClientConfiguration awsConf,
- final S3ClientCreationParameters parameters) {
- AmazonS3ClientBuilder b = AmazonS3Client.builder();
- configureBasicParams(b, awsConf, parameters);
-
- // endpoint set up is a PITA
- AwsClientBuilder.EndpointConfiguration epr
- = createEndpointConfiguration(parameters.getEndpoint(),
- awsConf, getConf().getTrimmed(AWS_REGION));
- configureEndpoint(b, epr);
- final AmazonS3 client = b.build();
- return client;
+ return builder
+ .overrideConfiguration(createClientOverrideConfiguration(parameters, conf))
+ .credentialsProvider(parameters.getCredentialSet())
+ .region(region)
+ .serviceConfiguration(serviceConfiguration);
}
/**
- * A method to configure basic AmazonS3Builder parameters.
- *
- * @param builder Instance of AmazonS3Builder used.
- * @param awsConf ClientConfiguration used.
- * @param parameters Parameters used to set in the builder.
+ * Create an override configuration for an S3 client.
+ * @param parameters parameter object
+ * @param conf configuration object
+ * @throws IOException any IOE raised, or translated exception
+ * @return the override configuration
*/
- private void configureBasicParams(AmazonS3Builder builder,
- ClientConfiguration awsConf, S3ClientCreationParameters parameters) {
- builder.withCredentials(parameters.getCredentialSet());
- builder.withClientConfiguration(awsConf);
- builder.withPathStyleAccessEnabled(parameters.isPathStyleAccess());
+ protected ClientOverrideConfiguration createClientOverrideConfiguration(
+ S3ClientCreationParameters parameters, Configuration conf) throws IOException {
+ final ClientOverrideConfiguration.Builder clientOverrideConfigBuilder =
+ AWSClientConfig.createClientConfigBuilder(conf, AWS_SERVICE_IDENTIFIER_S3);
- if (parameters.getMetrics() != null) {
- builder.withMetricsCollector(
- new AwsStatisticsCollector(parameters.getMetrics()));
- }
- if (parameters.getRequestHandlers() != null) {
- builder.withRequestHandlers(
- parameters.getRequestHandlers().toArray(new RequestHandler2[0]));
- }
- if (parameters.getMonitoringListener() != null) {
- builder.withMonitoringListener(parameters.getMonitoringListener());
- }
+ // add any headers
+ parameters.getHeaders().forEach((h, v) -> clientOverrideConfigBuilder.putHeader(h, v));
- }
+ if (parameters.isRequesterPays()) {
+ // All calls must acknowledge requester will pay via header.
+ clientOverrideConfigBuilder.putHeader(REQUESTER_PAYS_HEADER, REQUESTER_PAYS_HEADER_VALUE);
+ }
- /**
- * A method to configure endpoint and Region for an AmazonS3Builder.
- *
- * @param builder Instance of AmazonS3Builder used.
- * @param epr EndpointConfiguration used to set in builder.
- */
- private void configureEndpoint(
- AmazonS3Builder builder,
- AmazonS3Builder.EndpointConfiguration epr) {
- if (epr != null) {
- // an endpoint binding was constructed: use it.
- builder.withEndpointConfiguration(epr);
- } else {
- // no idea what the endpoint is, so tell the SDK
- // to work it out at the cost of an extra HEAD request
- builder.withForceGlobalBucketAccessEnabled(true);
- // HADOOP-17771 force set the region so the build process doesn't halt.
- String region = getConf().getTrimmed(AWS_REGION, AWS_S3_CENTRAL_REGION);
- LOG.debug("fs.s3a.endpoint.region=\"{}\"", region);
- if (!region.isEmpty()) {
- // there's either an explicit region or we have fallen back
- // to the central one.
- LOG.debug("Using default endpoint; setting region to {}", region);
- builder.setRegion(region);
- } else {
- // no region.
- // allow this if people really want it; it is OK to rely on this
- // when deployed in EC2.
- WARN_OF_DEFAULT_REGION_CHAIN.warn(SDK_REGION_CHAIN_IN_USE);
- LOG.debug(SDK_REGION_CHAIN_IN_USE);
- }
+ if (!StringUtils.isEmpty(parameters.getUserAgentSuffix())) {
+ clientOverrideConfigBuilder.putAdvancedOption(SdkAdvancedClientOption.USER_AGENT_SUFFIX,
+ parameters.getUserAgentSuffix());
}
- }
- /**
- * Configure classic S3 client.
- * <p>
- * This includes: endpoint, Path Access and possibly other
- * options.
- *
- * @param s3 S3 Client.
- * @param endPoint s3 endpoint, may be empty
- * @param pathStyleAccess enable path style access?
- * @return S3 client
- * @throws IllegalArgumentException if misconfigured
- */
- protected static AmazonS3 configureAmazonS3Client(AmazonS3 s3,
- final String endPoint,
- final boolean pathStyleAccess)
- throws IllegalArgumentException {
- if (!endPoint.isEmpty()) {
- try {
- s3.setEndpoint(endPoint);
- } catch (IllegalArgumentException e) {
- String msg = "Incorrect endpoint: " + e.getMessage();
- LOG.error(msg);
- throw new IllegalArgumentException(msg, e);
+ if (parameters.getExecutionInterceptors() != null) {
+ for (ExecutionInterceptor interceptor : parameters.getExecutionInterceptors()) {
+ clientOverrideConfigBuilder.addExecutionInterceptor(interceptor);
}
}
- if (pathStyleAccess) {
- LOG.debug("Enabling path style access!");
- s3.setS3ClientOptions(S3ClientOptions.builder()
- .setPathStyleAccess(true)
- .build());
+
+ if (parameters.getMetrics() != null) {
+ clientOverrideConfigBuilder.addMetricPublisher(
+ new AwsStatisticsCollector(parameters.getMetrics()));
}
- return s3;
+
+ final RetryPolicy.Builder retryPolicyBuilder = AWSClientConfig.createRetryPolicyBuilder(conf);
+ clientOverrideConfigBuilder.retryPolicy(retryPolicyBuilder.build());
+
+ return clientOverrideConfigBuilder.build();
}
/**
- * Given an endpoint string, return an endpoint config, or null, if none
- * is needed.
- * <p>
- * This is a pretty painful piece of code. It is trying to replicate
- * what AwsClient.setEndpoint() does, because you can't
- * call that setter on an AwsClient constructed via
- * the builder, and you can't pass a metrics collector
- * down except through the builder.
- * <p>
- * Note also that AWS signing is a mystery which nobody fully
- * understands, especially given all problems surface in a
- * "400 bad request" response, which, like all security systems,
- * provides minimal diagnostics out of fear of leaking
- * secrets.
+ * Given a endpoint string, create the endpoint URI.
*
* @param endpoint possibly null endpoint.
- * @param awsConf config to build the URI from.
- * @param awsRegion AWS S3 Region if the corresponding config is set.
- * @return a configuration for the S3 client builder.
+ * @param conf config to build the URI from.
+ * @return an endpoint uri
*/
- @VisibleForTesting
- public static AwsClientBuilder.EndpointConfiguration
- createEndpointConfiguration(
- final String endpoint, final ClientConfiguration awsConf,
- String awsRegion) {
- LOG.debug("Creating endpoint configuration for \"{}\"", endpoint);
+ private static URI getS3Endpoint(String endpoint, final Configuration conf) {
+
+ boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS);
+
+ String protocol = secureConnections ? "https" : "http";
+
if (endpoint == null || endpoint.isEmpty()) {
- // the default endpoint...we should be using null at this point.
- LOG.debug("Using default endpoint -no need to generate a configuration");
+ // don't set an endpoint if none is configured, instead let the SDK figure it out.
return null;
}
- final URI epr = RuntimeHttpUtils.toUri(endpoint, awsConf);
- LOG.debug("Endpoint URI = {}", epr);
- String region = awsRegion;
- if (StringUtils.isBlank(region)) {
- if (!ServiceUtils.isS3USStandardEndpoint(endpoint)) {
- LOG.debug("Endpoint {} is not the default; parsing", epr);
- region = AwsHostNameUtils.parseRegion(
- epr.getHost(),
- S3_SERVICE_NAME);
- } else {
- // US-east, set region == null.
- LOG.debug("Endpoint {} is the standard one; declare region as null",
- epr);
- region = null;
- }
+ if (!endpoint.contains("://")) {
+ endpoint = String.format("%s://%s", protocol, endpoint);
+ }
+
+ try {
+ return new URI(endpoint);
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException(e);
}
- LOG.debug("Region for endpoint {}, URI {} is determined as {}",
- endpoint, epr, region);
- return new AwsClientBuilder.EndpointConfiguration(endpoint, region);
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/FailureInjectionPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/FailureInjectionPolicy.java
index f3e00cc61d76..f5c9c43a49cb 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/FailureInjectionPolicy.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/FailureInjectionPolicy.java
@@ -36,7 +36,7 @@ public class FailureInjectionPolicy {
public static final String DEFAULT_DELAY_KEY_SUBSTRING = "DELAY_LISTING_ME";
private static final Logger LOG =
- LoggerFactory.getLogger(InconsistentAmazonS3Client.class);
+ LoggerFactory.getLogger(FailureInjectionPolicy.class);
/**
* Probability of throttling a request.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java
deleted file mode 100644
index 317f2c785763..000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java
+++ /dev/null
@@ -1,345 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import java.util.List;
-import java.util.concurrent.atomic.AtomicLong;
-
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.AmazonServiceException;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.SdkClientException;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
-import com.amazonaws.services.s3.model.DeleteObjectRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsResult;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
-import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ListObjectsV2Request;
-import com.amazonaws.services.s3.model.ListObjectsV2Result;
-import com.amazonaws.services.s3.model.MultipartUploadListing;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.PutObjectResult;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.amazonaws.services.s3.model.UploadPartRequest;
-import com.amazonaws.services.s3.model.UploadPartResult;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * A wrapper around {@link com.amazonaws.services.s3.AmazonS3} that injects
- * failures.
- * It used to also inject inconsistency, but this was removed with S3Guard;
- * what is retained is the ability to throttle AWS operations and for the
- * input stream to be inconsistent.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class InconsistentAmazonS3Client extends AmazonS3Client {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(InconsistentAmazonS3Client.class);
-
- private FailureInjectionPolicy policy;
-
- /**
- * Counter of failures since last reset.
- */
- private final AtomicLong failureCounter = new AtomicLong(0);
-
-
- /**
- * Instantiate.
- * This subclasses a deprecated constructor of the parent
- * {@code AmazonS3Client} class; we can't use the builder API because,
- * that only creates the consistent client.
- * @param credentials credentials to auth.
- * @param clientConfiguration connection settings
- * @param conf hadoop configuration.
- */
- @SuppressWarnings("deprecation")
- public InconsistentAmazonS3Client(AWSCredentialsProvider credentials,
- ClientConfiguration clientConfiguration, Configuration conf) {
- super(credentials, clientConfiguration);
- policy = new FailureInjectionPolicy(conf);
- }
-
- /**
- * A way for tests to patch in a different fault injection policy at runtime.
- * @param fs filesystem under test
- * @param policy failure injection settings to set
- * @throws Exception on failure
- */
- public static void setFailureInjectionPolicy(S3AFileSystem fs,
- FailureInjectionPolicy policy) throws Exception {
- AmazonS3 s3 = fs.getAmazonS3ClientForTesting("s3guard");
- InconsistentAmazonS3Client ic = InconsistentAmazonS3Client.castFrom(s3);
- ic.replacePolicy(policy);
- }
-
- private void replacePolicy(FailureInjectionPolicy pol) {
- this.policy = pol;
- }
-
- @Override
- public String toString() {
- return String.format("Inconsistent S3 Client: %s; failure count %d",
- policy, failureCounter.get());
- }
-
- /**
- * Convenience function for test code to cast from supertype.
- * @param c supertype to cast from
- * @return subtype, not null
- * @throws Exception on error
- */
- public static InconsistentAmazonS3Client castFrom(AmazonS3 c) throws
- Exception {
- InconsistentAmazonS3Client ic = null;
- if (c instanceof InconsistentAmazonS3Client) {
- ic = (InconsistentAmazonS3Client) c;
- }
- Preconditions.checkNotNull(ic, "Not an instance of " +
- "InconsistentAmazonS3Client");
- return ic;
- }
-
- @Override
- public DeleteObjectsResult deleteObjects(DeleteObjectsRequest
- deleteObjectsRequest)
- throws AmazonClientException, AmazonServiceException {
- maybeFail();
- return super.deleteObjects(deleteObjectsRequest);
- }
-
- @Override
- public void deleteObject(DeleteObjectRequest deleteObjectRequest)
- throws AmazonClientException, AmazonServiceException {
- String key = deleteObjectRequest.getKey();
- LOG.debug("key {}", key);
- maybeFail();
- super.deleteObject(deleteObjectRequest);
- }
-
- /* We should only need to override this version of putObject() */
- @Override
- public PutObjectResult putObject(PutObjectRequest putObjectRequest)
- throws AmazonClientException, AmazonServiceException {
- LOG.debug("key {}", putObjectRequest.getKey());
- maybeFail();
- return super.putObject(putObjectRequest);
- }
-
- /* We should only need to override these versions of listObjects() */
- @Override
- public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
- throws AmazonClientException, AmazonServiceException {
- maybeFail();
- return super.listObjects(listObjectsRequest);
- }
-
- /* consistent listing with possibility of failing. */
- @Override
- public ListObjectsV2Result listObjectsV2(ListObjectsV2Request request)
- throws AmazonClientException, AmazonServiceException {
- maybeFail();
- return super.listObjectsV2(request);
- }
-
-
- @Override
- public CompleteMultipartUploadResult completeMultipartUpload(
- CompleteMultipartUploadRequest completeMultipartUploadRequest)
- throws SdkClientException, AmazonServiceException {
- maybeFail();
- return super.completeMultipartUpload(completeMultipartUploadRequest);
- }
-
- @Override
- public UploadPartResult uploadPart(UploadPartRequest uploadPartRequest)
- throws SdkClientException, AmazonServiceException {
- maybeFail();
- return super.uploadPart(uploadPartRequest);
- }
-
- @Override
- public InitiateMultipartUploadResult initiateMultipartUpload(
- InitiateMultipartUploadRequest initiateMultipartUploadRequest)
- throws SdkClientException, AmazonServiceException {
- maybeFail();
- return super.initiateMultipartUpload(initiateMultipartUploadRequest);
- }
-
- @Override
- public MultipartUploadListing listMultipartUploads(
- ListMultipartUploadsRequest listMultipartUploadsRequest)
- throws SdkClientException, AmazonServiceException {
- maybeFail();
- return super.listMultipartUploads(listMultipartUploadsRequest);
- }
-
- /**
- * Set the probability of throttling a request.
- * @param throttleProbability the probability of a request being throttled.
- */
- public void setThrottleProbability(float throttleProbability) {
- policy.setThrottleProbability(throttleProbability);
- }
-
- /**
- * Conditionally fail the operation.
- * @param errorMsg description of failure
- * @param statusCode http status code for error
- * @throws AmazonClientException if the client chooses to fail
- * the request.
- */
- private void maybeFail(String errorMsg, int statusCode)
- throws AmazonClientException {
- // code structure here is to line up for more failures later
- AmazonServiceException ex = null;
- if (FailureInjectionPolicy.trueWithProbability(policy.getThrottleProbability())) {
- // throttle the request
- ex = new AmazonServiceException(errorMsg
- + " count = " + (failureCounter.get() + 1), null);
- ex.setStatusCode(statusCode);
- }
-
- int failureLimit = policy.getFailureLimit();
- if (ex != null) {
- long count = failureCounter.incrementAndGet();
- if (failureLimit == 0
- || (failureLimit > 0 && count < failureLimit)) {
- throw ex;
- }
- }
- }
-
- private void maybeFail() {
- maybeFail("throttled", 503);
- }
-
- /**
- * Set the limit on failures before all operations pass through.
- * This resets the failure count.
- * @param limit limit; "0" means "no limit"
- */
- public void setFailureLimit(int limit) {
- policy.setFailureLimit(limit);
- failureCounter.set(0);
- }
-
- @Override
- public S3Object getObject(GetObjectRequest var1) throws SdkClientException,
- AmazonServiceException {
- maybeFail();
- return super.getObject(var1);
- }
-
- @Override
- public S3Object getObject(String bucketName, String key)
- throws SdkClientException, AmazonServiceException {
- maybeFail();
- return super.getObject(bucketName, key);
-
- }
-
- /** Since ObjectListing is immutable, we just override it with wrapper. */
- @SuppressWarnings("serial")
- private static class CustomObjectListing extends ObjectListing {
-
- private final List<S3ObjectSummary> customListing;
- private final List<String> customPrefixes;
-
- CustomObjectListing(ObjectListing rawListing,
- List<S3ObjectSummary> customListing,
- List<String> customPrefixes) {
- super();
- this.customListing = customListing;
- this.customPrefixes = customPrefixes;
-
- this.setBucketName(rawListing.getBucketName());
- this.setCommonPrefixes(rawListing.getCommonPrefixes());
- this.setDelimiter(rawListing.getDelimiter());
- this.setEncodingType(rawListing.getEncodingType());
- this.setMarker(rawListing.getMarker());
- this.setMaxKeys(rawListing.getMaxKeys());
- this.setNextMarker(rawListing.getNextMarker());
- this.setPrefix(rawListing.getPrefix());
- this.setTruncated(rawListing.isTruncated());
- }
-
- @Override
- public List<S3ObjectSummary> getObjectSummaries() {
- return customListing;
- }
-
- @Override
- public List<String> getCommonPrefixes() {
- return customPrefixes;
- }
- }
-
- @SuppressWarnings("serial")
- private static class CustomListObjectsV2Result extends ListObjectsV2Result {
-
- private final List<S3ObjectSummary> customListing;
- private final List<String> customPrefixes;
-
- CustomListObjectsV2Result(ListObjectsV2Result raw,
- List<S3ObjectSummary> customListing, List<String> customPrefixes) {
- super();
- this.customListing = customListing;
- this.customPrefixes = customPrefixes;
-
- this.setBucketName(raw.getBucketName());
- this.setCommonPrefixes(raw.getCommonPrefixes());
- this.setDelimiter(raw.getDelimiter());
- this.setEncodingType(raw.getEncodingType());
- this.setStartAfter(raw.getStartAfter());
- this.setMaxKeys(raw.getMaxKeys());
- this.setContinuationToken(raw.getContinuationToken());
- this.setPrefix(raw.getPrefix());
- this.setTruncated(raw.isTruncated());
- }
-
- @Override
- public List<S3ObjectSummary> getObjectSummaries() {
- return customListing;
- }
-
- @Override
- public List<String> getCommonPrefixes() {
- return customPrefixes;
- }
- }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentS3ClientFactory.java
deleted file mode 100644
index 4bfcc8aba3af..000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentS3ClientFactory.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.services.s3.AmazonS3;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * S3 Client factory used for testing with eventual consistency fault injection.
- * This client is for testing <i>only</i>; it is in the production
- * {@code hadoop-aws} module to enable integration tests to use this
- * just by editing the Hadoop configuration used to bring up the client.
- *
- * The factory uses the older constructor-based instantiation/configuration
- * of the client, so does not wire up metrics, handlers etc.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class InconsistentS3ClientFactory extends DefaultS3ClientFactory {
-
- @Override
- protected AmazonS3 buildAmazonS3Client(
- final ClientConfiguration awsConf,
- final S3ClientCreationParameters parameters) {
- LOG.warn("** FAILURE INJECTION ENABLED. Do not run in production! **");
- LOG.warn("List inconsistency is no longer emulated; only throttling and read errors");
- InconsistentAmazonS3Client s3
- = new InconsistentAmazonS3Client(
- parameters.getCredentialSet(), awsConf, getConf());
- configureAmazonS3Client(s3,
- parameters.getEndpoint(),
- parameters.isPathStyleAccess());
- return s3;
- }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java
index 67c8e7d809cc..9b2c95a90c76 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java
@@ -24,8 +24,7 @@ import java.util.Optional;
import java.util.concurrent.Future;
import javax.annotation.Nullable;
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.SdkBaseException;
+import software.amazon.awssdk.core.exception.SdkException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,6 +38,7 @@ import org.apache.hadoop.util.functional.FutureIO;
import org.apache.hadoop.util.functional.InvocationRaisingIOE;
import org.apache.hadoop.util.Preconditions;
+
import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.invokeTrackingDuration;
/**
@@ -120,7 +120,7 @@ public class Invoker {
throws IOException {
try (DurationInfo ignored = new DurationInfo(LOG, false, "%s", action)) {
return operation.apply();
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw S3AUtils.translateException(action, path, e);
}
}
@@ -145,7 +145,7 @@ public class Invoker {
throws IOException {
try {
return invokeTrackingDuration(tracker, operation);
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw S3AUtils.translateException(action, path, e);
}
}
@@ -170,7 +170,7 @@ public class Invoker {
/**
*
- * Wait for a future, translating AmazonClientException into an IOException.
+ * Wait for a future, translating SdkException into an IOException.
* @param action action to execute (used in error messages)
* @param path path of work (used in error messages)
* @param future future to await for
@@ -186,7 +186,7 @@ public class Invoker {
throws IOException {
try (DurationInfo ignored = new DurationInfo(LOG, false, "%s", action)) {
return FutureIO.awaitFuture(future);
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw S3AUtils.translateException(action, path, e);
}
}
@@ -444,7 +444,7 @@ public class Invoker {
* @param operation operation to execute
* @return the result of the call
* @throws IOException any IOE raised
- * @throws SdkBaseException any AWS exception raised
+ * @throws SdkException any AWS exception raised
* @throws RuntimeException : these are never caught and retries.
*/
@Retries.RetryRaw
@@ -466,7 +466,7 @@ public class Invoker {
}
// execute the operation, returning if successful
return operation.apply();
- } catch (IOException | SdkBaseException e) {
+ } catch (IOException | SdkException e) {
caught = e;
}
// you only get here if the operation didn't complete
@@ -479,7 +479,7 @@ public class Invoker {
translated = (IOException) caught;
} else {
translated = S3AUtils.translateException(text, "",
- (SdkBaseException)caught);
+ (SdkException) caught);
}
try {
@@ -518,11 +518,10 @@ public class Invoker {
if (caught instanceof IOException) {
throw (IOException) caught;
} else {
- throw (SdkBaseException) caught;
+ throw (SdkException) caught;
}
}
-
/**
* Execute an operation; any exception raised is simply caught and
* logged at debug.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
index 6c39cc4b6424..490deaaab04d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
@@ -18,7 +18,8 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
+import software.amazon.awssdk.services.s3.model.CommonPrefix;
+import software.amazon.awssdk.services.s3.model.S3Object;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.VisibleForTesting;
@@ -277,19 +278,19 @@ public class Listing extends AbstractStoreOperation {
}
/**
- * Interface to implement by the logic deciding whether to accept a summary
+ * Interface to implement the logic deciding whether to accept a s3Object
* entry or path as a valid file or directory.
*/
interface FileStatusAcceptor {
/**
- * Predicate to decide whether or not to accept a summary entry.
+ * Predicate to decide whether or not to accept a s3Object entry.
* @param keyPath qualified path to the entry
- * @param summary summary entry
+ * @param s3Object s3Object entry
* @return true if the entry is accepted (i.e. that a status entry
* should be generated.
*/
- boolean accept(Path keyPath, S3ObjectSummary summary);
+ boolean accept(Path keyPath, S3Object s3Object);
/**
* Predicate to decide whether or not to accept a prefix.
@@ -451,21 +452,21 @@ public class Listing extends AbstractStoreOperation {
int added = 0, ignored = 0;
// list to fill in with results. Initial size will be list maximum.
List<S3AFileStatus> stats = new ArrayList<>(
- objects.getObjectSummaries().size() +
+ objects.getS3Objects().size() +
objects.getCommonPrefixes().size());
// objects
- for (S3ObjectSummary summary : objects.getObjectSummaries()) {
- String key = summary.getKey();
+ for (S3Object s3Object : objects.getS3Objects()) {
+ String key = s3Object.key();
Path keyPath = getStoreContext().getContextAccessors().keyToPath(key);
if (LOG.isDebugEnabled()) {
- LOG.debug("{}: {}", keyPath, stringify(summary));
+ LOG.debug("{}: {}", keyPath, stringify(s3Object));
}
// Skip over keys that are ourselves and old S3N _$folder$ files
- if (acceptor.accept(keyPath, summary) && filter.accept(keyPath)) {
- S3AFileStatus status = createFileStatus(keyPath, summary,
+ if (acceptor.accept(keyPath, s3Object) && filter.accept(keyPath)) {
+ S3AFileStatus status = createFileStatus(keyPath, s3Object,
listingOperationCallbacks.getDefaultBlockSize(keyPath),
getStoreContext().getUsername(),
- summary.getETag(), null, isCSEEnabled);
+ s3Object.eTag(), null, isCSEEnabled);
LOG.debug("Adding: {}", status);
stats.add(status);
added++;
@@ -476,11 +477,11 @@ public class Listing extends AbstractStoreOperation {
}
// prefixes: always directories
- for (String prefix : objects.getCommonPrefixes()) {
+ for (CommonPrefix prefix : objects.getCommonPrefixes()) {
Path keyPath = getStoreContext()
.getContextAccessors()
- .keyToPath(prefix);
- if (acceptor.accept(keyPath, prefix) && filter.accept(keyPath)) {
+ .keyToPath(prefix.prefix());
+ if (acceptor.accept(keyPath, prefix.prefix()) && filter.accept(keyPath)) {
S3AFileStatus status = new S3AFileStatus(Tristate.FALSE, keyPath,
getStoreContext().getUsername());
LOG.debug("Adding directory: {}", status);
@@ -731,18 +732,18 @@ public class Listing extends AbstractStoreOperation {
}
/**
- * Reject a summary entry if the key path is the qualified Path, or
+ * Reject a s3Object entry if the key path is the qualified Path, or
* it ends with {@code "_$folder$"}.
* @param keyPath key path of the entry
- * @param summary summary entry
+ * @param s3Object s3Object entry
* @return true if the entry is accepted (i.e. that a status entry
* should be generated.
*/
@Override
- public boolean accept(Path keyPath, S3ObjectSummary summary) {
+ public boolean accept(Path keyPath, S3Object s3Object) {
return !keyPath.equals(qualifiedPath)
- && !summary.getKey().endsWith(S3N_FOLDER_SUFFIX)
- && !objectRepresentsDirectory(summary.getKey());
+ && !s3Object.key().endsWith(S3N_FOLDER_SUFFIX)
+ && !objectRepresentsDirectory(s3Object.key());
}
/**
@@ -767,8 +768,8 @@ public class Listing extends AbstractStoreOperation {
*/
static class AcceptAllButS3nDirs implements FileStatusAcceptor {
- public boolean accept(Path keyPath, S3ObjectSummary summary) {
- return !summary.getKey().endsWith(S3N_FOLDER_SUFFIX);
+ public boolean accept(Path keyPath, S3Object s3Object) {
+ return !s3Object.key().endsWith(S3N_FOLDER_SUFFIX);
}
public boolean accept(Path keyPath, String prefix) {
@@ -799,17 +800,17 @@ public class Listing extends AbstractStoreOperation {
}
/**
- * Reject a summary entry if the key path is the qualified Path, or
+ * Reject a s3Object entry if the key path is the qualified Path, or
* it ends with {@code "_$folder$"}.
* @param keyPath key path of the entry
- * @param summary summary entry
+ * @param s3Object s3Object entry
* @return true if the entry is accepted (i.e. that a status entry
* should be generated.)
*/
@Override
- public boolean accept(Path keyPath, S3ObjectSummary summary) {
+ public boolean accept(Path keyPath, S3Object s3Object) {
return !keyPath.equals(qualifiedPath) &&
- !summary.getKey().endsWith(S3N_FOLDER_SUFFIX);
+ !s3Object.key().endsWith(S3N_FOLDER_SUFFIX);
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java
index d8c820cd8a12..296ec18dcf18 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java
@@ -23,13 +23,14 @@ import java.util.ListIterator;
import java.util.NoSuchElementException;
import javax.annotation.Nullable;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
-import com.amazonaws.services.s3.model.MultipartUpload;
-import com.amazonaws.services.s3.model.MultipartUploadListing;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest;
+import software.amazon.awssdk.services.s3.model.ListMultipartUploadsResponse;
+import software.amazon.awssdk.services.s3.model.MultipartUpload;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.s3a.api.RequestFactory;
import org.apache.hadoop.fs.s3a.impl.StoreContext;
@@ -43,7 +44,7 @@ import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDura
* MultipartUtils upload-specific functions for use by S3AFileSystem and Hadoop
* CLI.
* The Audit span active when
- * {@link #listMultipartUploads(StoreContext, AmazonS3, String, int)}
+ * {@link #listMultipartUploads(StoreContext, S3Client, String, int)}
* was invoked is retained for all subsequent operations.
*/
public final class MultipartUtils {
@@ -67,7 +68,7 @@ public final class MultipartUtils {
*/
static MultipartUtils.UploadIterator listMultipartUploads(
final StoreContext storeContext,
- AmazonS3 s3,
+ S3Client s3,
@Nullable String prefix,
int maxKeys)
throws IOException {
@@ -84,14 +85,14 @@ public final class MultipartUtils {
* at the time the iterator was constructed.
*/
static class ListingIterator implements
- RemoteIterator<MultipartUploadListing> {
+ RemoteIterator<ListMultipartUploadsResponse> {
private final String prefix;
private final RequestFactory requestFactory;
private final int maxKeys;
- private final AmazonS3 s3;
+ private final S3Client s3;
private final Invoker invoker;
private final AuditSpan auditSpan;
@@ -101,7 +102,7 @@ public final class MultipartUtils {
/**
* Most recent listing results.
*/
- private MultipartUploadListing listing;
+ private ListMultipartUploadsResponse listing;
/**
* Indicator that this is the first listing.
@@ -114,7 +115,7 @@ public final class MultipartUtils {
private int listCount = 0;
ListingIterator(final StoreContext storeContext,
- AmazonS3 s3,
+ S3Client s3,
@Nullable String prefix,
int maxKeys) throws IOException {
this.storeContext = storeContext;
@@ -153,7 +154,7 @@ public final class MultipartUtils {
*/
@Override
@Retries.RetryTranslated
- public MultipartUploadListing next() throws IOException {
+ public ListMultipartUploadsResponse next() throws IOException {
if (firstListing) {
firstListing = false;
} else {
@@ -171,32 +172,34 @@ public final class MultipartUtils {
public String toString() {
return "Upload iterator: prefix " + prefix
+ "; list count " + listCount
- + "; upload count " + listing.getMultipartUploads().size()
+ + "; upload count " + listing.uploads().size()
+ "; isTruncated=" + listing.isTruncated();
}
@Retries.RetryTranslated
private void requestNextBatch() throws IOException {
try (AuditSpan span = auditSpan.activate()) {
- ListMultipartUploadsRequest req = requestFactory
- .newListMultipartUploadsRequest(prefix);
+ ListMultipartUploadsRequest.Builder requestBuilder = requestFactory
+ .newListMultipartUploadsRequestBuilder(prefix);
if (!firstListing) {
- req.setKeyMarker(listing.getNextKeyMarker());
- req.setUploadIdMarker(listing.getNextUploadIdMarker());
+ requestBuilder.keyMarker(listing.nextKeyMarker());
+ requestBuilder.uploadIdMarker(listing.nextUploadIdMarker());
}
- req.setMaxUploads(maxKeys);
+ requestBuilder.maxUploads(maxKeys);
+
+ ListMultipartUploadsRequest request = requestBuilder.build();
LOG.debug("[{}], Requesting next {} uploads prefix {}, " +
"next key {}, next upload id {}", listCount, maxKeys, prefix,
- req.getKeyMarker(), req.getUploadIdMarker());
+ request.keyMarker(), request.uploadIdMarker());
listCount++;
listing = invoker.retry("listMultipartUploads", prefix, true,
trackDurationOfOperation(storeContext.getInstrumentation(),
MULTIPART_UPLOAD_LIST.getSymbol(),
- () -> s3.listMultipartUploads(req)));
+ () -> s3.listMultipartUploads(requestBuilder.build())));
LOG.debug("Listing found {} upload(s)",
- listing.getMultipartUploads().size());
+ listing.uploads().size());
LOG.debug("New listing state: {}", this);
}
}
@@ -216,14 +219,14 @@ public final class MultipartUtils {
*/
private ListingIterator lister;
/** Current listing: the last upload listing we fetched. */
- private MultipartUploadListing listing;
+ private ListMultipartUploadsResponse listing;
/** Iterator over the current listing. */
private ListIterator<MultipartUpload> batchIterator;
@Retries.RetryTranslated
public UploadIterator(
final StoreContext storeContext,
- AmazonS3 s3,
+ S3Client s3,
int maxKeys,
@Nullable String prefix)
throws IOException {
@@ -249,7 +252,7 @@ public final class MultipartUtils {
private boolean requestNextBatch() throws IOException {
if (lister.hasNext()) {
listing = lister.next();
- batchIterator = listing.getMultipartUploads().listIterator();
+ batchIterator = listing.uploads().listIterator();
return batchIterator.hasNext();
}
return false;
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProgressableProgressListener.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProgressableProgressListener.java
index 0ce022aa8858..7ee6c55c191b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProgressableProgressListener.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProgressableProgressListener.java
@@ -18,59 +18,55 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.event.ProgressEvent;
-import com.amazonaws.event.ProgressEventType;
-import com.amazonaws.event.ProgressListener;
-import com.amazonaws.services.s3.transfer.Upload;
+import software.amazon.awssdk.transfer.s3.model.ObjectTransfer;
+import software.amazon.awssdk.transfer.s3.progress.TransferListener;
import org.apache.hadoop.util.Progressable;
import org.slf4j.Logger;
-import static com.amazonaws.event.ProgressEventType.TRANSFER_COMPLETED_EVENT;
-import static com.amazonaws.event.ProgressEventType.TRANSFER_PART_STARTED_EVENT;
/**
* Listener to progress from AWS regarding transfers.
*/
-public class ProgressableProgressListener implements ProgressListener {
+public class ProgressableProgressListener implements TransferListener {
private static final Logger LOG = S3AFileSystem.LOG;
private final S3AFileSystem fs;
private final String key;
private final Progressable progress;
private long lastBytesTransferred;
- private final Upload upload;
/**
* Instantiate.
* @param fs filesystem: will be invoked with statistics updates
* @param key key for the upload
- * @param upload source of events
* @param progress optional callback for progress.
*/
public ProgressableProgressListener(S3AFileSystem fs,
String key,
- Upload upload,
Progressable progress) {
this.fs = fs;
this.key = key;
- this.upload = upload;
this.progress = progress;
this.lastBytesTransferred = 0;
}
@Override
- public void progressChanged(ProgressEvent progressEvent) {
- if (progress != null) {
- progress.progress();
- }
+ public void transferInitiated(TransferListener.Context.TransferInitiated context) {
+ fs.incrementWriteOperations();
+ }
+
+ @Override
+ public void transferComplete(TransferListener.Context.TransferComplete context) {
+ fs.incrementWriteOperations();
+ }
+
+ @Override
+ public void bytesTransferred(TransferListener.Context.BytesTransferred context) {
- // There are 3 http ops here, but this should be close enough for now
- ProgressEventType pet = progressEvent.getEventType();
- if (pet == TRANSFER_PART_STARTED_EVENT ||
- pet == TRANSFER_COMPLETED_EVENT) {
- fs.incrementWriteOperations();
+ if(progress != null) {
+ progress.progress();
}
- long transferred = upload.getProgress().getBytesTransferred();
+ long transferred = context.progressSnapshot().transferredBytes();
long delta = transferred - lastBytesTransferred;
fs.incrementPutProgressStatistics(key, delta);
lastBytesTransferred = transferred;
@@ -79,11 +75,13 @@ public class ProgressableProgressListener implements ProgressListener {
/**
* Method to invoke after upload has completed.
* This can handle race conditions in setup/teardown.
+ * @param upload upload which has just completed.
* @return the number of bytes which were transferred after the notification
*/
- public long uploadCompleted() {
- long delta = upload.getProgress().getBytesTransferred() -
- lastBytesTransferred;
+ public long uploadCompleted(ObjectTransfer upload) {
+
+ long delta =
+ upload.progress().snapshot().transferredBytes() - lastBytesTransferred;
if (delta > 0) {
LOG.debug("S3A write delta changed after finished: {} bytes", delta);
fs.incrementPutProgressStatistics(key, delta);
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
index 43a2b7e0dbd5..de0f59154e99 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
@@ -31,15 +31,16 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
-import com.amazonaws.SdkBaseException;
-import com.amazonaws.event.ProgressEvent;
-import com.amazonaws.event.ProgressEventType;
-import com.amazonaws.event.ProgressListener;
-import com.amazonaws.services.s3.model.PartETag;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.PutObjectResult;
-import com.amazonaws.services.s3.model.UploadPartRequest;
-
+import software.amazon.awssdk.core.exception.SdkException;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.model.CompletedPart;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartResponse;
+
+import org.apache.hadoop.fs.s3a.impl.ProgressListener;
+import org.apache.hadoop.fs.s3a.impl.ProgressListenerEvent;
import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
import org.apache.hadoop.fs.statistics.IOStatisticsAggregator;
import org.apache.hadoop.util.Preconditions;
@@ -69,6 +70,7 @@ import org.apache.hadoop.util.Progressable;
import static java.util.Objects.requireNonNull;
import static org.apache.hadoop.fs.s3a.S3AUtils.*;
import static org.apache.hadoop.fs.s3a.Statistic.*;
+import static org.apache.hadoop.fs.s3a.impl.ProgressListenerEvent.*;
import static org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext.EMPTY_BLOCK_OUTPUT_STREAM_STATISTICS;
import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDuration;
import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfInvocation;
@@ -192,7 +194,7 @@ class S3ABlockOutputStream extends OutputStream implements
this.executorService = MoreExecutors.listeningDecorator(
builder.executorService);
this.multiPartUpload = null;
- final Progressable progress = builder.progress;
+ Progressable progress = builder.progress;
this.progressListener = (progress instanceof ProgressListener) ?
(ProgressListener) progress
: new ProgressableListener(progress);
@@ -439,7 +441,7 @@ class S3ABlockOutputStream extends OutputStream implements
uploadCurrentBlock(true);
}
// wait for the partial uploads to finish
- final List<PartETag> partETags =
+ final List<CompletedPart> partETags =
multiPartUpload.waitForAllPartUploads();
bytes = bytesSubmitted;
@@ -597,27 +599,28 @@ class S3ABlockOutputStream extends OutputStream implements
final PutObjectRequest putObjectRequest = uploadData.hasFile() ?
writeOperationHelper.createPutObjectRequest(
key,
- uploadData.getFile(),
- builder.putOptions)
+ uploadData.getFile().length(),
+ builder.putOptions,
+ true)
: writeOperationHelper.createPutObjectRequest(
key,
- uploadData.getUploadStream(),
size,
- builder.putOptions);
- BlockUploadProgress callback =
- new BlockUploadProgress(
- block, progressListener, now());
- putObjectRequest.setGeneralProgressListener(callback);
+ builder.putOptions,
+ false);
+
+ BlockUploadProgress progressCallback =
+ new BlockUploadProgress(block, progressListener, now());
statistics.blockUploadQueued(size);
- ListenableFuture<PutObjectResult> putObjectResult =
+ ListenableFuture<PutObjectResponse> putObjectResult =
executorService.submit(() -> {
try {
// the putObject call automatically closes the input
// stream afterwards.
- return writeOperationHelper.putObject(
- putObjectRequest,
- builder.putOptions,
- statistics);
+ PutObjectResponse response =
+ writeOperationHelper.putObject(putObjectRequest, builder.putOptions, uploadData,
+ uploadData.hasFile(), statistics);
+ progressCallback.progressChanged(REQUEST_BYTE_TRANSFER_EVENT);
+ return response;
} finally {
cleanupWithLogger(LOG, uploadData, block);
}
@@ -761,7 +764,7 @@ class S3ABlockOutputStream extends OutputStream implements
*/
private class MultiPartUpload {
private final String uploadId;
- private final List<ListenableFuture<PartETag>> partETagsFutures;
+ private final List<ListenableFuture<CompletedPart>> partETagsFutures;
private int partsSubmitted;
private int partsUploaded;
private long bytesSubmitted;
@@ -866,18 +869,19 @@ class S3ABlockOutputStream extends OutputStream implements
final int currentPartNumber = partETagsFutures.size() + 1;
final UploadPartRequest request;
final S3ADataBlocks.BlockUploadData uploadData;
+ final RequestBody requestBody;
try {
uploadData = block.startUpload();
- request = writeOperationHelper.newUploadPartRequest(
+ requestBody = uploadData.hasFile()
+ ? RequestBody.fromFile(uploadData.getFile())
+ : RequestBody.fromInputStream(uploadData.getUploadStream(), size);
+
+ request = writeOperationHelper.newUploadPartRequestBuilder(
key,
uploadId,
currentPartNumber,
- size,
- uploadData.getUploadStream(),
- uploadData.getFile(),
- 0L);
- request.setLastPart(isLast);
- } catch (SdkBaseException aws) {
+ size).build();
+ } catch (SdkException aws) {
// catch and translate
IOException e = translateException("upload", key, aws);
// failure to start the upload.
@@ -888,28 +892,38 @@ class S3ABlockOutputStream extends OutputStream implements
noteUploadFailure(e);
throw e;
}
- BlockUploadProgress callback =
- new BlockUploadProgress(
- block, progressListener, now());
- request.setGeneralProgressListener(callback);
+
+ BlockUploadProgress progressCallback =
+ new BlockUploadProgress(block, progressListener, now());
+
statistics.blockUploadQueued(block.dataSize());
- ListenableFuture<PartETag> partETagFuture =
+ ListenableFuture<CompletedPart> partETagFuture =
executorService.submit(() -> {
// this is the queued upload operation
// do the upload
try {
LOG.debug("Uploading part {} for id '{}'",
currentPartNumber, uploadId);
- PartETag partETag = writeOperationHelper.uploadPart(request, statistics)
- .getPartETag();
+
+ progressCallback.progressChanged(TRANSFER_PART_STARTED_EVENT);
+
+ UploadPartResponse response = writeOperationHelper
+ .uploadPart(request, requestBody, statistics);
LOG.debug("Completed upload of {} to part {}",
- block, partETag.getETag());
+ block, response.eTag());
LOG.debug("Stream statistics of {}", statistics);
partsUploaded++;
- return partETag;
+
+ progressCallback.progressChanged(TRANSFER_PART_COMPLETED_EVENT);
+
+ return CompletedPart.builder()
+ .eTag(response.eTag())
+ .partNumber(currentPartNumber)
+ .build();
} catch (IOException e) {
// save immediately.
noteUploadFailure(e);
+ progressCallback.progressChanged(TRANSFER_PART_FAILED_EVENT);
throw e;
} finally {
// close the stream and block
@@ -924,7 +938,7 @@ class S3ABlockOutputStream extends OutputStream implements
* @return list of results
* @throws IOException IO Problems
*/
- private List<PartETag> waitForAllPartUploads() throws IOException {
+ private List<CompletedPart> waitForAllPartUploads() throws IOException {
LOG.debug("Waiting for {} uploads to complete", partETagsFutures.size());
try {
return Futures.allAsList(partETagsFutures).get();
@@ -948,7 +962,7 @@ class S3ABlockOutputStream extends OutputStream implements
*/
private void cancelAllActiveFutures() {
LOG.debug("Cancelling futures");
- for (ListenableFuture<PartETag> future : partETagsFutures) {
+ for (ListenableFuture<CompletedPart> future : partETagsFutures) {
future.cancel(true);
}
}
@@ -960,7 +974,7 @@ class S3ABlockOutputStream extends OutputStream implements
* @param partETags list of partial uploads
* @throws IOException on any problem
*/
- private void complete(List<PartETag> partETags)
+ private void complete(List<CompletedPart> partETags)
throws IOException {
maybeRethrowUploadFailure();
AtomicInteger errorCount = new AtomicInteger(0);
@@ -1005,22 +1019,24 @@ class S3ABlockOutputStream extends OutputStream implements
}
}
+
/**
* The upload progress listener registered for events returned
* during the upload of a single block.
* It updates statistics and handles the end of the upload.
* Transfer failures are logged at WARN.
*/
- private final class BlockUploadProgress implements ProgressListener {
+ private final class BlockUploadProgress {
+
private final S3ADataBlocks.DataBlock block;
private final ProgressListener nextListener;
private final Instant transferQueueTime;
private Instant transferStartTime;
+ private long size;
/**
* Track the progress of a single block upload.
* @param block block to monitor
- * @param nextListener optional next progress listener
* @param transferQueueTime time the block was transferred
* into the queue
*/
@@ -1029,20 +1045,17 @@ class S3ABlockOutputStream extends OutputStream implements
Instant transferQueueTime) {
this.block = block;
this.transferQueueTime = transferQueueTime;
+ this.size = block.dataSize();
this.nextListener = nextListener;
}
- @Override
- public void progressChanged(ProgressEvent progressEvent) {
- ProgressEventType eventType = progressEvent.getEventType();
- long bytesTransferred = progressEvent.getBytesTransferred();
+ public void progressChanged(ProgressListenerEvent eventType) {
- long size = block.dataSize();
switch (eventType) {
case REQUEST_BYTE_TRANSFER_EVENT:
// bytes uploaded
- statistics.bytesTransferred(bytesTransferred);
+ statistics.bytesTransferred(size);
break;
case TRANSFER_PART_STARTED_EVENT:
@@ -1057,6 +1070,7 @@ class S3ABlockOutputStream extends OutputStream implements
statistics.blockUploadCompleted(
Duration.between(transferStartTime, now()),
size);
+ statistics.bytesTransferred(size);
break;
case TRANSFER_PART_FAILED_EVENT:
@@ -1071,13 +1085,13 @@ class S3ABlockOutputStream extends OutputStream implements
}
if (nextListener != null) {
- nextListener.progressChanged(progressEvent);
+ nextListener.progressChanged(eventType, size);
}
}
}
/**
- * Bridge from AWS {@code ProgressListener} to Hadoop {@link Progressable}.
+ * Bridge from {@link ProgressListener} to Hadoop {@link Progressable}.
*/
private static class ProgressableListener implements ProgressListener {
private final Progressable progress;
@@ -1086,7 +1100,7 @@ class S3ABlockOutputStream extends OutputStream implements
this.progress = progress;
}
- public void progressChanged(ProgressEvent progressEvent) {
+ public void progressChanged(ProgressListenerEvent eventType, int bytesTransferred) {
if (progress != null) {
progress.progress();
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
index ddaf059b055d..5aa43245c2f0 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
@@ -47,7 +47,7 @@ import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
* Set of classes to support output streaming into blocks which are then
* uploaded as to S3 as a single PUT, or as part of a multipart request.
*/
-final class S3ADataBlocks {
+public final class S3ADataBlocks {
private static final Logger LOG =
LoggerFactory.getLogger(S3ADataBlocks.class);
@@ -101,7 +101,7 @@ final class S3ADataBlocks {
* It can be one of a file or an input stream.
* When closed, any stream is closed. Any source file is untouched.
*/
- static final class BlockUploadData implements Closeable {
+ public static final class BlockUploadData implements Closeable {
private final File file;
private final InputStream uploadStream;
@@ -109,7 +109,7 @@ final class S3ADataBlocks {
* File constructor; input stream will be null.
* @param file file to upload
*/
- BlockUploadData(File file) {
+ public BlockUploadData(File file) {
Preconditions.checkArgument(file.exists(), "No file: " + file);
this.file = file;
this.uploadStream = null;
@@ -119,7 +119,7 @@ final class S3ADataBlocks {
* Stream constructor, file field will be null.
* @param uploadStream stream to upload
*/
- BlockUploadData(InputStream uploadStream) {
+ public BlockUploadData(InputStream uploadStream) {
Preconditions.checkNotNull(uploadStream, "rawUploadStream");
this.uploadStream = uploadStream;
this.file = null;
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 07a14f63aa41..d0a32148c9d5 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -21,17 +21,20 @@ package org.apache.hadoop.fs.s3a;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InterruptedIOException;
import java.io.UncheckedIOException;
import java.net.URI;
import java.nio.file.AccessDeniedException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
@@ -41,6 +44,7 @@ import java.util.Set;
import java.util.Objects;
import java.util.TreeSet;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
@@ -48,43 +52,53 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.Nullable;
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.AmazonServiceException;
-import com.amazonaws.SdkBaseException;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.Headers;
-import com.amazonaws.services.s3.model.CannedAccessControlList;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
-import com.amazonaws.services.s3.model.CopyObjectRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsResult;
-import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
-import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ListObjectsV2Request;
-import com.amazonaws.services.s3.model.MultiObjectDeleteException;
-import com.amazonaws.services.s3.model.MultipartUpload;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.PutObjectResult;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.SelectObjectContentRequest;
-import com.amazonaws.services.s3.model.SelectObjectContentResult;
-import com.amazonaws.services.s3.model.StorageClass;
-import com.amazonaws.services.s3.model.UploadPartRequest;
-import com.amazonaws.services.s3.model.UploadPartResult;
-import com.amazonaws.services.s3.transfer.Copy;
-import com.amazonaws.services.s3.transfer.TransferManager;
-import com.amazonaws.services.s3.transfer.TransferManagerConfiguration;
-import com.amazonaws.services.s3.transfer.Upload;
-import com.amazonaws.services.s3.transfer.model.CopyResult;
-import com.amazonaws.services.s3.transfer.model.UploadResult;
-import com.amazonaws.event.ProgressListener;
-
+import software.amazon.awssdk.core.ResponseInputStream;
+import software.amazon.awssdk.core.exception.SdkException;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse;
+import software.amazon.awssdk.services.s3.model.GetBucketLocationRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
+import software.amazon.awssdk.services.s3.model.HeadBucketResponse;
+import software.amazon.awssdk.services.s3.model.MultipartUpload;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
+import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
+import software.amazon.awssdk.services.s3.model.CopyObjectResponse;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
+import software.amazon.awssdk.services.s3.model.NoSuchBucketException;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
+import software.amazon.awssdk.services.s3.model.S3Error;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest;
+import software.amazon.awssdk.services.s3.model.SelectObjectContentResponseHandler;
+import software.amazon.awssdk.services.s3.model.StorageClass;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartResponse;
+import software.amazon.awssdk.transfer.s3.model.CompletedCopy;
+import software.amazon.awssdk.transfer.s3.model.CompletedFileUpload;
+import software.amazon.awssdk.transfer.s3.model.Copy;
+import software.amazon.awssdk.transfer.s3.S3TransferManager;
+import software.amazon.awssdk.transfer.s3.model.CopyRequest;
+import software.amazon.awssdk.transfer.s3.model.FileUpload;
+import software.amazon.awssdk.transfer.s3.model.UploadFileRequest;
+
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.impl.prefetch.ExecutorServiceFuturePool;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -108,11 +122,12 @@ import org.apache.hadoop.fs.s3a.audit.AuditSpanS3A;
import org.apache.hadoop.fs.s3a.auth.SignerManager;
import org.apache.hadoop.fs.s3a.auth.delegation.DelegationOperations;
import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider;
+import org.apache.hadoop.fs.s3a.impl.AWSCannedACL;
+import org.apache.hadoop.fs.s3a.impl.AWSHeaders;
import org.apache.hadoop.fs.s3a.impl.BulkDeleteRetryHandler;
import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy;
import org.apache.hadoop.fs.s3a.impl.ContextAccessors;
import org.apache.hadoop.fs.s3a.impl.CopyFromLocalOperation;
-import org.apache.hadoop.fs.s3a.impl.CopyOutcome;
import org.apache.hadoop.fs.s3a.impl.CreateFileBuilder;
import org.apache.hadoop.fs.s3a.impl.DeleteOperation;
import org.apache.hadoop.fs.s3a.impl.DirectoryPolicy;
@@ -122,6 +137,7 @@ import org.apache.hadoop.fs.s3a.impl.HeaderProcessing;
import org.apache.hadoop.fs.s3a.impl.InternalConstants;
import org.apache.hadoop.fs.s3a.impl.ListingOperationCallbacks;
import org.apache.hadoop.fs.s3a.impl.MkdirOperation;
+import org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteException;
import org.apache.hadoop.fs.s3a.impl.OpenFileSupport;
import org.apache.hadoop.fs.s3a.impl.OperationCallbacks;
import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
@@ -131,7 +147,6 @@ import org.apache.hadoop.fs.s3a.impl.S3AMultipartUploaderBuilder;
import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum;
import org.apache.hadoop.fs.s3a.impl.StoreContext;
import org.apache.hadoop.fs.s3a.impl.StoreContextBuilder;
-import org.apache.hadoop.fs.s3a.impl.V2Migration;
import org.apache.hadoop.fs.s3a.prefetch.S3APrefetchingInputStream;
import org.apache.hadoop.fs.s3a.tools.MarkerToolOperations;
import org.apache.hadoop.fs.s3a.tools.MarkerToolOperationsImpl;
@@ -141,6 +156,7 @@ import org.apache.hadoop.fs.statistics.IOStatistics;
import org.apache.hadoop.fs.statistics.IOStatisticsSource;
import org.apache.hadoop.fs.statistics.IOStatisticsContext;
import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+import org.apache.hadoop.fs.store.LogExactlyOnce;
import org.apache.hadoop.fs.store.audit.AuditEntryPoint;
import org.apache.hadoop.fs.store.audit.ActiveThreadSpanSource;
import org.apache.hadoop.fs.store.audit.AuditSpan;
@@ -213,6 +229,7 @@ import static org.apache.hadoop.fs.s3a.Listing.toLocatedFileStatusIterator;
import static org.apache.hadoop.fs.s3a.S3AUtils.*;
import static org.apache.hadoop.fs.s3a.Statistic.*;
import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.INITIALIZE_SPAN;
+import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.createAWSCredentialProviderList;
import static org.apache.hadoop.fs.s3a.auth.RolePolicies.STATEMENT_ALLOW_KMS_RW;
import static org.apache.hadoop.fs.s3a.auth.RolePolicies.allowS3Operations;
import static org.apache.hadoop.fs.s3a.auth.delegation.S3ADelegationTokens.TokenIssuingPolicy.NoTokensAvailable;
@@ -224,14 +241,14 @@ import static org.apache.hadoop.fs.s3a.impl.CreateFileBuilder.OPTIONS_CREATE_FIL
import static org.apache.hadoop.fs.s3a.impl.CreateFileBuilder.OPTIONS_CREATE_FILE_OVERWRITE;
import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isObjectNotFound;
import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isUnknownBucket;
-import static org.apache.hadoop.fs.s3a.impl.InternalConstants.AP_INACCESSIBLE;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.AP_REQUIRED_EXCEPTION;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.ARN_BUCKET_OPTION;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DEFAULT_UPLOAD_PART_COUNT_LIMIT;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DELETE_CONSIDERED_IDEMPOTENT;
-import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_403;
-import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_404;
+import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_301_MOVED_PERMANENTLY;
+import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_403_FORBIDDEN;
+import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_404_NOT_FOUND;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.UPLOAD_PART_COUNT_LIMIT;
import static org.apache.hadoop.fs.s3a.impl.NetworkBinding.fixBucketRegion;
import static org.apache.hadoop.fs.s3a.impl.NetworkBinding.logDnsLookup;
@@ -275,7 +292,9 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
private URI uri;
private Path workingDir;
private String username;
- private AmazonS3 s3;
+ private S3Client s3Client;
+ /** Async client is used for transfer manager and s3 select. */
+ private S3AsyncClient s3AsyncClient;
// initial callback policy is fail-once; it's there just to assist
// some mock tests and other codepaths trying to call the low level
// APIs on an uninitialized filesystem.
@@ -294,7 +313,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
private Listing listing;
private long partSize;
private boolean enableMultiObjectsDelete;
- private TransferManager transfers;
+ private S3TransferManager transferManager;
private ExecutorService boundedThreadPool;
private ThreadPoolExecutor unboundedThreadPool;
@@ -313,10 +332,12 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
private int executorCapacity;
private long multiPartThreshold;
public static final Logger LOG = LoggerFactory.getLogger(S3AFileSystem.class);
+ /** Exactly once log to warn about setting the region in config to avoid probe. */
+ private static final LogExactlyOnce SET_REGION_WARNING = new LogExactlyOnce(LOG);
private static final Logger PROGRESS =
LoggerFactory.getLogger("org.apache.hadoop.fs.s3a.S3AFileSystem.Progress");
private LocalDirAllocator directoryAllocator;
- private CannedAccessControlList cannedACL;
+ private String cannedACL;
/**
* This must never be null; until initialized it just declares that there
@@ -364,6 +385,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
private AWSCredentialProviderList credentials;
private SignerManager signerManager;
+ private S3AInternals s3aInternals;
/**
* Page size for deletions.
@@ -429,6 +451,8 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
*/
private String scheme = FS_S3A;
+ private final static Map<String, Region> BUCKET_REGIONS = new HashMap<>();
+
/** Add any deprecated keys. */
@SuppressWarnings("deprecation")
private static void addDeprecatedKeys() {
@@ -510,6 +534,8 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
super.initialize(uri, conf);
setConf(conf);
+ s3aInternals = createS3AInternals();
+
// look for encryption data
// DT Bindings may override this
setEncryptionSecrets(
@@ -584,9 +610,6 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
// the encryption algorithms)
bindAWSClient(name, delegationTokensEnabled);
- initTransferManager();
-
-
// This initiates a probe against S3 for the bucket existing.
doBucketProbing();
@@ -654,7 +677,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
AWS_S3_VECTOR_ACTIVE_RANGE_READS, DEFAULT_AWS_S3_VECTOR_ACTIVE_RANGE_READS, 1);
vectoredIOContext = populateVectoredIOContext(conf);
scheme = (this.uri != null && this.uri.getScheme() != null) ? this.uri.getScheme() : FS_S3A;
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
// amazon client exception: stop all services then throw the translation
cleanupWithLogger(LOG, span);
stopAllServices();
@@ -706,7 +729,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* bucket existence check is not done to improve performance of
* S3AFileSystem initialization. When set to 1 or 2, bucket existence check
* will be performed which is potentially slow.
- * If 3 or higher: warn and use the v2 check.
+ * If 3 or higher: warn and skip check.
* Also logging DNS address of the s3 endpoint if the bucket probe value is
* greater than 0 else skipping it for increased performance.
* @throws UnknownStoreException the bucket is absent
@@ -723,18 +746,14 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
LOG.debug("skipping check for bucket existence");
break;
case 1:
- logDnsLookup(getConf());
- verifyBucketExists();
- break;
case 2:
logDnsLookup(getConf());
- verifyBucketExistsV2();
+ verifyBucketExists();
break;
default:
// we have no idea what this is, assume it is from a later release.
- LOG.warn("Unknown bucket probe option {}: {}; falling back to check #2",
+ LOG.warn("Unknown bucket probe option {}: {}; skipping check for bucket existence",
S3A_BUCKET_PROBE, bucketProbe);
- verifyBucketExistsV2();
break;
}
}
@@ -827,54 +846,37 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
}
/**
- * Verify that the bucket exists. This does not check permissions,
- * not even read access.
+ * Verify that the bucket exists.
* Retry policy: retrying, translated.
* @throws UnknownStoreException the bucket is absent
* @throws IOException any other problem talking to S3
*/
@Retries.RetryTranslated
- protected void verifyBucketExists()
- throws UnknownStoreException, IOException {
- if (!invoker.retry("doesBucketExist", bucket, true,
- trackDurationOfOperation(getDurationTrackerFactory(),
- STORE_EXISTS_PROBE.getSymbol(),
- () -> s3.doesBucketExist(bucket)))) {
- throw new UnknownStoreException("s3a://" + bucket + "/", " Bucket does "
- + "not exist");
- }
- }
+ protected void verifyBucketExists() throws UnknownStoreException, IOException {
- /**
- * Verify that the bucket exists. This will correctly throw an exception
- * when credentials are invalid.
- * Retry policy: retrying, translated.
- * @throws UnknownStoreException the bucket is absent
- * @throws IOException any other problem talking to S3
- */
- @Retries.RetryTranslated
- protected void verifyBucketExistsV2()
- throws UnknownStoreException, IOException {
- if (!invoker.retry("doesBucketExistV2", bucket, true,
- trackDurationOfOperation(getDurationTrackerFactory(),
- STORE_EXISTS_PROBE.getSymbol(),
- () -> {
- // Bug in SDK always returns `true` for AccessPoint ARNs with `doesBucketExistV2()`
- // expanding implementation to use ARNs and buckets correctly
+ if(!trackDurationAndSpan(
+ STORE_EXISTS_PROBE, bucket, null, () ->
+ invoker.retry("doesBucketExist", bucket, true, () -> {
try {
- s3.getBucketAcl(bucket);
- } catch (AmazonServiceException ex) {
- int statusCode = ex.getStatusCode();
- if (statusCode == SC_404 ||
- (statusCode == SC_403 && ex.getMessage().contains(AP_INACCESSIBLE))) {
+ if (BUCKET_REGIONS.containsKey(bucket)) {
+ return true;
+ }
+ s3Client.headBucket(HeadBucketRequest.builder().bucket(bucket).build());
+ return true;
+ } catch (AwsServiceException ex) {
+ int statusCode = ex.statusCode();
+ if (statusCode == SC_404_NOT_FOUND ||
+ (statusCode == SC_403_FORBIDDEN && accessPoint != null)) {
return false;
}
}
return true;
}))) {
- throw new UnknownStoreException("s3a://" + bucket + "/", " Bucket does "
- + "not exist");
+
+ throw new UnknownStoreException("s3a://" + bucket + "/",
+ " Bucket does " + "not exist. " + "Accessing with " + ENDPOINT + " set to "
+ + getConf().getTrimmed(ENDPOINT, null));
}
}
@@ -915,7 +917,6 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @param dtEnabled are delegation tokens enabled?
* @throws IOException failure.
*/
- @SuppressWarnings("deprecation")
private void bindAWSClient(URI name, boolean dtEnabled) throws IOException {
Configuration conf = getConf();
credentials = null;
@@ -928,7 +929,6 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
// with it if so.
LOG.debug("Using delegation tokens");
- V2Migration.v1DelegationTokenCredentialProvidersUsed();
S3ADelegationTokens tokens = new S3ADelegationTokens();
this.delegationTokens = Optional.of(tokens);
tokens.bindToFileSystem(getCanonicalUri(),
@@ -955,7 +955,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
uaSuffix = tokens.getUserAgentField();
} else {
// DT support is disabled, so create the normal credential chain
- credentials = createAWSCredentialProviderSet(name, conf);
+ credentials = createAWSCredentialProviderList(name, conf);
}
LOG.debug("Using credential provider {}", credentials);
Class<? extends S3ClientFactory> s3ClientFactoryClass = conf.getClass(
@@ -966,8 +966,14 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
? conf.getTrimmed(ENDPOINT, DEFAULT_ENDPOINT)
: accessPoint.getEndpoint();
- S3ClientFactory.S3ClientCreationParameters parameters = null;
- parameters = new S3ClientFactory.S3ClientCreationParameters()
+ String configuredRegion = accessPoint == null
+ ? conf.getTrimmed(AWS_REGION)
+ : accessPoint.getRegion();
+
+ Region region = getS3Region(configuredRegion);
+
+ S3ClientFactory.S3ClientCreationParameters parameters =
+ new S3ClientFactory.S3ClientCreationParameters()
.withCredentialSet(credentials)
.withPathUri(name)
.withEndpoint(endpoint)
@@ -975,11 +981,97 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
.withPathStyleAccess(conf.getBoolean(PATH_STYLE_ACCESS, false))
.withUserAgentSuffix(uaSuffix)
.withRequesterPays(conf.getBoolean(ALLOW_REQUESTER_PAYS, DEFAULT_ALLOW_REQUESTER_PAYS))
- .withRequestHandlers(auditManager.createRequestHandlers());
+ .withExecutionInterceptors(auditManager.createExecutionInterceptors())
+ .withMinimumPartSize(partSize)
+ .withMultipartThreshold(multiPartThreshold)
+ .withTransferManagerExecutor(unboundedThreadPool)
+ .withRegion(region);
- s3 = ReflectionUtils.newInstance(s3ClientFactoryClass, conf)
- .createS3Client(getUri(),
- parameters);
+ S3ClientFactory clientFactory = ReflectionUtils.newInstance(s3ClientFactoryClass, conf);
+ s3Client = clientFactory.createS3Client(getUri(), parameters);
+ createS3AsyncClient(clientFactory, parameters);
+ transferManager = clientFactory.createS3TransferManager(getS3AsyncClient());
+ }
+
+ /**
+ * Creates and configures the S3AsyncClient.
+ * Uses synchronized method to suppress spotbugs error.
+ *
+ * @param clientFactory factory used to create S3AsyncClient
+ * @param parameters parameter object
+ * @throws IOException on any IO problem
+ */
+ private void createS3AsyncClient(S3ClientFactory clientFactory,
+ S3ClientFactory.S3ClientCreationParameters parameters) throws IOException {
+ s3AsyncClient = clientFactory.createS3AsyncClient(getUri(), parameters);
+ }
+
+ /**
+ * Get the bucket region.
+ *
+ * @param region AWS S3 Region set in the config. This property may not be set, in which case
+ * ask S3 for the region.
+ * @return region of the bucket.
+ */
+ private Region getS3Region(String region) throws IOException {
+
+ if (!StringUtils.isBlank(region)) {
+ return Region.of(region);
+ }
+
+ Region cachedRegion = BUCKET_REGIONS.get(bucket);
+
+ if (cachedRegion != null) {
+ LOG.debug("Got region {} for bucket {} from cache", cachedRegion, bucket);
+ return cachedRegion;
+ }
+
+ Region s3Region = trackDurationAndSpan(STORE_REGION_PROBE, bucket, null,
+ () -> invoker.retry("getS3Region", bucket, true, () -> {
+ try {
+
+ SET_REGION_WARNING.warn(
+ "Getting region for bucket {} from S3, this will slow down FS initialisation. "
+ + "To avoid this, set the region using property {}", bucket,
+ FS_S3A_BUCKET_PREFIX + bucket + ".endpoint.region");
+
+ // build a s3 client with region eu-west-1 that can be used to get the region of the
+ // bucket. Using eu-west-1, as headBucket() doesn't work with us-east-1. This is because
+ // us-east-1 uses the endpoint s3.amazonaws.com, which resolves bucket.s3.amazonaws.com
+ // to the actual region the bucket is in. As the request is signed with us-east-1 and
+ // not the bucket's region, it fails.
+ S3Client getRegionS3Client =
+ S3Client.builder().region(Region.EU_WEST_1).credentialsProvider(credentials)
+ .build();
+
+ HeadBucketResponse headBucketResponse =
+ getRegionS3Client.headBucket(HeadBucketRequest.builder().bucket(bucket).build());
+
+ Region bucketRegion = Region.of(
+ headBucketResponse.sdkHttpResponse().headers().get(BUCKET_REGION_HEADER).get(0));
+ BUCKET_REGIONS.put(bucket, bucketRegion);
+
+ return bucketRegion;
+ } catch (S3Exception exception) {
+ if (exception.statusCode() == SC_301_MOVED_PERMANENTLY) {
+ Region bucketRegion = Region.of(
+ exception.awsErrorDetails().sdkHttpResponse().headers().get(BUCKET_REGION_HEADER)
+ .get(0));
+ BUCKET_REGIONS.put(bucket, bucketRegion);
+
+ return bucketRegion;
+ }
+
+ if (exception.statusCode() == SC_404_NOT_FOUND) {
+ throw new UnknownStoreException("s3a://" + bucket + "/",
+ " Bucket does " + "not exist");
+ }
+
+ throw exception;
+ }
+ }));
+
+ return s3Region;
}
/**
@@ -1079,12 +1171,14 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
.toUpperCase(Locale.US);
StorageClass storageClass = null;
if (!storageClassConf.isEmpty()) {
- try {
- storageClass = StorageClass.fromValue(storageClassConf);
- } catch (IllegalArgumentException e) {
+ storageClass = StorageClass.fromValue(storageClassConf);
+
+ if (storageClass.equals(StorageClass.UNKNOWN_TO_SDK_VERSION)) {
LOG.warn("Unknown storage class property {}: {}; falling back to default storage class",
STORAGE_CLASS, storageClassConf);
+ storageClass = null;
}
+
} else {
LOG.debug("Unset storage class property {}; falling back to default storage class",
STORAGE_CLASS);
@@ -1110,6 +1204,14 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
return requestFactory;
}
+ /**
+ * Get the S3 Async client.
+ * @return the async s3 client.
+ */
+ private S3AsyncClient getS3AsyncClient() {
+ return s3AsyncClient;
+ }
+
/**
* Implementation of all operations used by delegation tokens.
*/
@@ -1150,22 +1252,10 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
return encryptionSecrets;
}
- private void initTransferManager() {
- TransferManagerConfiguration transferConfiguration =
- new TransferManagerConfiguration();
- transferConfiguration.setMinimumUploadPartSize(partSize);
- transferConfiguration.setMultipartUploadThreshold(multiPartThreshold);
- transferConfiguration.setMultipartCopyPartSize(partSize);
- transferConfiguration.setMultipartCopyThreshold(multiPartThreshold);
-
- transfers = new TransferManager(s3, unboundedThreadPool);
- transfers.setConfiguration(transferConfiguration);
- }
-
private void initCannedAcls(Configuration conf) {
String cannedACLName = conf.get(CANNED_ACL, DEFAULT_CANNED_ACL);
if (!cannedACLName.isEmpty()) {
- cannedACL = CannedAccessControlList.valueOf(cannedACLName);
+ cannedACL = AWSCannedACL.valueOf(cannedACLName).toString();
} else {
cannedACL = null;
}
@@ -1198,12 +1288,22 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
public void abortOutstandingMultipartUploads(long seconds)
throws IOException {
Preconditions.checkArgument(seconds >= 0);
- Date purgeBefore =
- new Date(new Date().getTime() - seconds * 1000);
+ Instant purgeBefore =
+ Instant.now().minusSeconds(seconds);
LOG.debug("Purging outstanding multipart uploads older than {}",
purgeBefore);
invoker.retry("Purging multipart uploads", bucket, true,
- () -> transfers.abortMultipartUploads(bucket, purgeBefore));
+ () -> {
+ MultipartUtils.UploadIterator uploadIterator =
+ MultipartUtils.listMultipartUploads(createStoreContext(), s3Client, null, maxKeys);
+
+ while (uploadIterator.hasNext()) {
+ MultipartUpload upload = uploadIterator.next();
+ if (upload.initiated().compareTo(purgeBefore) < 0) {
+ abortMultipartUpload(upload);
+ }
+ }
+ });
}
/**
@@ -1252,81 +1352,122 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
}
/**
- * Returns the S3 client used by this filesystem.
- * This is for internal use within the S3A code itself.
- * @return AmazonS3Client
+ * Set the client -used in mocking tests to force in a different client.
+ * @param client client.
*/
- private AmazonS3 getAmazonS3Client() {
- return s3;
+ protected void setAmazonS3Client(S3Client client) {
+ Preconditions.checkNotNull(client, "clientV2");
+ LOG.debug("Setting S3V2 client to {}", client);
+ s3Client = client;
}
/**
- * Returns the S3 client used by this filesystem.
- * <i>Warning: this must only be used for testing, as it bypasses core
- * S3A operations. </i>
- * @param reason a justification for requesting access.
- * @return AmazonS3Client
+ * S3AInternals method.
+ * {@inheritDoc}.
*/
- @VisibleForTesting
- public AmazonS3 getAmazonS3ClientForTesting(String reason) {
- LOG.warn("Access to S3A client requested, reason {}", reason);
- V2Migration.v1S3ClientRequested();
- return s3;
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ public String getBucketLocation() throws IOException {
+ return s3aInternals.getBucketLocation(bucket);
}
/**
- * Set the client -used in mocking tests to force in a different client.
- * @param client client.
+ * Create the S3AInternals; left as something mocking
+ * subclasses may want to override.
+ * @return the internal implementation
*/
- protected void setAmazonS3Client(AmazonS3 client) {
- Preconditions.checkNotNull(client, "client");
- LOG.debug("Setting S3 client to {}", client);
- s3 = client;
-
- // Need to use a new TransferManager that uses the new client.
- // Also, using a new TransferManager requires a new threadpool as the old
- // TransferManager will shut the thread pool down when it is garbage
- // collected.
- initThreadPools(getConf());
- initTransferManager();
+ protected S3AInternals createS3AInternals() {
+ return new S3AInternalsImpl();
}
/**
- * Get the region of a bucket.
- * Invoked from StoreContext; consider an entry point.
- * @return the region in which a bucket is located
- * @throws AccessDeniedException if the caller lacks permission.
- * @throws IOException on any failure.
+ * Get the S3AInternals.
+ * @return the internal implementation
*/
- @Retries.RetryTranslated
- @InterfaceAudience.LimitedPrivate("diagnostics")
- public String getBucketLocation() throws IOException {
- return getBucketLocation(bucket);
+ public S3AInternals getS3AInternals() {
+ return s3aInternals;
}
/**
- * Get the region of a bucket; fixing up the region so it can be used
- * in the builders of other AWS clients.
- * Requires the caller to have the AWS role permission
- * {@code s3:GetBucketLocation}.
- * Retry policy: retrying, translated.
- * @param bucketName the name of the bucket
- * @return the region in which a bucket is located
- * @throws AccessDeniedException if the caller lacks permission.
- * @throws IOException on any failure.
+ * Implementation of the S3A Internals operations; pulled out of S3AFileSystem to
+ * force code accessing it to call {@link #getS3AInternals()}.
*/
- @VisibleForTesting
- @AuditEntryPoint
- @Retries.RetryTranslated
- public String getBucketLocation(String bucketName) throws IOException {
- final String region = trackDurationAndSpan(
- STORE_EXISTS_PROBE, bucketName, null, () ->
- invoker.retry("getBucketLocation()", bucketName, true, () ->
- // If accessPoint then region is known from Arn
- accessPoint != null
- ? accessPoint.getRegion()
- : s3.getBucketLocation(bucketName)));
- return fixBucketRegion(region);
+ private final class S3AInternalsImpl implements S3AInternals {
+
+ @Override
+ public S3Client getAmazonS3Client(String reason) {
+ LOG.debug("Access to S3 client requested, reason {}", reason);
+ return s3Client;
+ }
+
+ /**
+ * S3AInternals method.
+ * {@inheritDoc}.
+ */
+ @Override
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ public String getBucketLocation() throws IOException {
+ return s3aInternals.getBucketLocation(bucket);
+ }
+
+ /**
+ * S3AInternals method.
+ * {@inheritDoc}.
+ */
+ @Override
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ public String getBucketLocation(String bucketName) throws IOException {
+ final String region = trackDurationAndSpan(
+ STORE_EXISTS_PROBE, bucketName, null, () ->
+ invoker.retry("getBucketLocation()", bucketName, true, () ->
+ // If accessPoint then region is known from Arn
+ accessPoint != null
+ ? accessPoint.getRegion()
+ : s3Client.getBucketLocation(GetBucketLocationRequest.builder()
+ .bucket(bucketName)
+ .build())
+ .locationConstraintAsString()));
+ return fixBucketRegion(region);
+ }
+
+ /**
+ * S3AInternals method.
+ * {@inheritDoc}.
+ */
+ @Override
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ public HeadObjectResponse getObjectMetadata(Path path) throws IOException {
+ return trackDurationAndSpan(INVOCATION_GET_FILE_STATUS, path, () ->
+ S3AFileSystem.this.getObjectMetadata(makeQualified(path), null, invoker,
+ "getObjectMetadata"));
+ }
+
+ /**
+ * S3AInternals method.
+ * {@inheritDoc}.
+ */
+ @Override
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ public HeadBucketResponse getBucketMetadata() throws IOException {
+ return S3AFileSystem.this.getBucketMetadata();
+ }
+
+ /**
+ * Get a shared copy of the AWS credentials, with its reference
+ * counter updated.
+ * Caller is required to call {@code close()} on this after
+ * they have finished using it.
+ * @param purpose what is this for? This is initially for logging
+ * @return a reference to shared credentials.
+ */
+ public AWSCredentialProviderList shareCredentials(final String purpose) {
+ LOG.debug("Sharing credentials for: {}", purpose);
+ return credentials.share();
+ }
}
/**
@@ -1349,7 +1490,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
}
/**
- * Get the encryption algorithm of this endpoint.
+ * Get the encryption algorithm of this connector.
* @return the encryption algorithm.
*/
public S3AEncryptionMethods getS3EncryptionAlgorithm() {
@@ -1396,6 +1537,8 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* Get the bucket of this filesystem.
* @return the bucket
*/
+ @InterfaceAudience.Public
+ @InterfaceStability.Stable
public String getBucket() {
return bucket;
}
@@ -1413,7 +1556,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* Get the canned ACL of this FS.
* @return an ACL, if any
*/
- CannedAccessControlList getCannedACL() {
+ String getCannedACL() {
return cannedACL;
}
@@ -1643,18 +1786,18 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
}
@Override
- public GetObjectRequest newGetRequest(final String key) {
+ public GetObjectRequest.Builder newGetRequestBuilder(final String key) {
// active the audit span used for the operation
try (AuditSpan span = auditSpan.activate()) {
- return getRequestFactory().newGetObjectRequest(key);
+ return getRequestFactory().newGetObjectRequestBuilder(key);
}
}
@Override
- public S3Object getObject(GetObjectRequest request) {
+ public ResponseInputStream<GetObjectResponse> getObject(GetObjectRequest request) {
// active the audit span used for the operation
try (AuditSpan span = auditSpan.activate()) {
- return s3.getObject(request);
+ return s3Client.getObject(request);
}
}
@@ -1681,18 +1824,19 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
implements WriteOperationHelper.WriteOperationHelperCallbacks {
@Override
- public SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request) {
- return s3.selectObjectContent(request);
+ public CompletableFuture<Void> selectObjectContent(
+ SelectObjectContentRequest request,
+ SelectObjectContentResponseHandler responseHandler) {
+ return getS3AsyncClient().selectObjectContent(request, responseHandler);
}
@Override
- public CompleteMultipartUploadResult completeMultipartUpload(
+ public CompleteMultipartUploadResponse completeMultipartUpload(
CompleteMultipartUploadRequest request) {
- return s3.completeMultipartUpload(request);
+ return s3Client.completeMultipartUpload(request);
}
}
-
/**
* Create the read context for reading from the referenced file,
* using FS state as well as the status.
@@ -2054,7 +2198,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
innerRename(src, dst));
LOG.debug("Copied {} bytes", bytesCopied);
return true;
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw translateException("rename(" + src +", " + dst + ")", src, e);
} catch (RenameFailedException e) {
LOG.info("{}", e.getMessage());
@@ -2165,7 +2309,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* This operation throws an exception on any failure which needs to be
* reported and downgraded to a failure.
* Retries: retry translated, assuming all operations it is called do
- * so. For safely, consider catch and handle AmazonClientException
+ * so. For safely, consider catch and handle SdkException
* because this is such a complex method there's a risk it could surface.
* @param source path to be renamed
* @param dest new path after rename
@@ -2176,12 +2320,12 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @return the number of bytes copied.
* @throws FileNotFoundException there's no source file.
* @throws IOException on IO failure.
- * @throws AmazonClientException on failures inside the AWS SDK
+ * @throws SdkException on failures inside the AWS SDK
*/
@Retries.RetryMixed
private long innerRename(Path source, Path dest)
throws RenameFailedException, FileNotFoundException, IOException,
- AmazonClientException {
+ SdkException {
Path src = qualify(source);
Path dst = qualify(dest);
@@ -2276,7 +2420,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
}
@Override
- public CopyResult copyFile(final String srcKey,
+ public CopyObjectResponse copyFile(final String srcKey,
final String destKey,
final S3ObjectAttributes srcAttributes,
final S3AReadOpContext readContext) throws IOException {
@@ -2287,9 +2431,9 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
@Override
public void removeKeys(
- final List<DeleteObjectsRequest.KeyVersion> keysToDelete,
+ final List<ObjectIdentifier> keysToDelete,
final boolean deleteFakeDir)
- throws MultiObjectDeleteException, AmazonClientException, IOException {
+ throws MultiObjectDeleteException, SdkException, IOException {
auditSpan.activate();
S3AFileSystem.this.removeKeys(keysToDelete, deleteFakeDir);
}
@@ -2388,21 +2532,17 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* Low-level call to get at the object metadata.
* This method is used in some external applications and so
* must be viewed as a public entry point.
- * Auditing: An audit entry point.
+ * @deprecated use S3AInternals API.
* @param path path to the object. This will be qualified.
* @return metadata
* @throws IOException IO and object access problems.
*/
- @VisibleForTesting
@AuditEntryPoint
@InterfaceAudience.LimitedPrivate("utilities")
@Retries.RetryTranslated
- @InterfaceStability.Evolving
- public ObjectMetadata getObjectMetadata(Path path) throws IOException {
- V2Migration.v1GetObjectMetadataCalled();
- return trackDurationAndSpan(INVOCATION_GET_FILE_STATUS, path, () ->
- getObjectMetadata(makeQualified(path), null, invoker,
- "getObjectMetadata"));
+ @Deprecated
+ public HeadObjectResponse getObjectMetadata(Path path) throws IOException {
+ return getS3AInternals().getObjectMetadata(path);
}
/**
@@ -2415,7 +2555,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @throws IOException IO and object access problems.
*/
@Retries.RetryTranslated
- private ObjectMetadata getObjectMetadata(Path path,
+ private HeadObjectResponse getObjectMetadata(Path path,
ChangeTracker changeTracker, Invoker changeInvoker, String operation)
throws IOException {
String key = pathToKey(path);
@@ -2628,7 +2768,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
@Retries.RetryRaw
@VisibleForTesting
@InterfaceAudience.LimitedPrivate("external utilities")
- ObjectMetadata getObjectMetadata(String key) throws IOException {
+ HeadObjectResponse getObjectMetadata(String key) throws IOException {
return getObjectMetadata(key, null, invoker, "getObjectMetadata");
}
@@ -2645,28 +2785,28 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @throws RemoteFileChangedException if an unexpected version is detected
*/
@Retries.RetryRaw
- protected ObjectMetadata getObjectMetadata(String key,
+ protected HeadObjectResponse getObjectMetadata(String key,
ChangeTracker changeTracker,
Invoker changeInvoker,
String operation) throws IOException {
- ObjectMetadata meta = changeInvoker.retryUntranslated("GET " + key, true,
+ HeadObjectResponse response = changeInvoker.retryUntranslated("GET " + key, true,
() -> {
- GetObjectMetadataRequest request
- = getRequestFactory().newGetObjectMetadataRequest(key);
+ HeadObjectRequest.Builder requestBuilder =
+ getRequestFactory().newHeadObjectRequestBuilder(key);
incrementStatistic(OBJECT_METADATA_REQUESTS);
DurationTracker duration = getDurationTrackerFactory()
.trackDuration(ACTION_HTTP_HEAD_REQUEST.getSymbol());
try {
LOG.debug("HEAD {} with change tracker {}", key, changeTracker);
if (changeTracker != null) {
- changeTracker.maybeApplyConstraint(request);
+ changeTracker.maybeApplyConstraint(requestBuilder);
}
- ObjectMetadata objectMetadata = s3.getObjectMetadata(request);
+ HeadObjectResponse headObjectResponse = s3Client.headObject(requestBuilder.build());
if (changeTracker != null) {
- changeTracker.processMetadata(objectMetadata, operation);
+ changeTracker.processMetadata(headObjectResponse, operation);
}
- return objectMetadata;
- } catch(AmazonServiceException ase) {
+ return headObjectResponse;
+ } catch (AwsServiceException ase) {
if (!isObjectNotFound(ase)) {
// file not found is not considered a failure of the call,
// so only switch the duration tracker to update failure
@@ -2680,7 +2820,28 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
}
});
incrementReadOperations();
- return meta;
+ return response;
+ }
+
+ /**
+ * Request bucket metadata.
+ * @return the metadata
+ * @throws UnknownStoreException the bucket is absent
+ * @throws IOException any other problem talking to S3
+ */
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ protected HeadBucketResponse getBucketMetadata() throws IOException {
+ final HeadBucketResponse response = trackDurationAndSpan(STORE_EXISTS_PROBE, bucket, null,
+ () -> invoker.retry("getBucketMetadata()", bucket, true, () -> {
+ try {
+ return s3Client.headBucket(
+ getRequestFactory().newHeadBucketRequestBuilder(bucket).build());
+ } catch (NoSuchBucketException e) {
+ throw new UnknownStoreException("s3a://" + bucket + "/", " Bucket does " + "not exist");
+ }
+ }));
+ return response;
}
/**
@@ -2709,9 +2870,9 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
OBJECT_LIST_REQUEST,
() -> {
if (useListV1) {
- return S3ListResult.v1(s3.listObjects(request.getV1()));
+ return S3ListResult.v1(s3Client.listObjects(request.getV1()));
} else {
- return S3ListResult.v2(s3.listObjectsV2(request.getV2()));
+ return S3ListResult.v2(s3Client.listObjectsV2(request.getV2()));
}
}));
}
@@ -2754,15 +2915,21 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
OBJECT_CONTINUE_LIST_REQUEST,
() -> {
if (useListV1) {
- return S3ListResult.v1(
- s3.listNextBatchOfObjects(
- getRequestFactory()
- .newListNextBatchOfObjectsRequest(
- prevResult.getV1())));
+ List<S3Object> prevListResult = prevResult.getV1().contents();
+
+ // Next markers are only present when a delimiter is specified.
+ String nextMarker;
+ if (prevResult.getV1().nextMarker() != null) {
+ nextMarker = prevResult.getV1().nextMarker();
+ } else {
+ nextMarker = prevListResult.get(prevListResult.size() - 1).key();
+ }
+
+ return S3ListResult.v1(s3Client.listObjects(
+ request.getV1().toBuilder().marker(nextMarker).build()));
} else {
- request.getV2().setContinuationToken(prevResult.getV2()
- .getNextContinuationToken());
- return S3ListResult.v2(s3.listObjectsV2(request.getV2()));
+ return S3ListResult.v2(s3Client.listObjectsV2(request.getV2().toBuilder()
+ .continuationToken(prevResult.getV2().nextContinuationToken()).build()));
}
}));
}
@@ -2792,14 +2959,14 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
*
* Retry policy: retry untranslated; delete considered idempotent.
* @param key key to blob to delete.
- * @throws AmazonClientException problems working with S3
+ * @throws SdkException problems working with S3
* @throws InvalidRequestException if the request was rejected due to
* a mistaken attempt to delete the root directory.
*/
@VisibleForTesting
@Retries.RetryRaw
protected void deleteObject(String key)
- throws AmazonClientException, IOException {
+ throws SdkException, IOException {
blockRootDelete(key);
incrementWriteOperations();
try (DurationInfo ignored =
@@ -2811,8 +2978,9 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
incrementStatistic(OBJECT_DELETE_OBJECTS);
trackDurationOfInvocation(getDurationTrackerFactory(),
OBJECT_DELETE_REQUEST.getSymbol(),
- () -> s3.deleteObject(getRequestFactory()
- .newDeleteObjectRequest(key)));
+ () -> s3Client.deleteObject(getRequestFactory()
+ .newDeleteObjectRequestBuilder(key)
+ .build()));
return null;
});
}
@@ -2825,14 +2993,14 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @param f path path to delete
* @param key key of entry
* @param isFile is the path a file (used for instrumentation only)
- * @throws AmazonClientException problems working with S3
+ * @throws SdkException problems working with S3
* @throws IOException from invoker signature only -should not be raised.
*/
@Retries.RetryRaw
void deleteObjectAtPath(Path f,
String key,
boolean isFile)
- throws AmazonClientException, IOException {
+ throws SdkException, IOException {
if (isFile) {
instrumentation.fileDeleted(1);
} else {
@@ -2874,66 +3042,58 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @return the AWS response
* @throws MultiObjectDeleteException one or more of the keys could not
* be deleted.
- * @throws AmazonClientException amazon-layer failure.
+ * @throws SdkException amazon-layer failure.
*/
@Retries.RetryRaw
- private DeleteObjectsResult deleteObjects(DeleteObjectsRequest deleteRequest)
- throws MultiObjectDeleteException, AmazonClientException, IOException {
+ private DeleteObjectsResponse deleteObjects(DeleteObjectsRequest deleteRequest)
+ throws MultiObjectDeleteException, SdkException, IOException {
incrementWriteOperations();
BulkDeleteRetryHandler retryHandler =
new BulkDeleteRetryHandler(createStoreContext());
- int keyCount = deleteRequest.getKeys().size();
- try(DurationInfo ignored =
+ int keyCount = deleteRequest.delete().objects().size();
+ try (DurationInfo ignored =
new DurationInfo(LOG, false, "DELETE %d keys",
keyCount)) {
- return invoker.retryUntranslated("delete",
- DELETE_CONSIDERED_IDEMPOTENT,
- (text, e, r, i) -> {
- // handle the failure
- retryHandler.bulkDeleteRetried(deleteRequest, e);
- },
- // duration is tracked in the bulk delete counters
- trackDurationOfOperation(getDurationTrackerFactory(),
- OBJECT_BULK_DELETE_REQUEST.getSymbol(), () -> {
- incrementStatistic(OBJECT_DELETE_OBJECTS, keyCount);
- return s3.deleteObjects(deleteRequest);
- }));
- } catch (MultiObjectDeleteException e) {
- // one or more of the keys could not be deleted.
- // log and rethrow
- List<MultiObjectDeleteException.DeleteError> errors = e.getErrors();
- LOG.debug("Partial failure of delete, {} errors", errors.size(), e);
- for (MultiObjectDeleteException.DeleteError error : errors) {
- LOG.debug("{}: \"{}\" - {}",
- error.getKey(), error.getCode(), error.getMessage());
+ DeleteObjectsResponse response =
+ invoker.retryUntranslated("delete", DELETE_CONSIDERED_IDEMPOTENT,
+ (text, e, r, i) -> {
+ // handle the failure
+ retryHandler.bulkDeleteRetried(deleteRequest, e);
+ },
+ // duration is tracked in the bulk delete counters
+ trackDurationOfOperation(getDurationTrackerFactory(),
+ OBJECT_BULK_DELETE_REQUEST.getSymbol(), () -> {
+ incrementStatistic(OBJECT_DELETE_OBJECTS, keyCount);
+ return s3Client.deleteObjects(deleteRequest);
+ }));
+
+ if (!response.errors().isEmpty()) {
+ // one or more of the keys could not be deleted.
+ // log and then throw
+ List<S3Error> errors = response.errors();
+ LOG.debug("Partial failure of delete, {} errors", errors.size());
+ for (S3Error error : errors) {
+ LOG.debug("{}: \"{}\" - {}", error.key(), error.code(), error.message());
+ }
+ throw new MultiObjectDeleteException(errors);
}
- throw e;
+
+ return response;
}
}
/**
- * Create a putObject request.
+ * Create a putObject request builder.
* Adds the ACL and metadata
* @param key key of object
- * @param metadata metadata header
- * @param srcfile source file
+ * @param length length of object to be uploaded
+ * @param isDirectoryMarker true if object to be uploaded is a directory marker
* @return the request
*/
- public PutObjectRequest newPutObjectRequest(String key,
- ObjectMetadata metadata, File srcfile) {
- return requestFactory.newPutObjectRequest(key, metadata, null, srcfile);
- }
-
- /**
- * Create a new object metadata instance.
- * Any standard metadata headers are added here, for example:
- * encryption.
- *
- * @param length length of data to set in header.
- * @return a new metadata instance
- */
- public ObjectMetadata newObjectMetadata(long length) {
- return requestFactory.newObjectMetadata(length);
+ public PutObjectRequest.Builder newPutObjectRequestBuilder(String key,
+ long length,
+ boolean isDirectoryMarker) {
+ return requestFactory.newPutObjectRequestBuilder(key, null, length, isDirectoryMarker);
}
/**
@@ -2950,15 +3110,24 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* Retry policy: N/A: the transfer manager is performing the upload.
* Auditing: must be inside an audit span.
* @param putObjectRequest the request
+ * @param file the file to be uploaded
+ * @param listener the progress listener for the request
* @return the upload initiated
*/
@Retries.OnceRaw
- public UploadInfo putObject(PutObjectRequest putObjectRequest) {
+ public UploadInfo putObject(PutObjectRequest putObjectRequest, File file,
+ ProgressableProgressListener listener) {
long len = getPutRequestLength(putObjectRequest);
- LOG.debug("PUT {} bytes to {} via transfer manager ",
- len, putObjectRequest.getKey());
+ LOG.debug("PUT {} bytes to {} via transfer manager ", len, putObjectRequest.key());
incrementPutStartStatistics(len);
- Upload upload = transfers.upload(putObjectRequest);
+
+ FileUpload upload = transferManager.uploadFile(
+ UploadFileRequest.builder()
+ .putObjectRequest(putObjectRequest)
+ .source(file)
+ .addTransferListener(listener)
+ .build());
+
return new UploadInfo(upload, len);
}
@@ -2973,30 +3142,37 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @param putObjectRequest the request
* @param putOptions put object options
* @param durationTrackerFactory factory for duration tracking
+ * @param uploadData data to be uploaded
+ * @param isFile represents if data to be uploaded is a file
* @return the upload initiated
- * @throws AmazonClientException on problems
+ * @throws SdkException on problems
*/
@VisibleForTesting
@Retries.OnceRaw("For PUT; post-PUT actions are RetryExceptionsSwallowed")
- PutObjectResult putObjectDirect(PutObjectRequest putObjectRequest,
+ PutObjectResponse putObjectDirect(PutObjectRequest putObjectRequest,
PutObjectOptions putOptions,
+ S3ADataBlocks.BlockUploadData uploadData, boolean isFile,
DurationTrackerFactory durationTrackerFactory)
- throws AmazonClientException {
+ throws SdkException {
long len = getPutRequestLength(putObjectRequest);
- LOG.debug("PUT {} bytes to {}", len, putObjectRequest.getKey());
+ LOG.debug("PUT {} bytes to {}", len, putObjectRequest.key());
incrementPutStartStatistics(len);
try {
- PutObjectResult result = trackDurationOfSupplier(
- nonNullDurationTrackerFactory(durationTrackerFactory),
- OBJECT_PUT_REQUESTS.getSymbol(), () ->
- s3.putObject(putObjectRequest));
+ PutObjectResponse response =
+ trackDurationOfSupplier(nonNullDurationTrackerFactory(durationTrackerFactory),
+ OBJECT_PUT_REQUESTS.getSymbol(),
+ () -> isFile ?
+ s3Client.putObject(putObjectRequest, RequestBody.fromFile(uploadData.getFile())) :
+ s3Client.putObject(putObjectRequest,
+ RequestBody.fromInputStream(uploadData.getUploadStream(),
+ putObjectRequest.contentLength())));
incrementPutCompletedStatistics(true, len);
// apply any post-write actions.
- finishedWrite(putObjectRequest.getKey(), len,
- result.getETag(), result.getVersionId(),
+ finishedWrite(putObjectRequest.key(), len,
+ response.eTag(), response.versionId(),
putOptions);
- return result;
- } catch (SdkBaseException e) {
+ return response;
+ } catch (SdkException e) {
incrementPutCompletedStatistics(false, len);
throw e;
}
@@ -3009,12 +3185,8 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @throws IllegalArgumentException if the length is negative
*/
private long getPutRequestLength(PutObjectRequest putObjectRequest) {
- long len;
- if (putObjectRequest.getFile() != null) {
- len = putObjectRequest.getFile().length();
- } else {
- len = putObjectRequest.getMetadata().getContentLength();
- }
+ long len = putObjectRequest.contentLength();
+
Preconditions.checkState(len >= 0, "Cannot PUT object of unknown length");
return len;
}
@@ -3022,28 +3194,29 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
/**
* Upload part of a multi-partition file.
* Increments the write and put counters.
- * <i>Important: this call does not close any input stream in the request.</i>
+ * <i>Important: this call does not close any input stream in the body.</i>
*
* Retry Policy: none.
- * @param request request
* @param durationTrackerFactory duration tracker factory for operation
+ * @param request the upload part request.
+ * @param body the request body.
* @return the result of the operation.
- * @throws AmazonClientException on problems
+ * @throws AwsServiceException on problems
*/
@Retries.OnceRaw
- UploadPartResult uploadPart(UploadPartRequest request,
+ UploadPartResponse uploadPart(UploadPartRequest request, RequestBody body,
final DurationTrackerFactory durationTrackerFactory)
- throws AmazonClientException {
- long len = request.getPartSize();
+ throws AwsServiceException {
+ long len = request.contentLength();
incrementPutStartStatistics(len);
try {
- UploadPartResult uploadPartResult = trackDurationOfSupplier(
+ UploadPartResponse uploadPartResponse = trackDurationOfSupplier(
nonNullDurationTrackerFactory(durationTrackerFactory),
MULTIPART_UPLOAD_PART_PUT.getSymbol(), () ->
- s3.uploadPart(request));
+ s3Client.uploadPart(request, body));
incrementPutCompletedStatistics(true, len);
- return uploadPartResult;
- } catch (AmazonClientException e) {
+ return uploadPartResponse;
+ } catch (AwsServiceException e) {
incrementPutCompletedStatistics(false, len);
throw e;
}
@@ -3108,56 +3281,57 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* be deleted in a multiple object delete operation.
* The number of rejected objects will be added to the metric
* {@link Statistic#FILES_DELETE_REJECTED}.
- * @throws AmazonClientException other amazon-layer failure.
+ * @throws AwsServiceException other amazon-layer failure.
*/
@Retries.RetryRaw
private void removeKeysS3(
- List<DeleteObjectsRequest.KeyVersion> keysToDelete,
+ List<ObjectIdentifier> keysToDelete,
boolean deleteFakeDir)
- throws MultiObjectDeleteException, AmazonClientException,
- IOException {
+ throws MultiObjectDeleteException, AwsServiceException, IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("Initiating delete operation for {} objects",
keysToDelete.size());
- for (DeleteObjectsRequest.KeyVersion key : keysToDelete) {
- LOG.debug(" {} {}", key.getKey(),
- key.getVersion() != null ? key.getVersion() : "");
+ for (ObjectIdentifier objectIdentifier : keysToDelete) {
+ LOG.debug(" {} {}", objectIdentifier.key(),
+ objectIdentifier.versionId() != null ? objectIdentifier.versionId() : "");
}
}
if (keysToDelete.isEmpty()) {
// exit fast if there are no keys to delete
return;
}
- for (DeleteObjectsRequest.KeyVersion keyVersion : keysToDelete) {
- blockRootDelete(keyVersion.getKey());
+ for (ObjectIdentifier objectIdentifier : keysToDelete) {
+ blockRootDelete(objectIdentifier.key());
}
try {
if (enableMultiObjectsDelete) {
if (keysToDelete.size() <= pageSize) {
deleteObjects(getRequestFactory()
- .newBulkDeleteRequest(keysToDelete));
+ .newBulkDeleteRequestBuilder(keysToDelete)
+ .build());
} else {
// Multi object deletion of more than 1000 keys is not supported
// by s3. So we are paging the keys by page size.
LOG.debug("Partitioning the keys to delete as it is more than " +
"page size. Number of keys: {}, Page size: {}",
keysToDelete.size(), pageSize);
- for (List<DeleteObjectsRequest.KeyVersion> batchOfKeysToDelete :
+ for (List<ObjectIdentifier> batchOfKeysToDelete :
Lists.partition(keysToDelete, pageSize)) {
deleteObjects(getRequestFactory()
- .newBulkDeleteRequest(batchOfKeysToDelete));
+ .newBulkDeleteRequestBuilder(batchOfKeysToDelete)
+ .build());
}
}
} else {
- for (DeleteObjectsRequest.KeyVersion keyVersion : keysToDelete) {
- deleteObject(keyVersion.getKey());
+ for (ObjectIdentifier objectIdentifier : keysToDelete) {
+ deleteObject(objectIdentifier.key());
}
}
} catch (MultiObjectDeleteException ex) {
// partial delete.
// Update the stats with the count of the actual number of successful
// deletions.
- int rejected = ex.getErrors().size();
+ int rejected = ex.errors().size();
noteDeleted(keysToDelete.size() - rejected, deleteFakeDir);
incrementStatistic(FILES_DELETE_REJECTED, rejected);
throw ex;
@@ -3190,15 +3364,15 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* a mistaken attempt to delete the root directory.
* @throws MultiObjectDeleteException one or more of the keys could not
* be deleted in a multiple object delete operation.
- * @throws AmazonClientException amazon-layer failure.
+ * @throws AwsServiceException amazon-layer failure.
* @throws IOException other IO Exception.
*/
@VisibleForTesting
@Retries.RetryRaw
public void removeKeys(
- final List<DeleteObjectsRequest.KeyVersion> keysToDelete,
+ final List<ObjectIdentifier> keysToDelete,
final boolean deleteFakeDir)
- throws MultiObjectDeleteException, AmazonClientException,
+ throws MultiObjectDeleteException, AwsServiceException,
IOException {
try (DurationInfo ignored = new DurationInfo(LOG, false,
"Deleting %d keys", keysToDelete.size())) {
@@ -3268,7 +3442,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
LOG.debug("Couldn't delete {} - does not exist: {}", path, e.toString());
instrumentation.errorIgnored();
return false;
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw translateException("delete", path, e);
}
}
@@ -3282,7 +3456,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
*/
@Retries.RetryTranslated
private void createFakeDirectoryIfNecessary(Path f)
- throws IOException, AmazonClientException {
+ throws IOException, SdkException {
String key = pathToKey(f);
// we only make the LIST call; the codepaths to get here should not
// be reached if there is an empty dir marker -and if they do, it
@@ -3302,7 +3476,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
@Retries.RetryTranslated
@VisibleForTesting
protected void maybeCreateFakeParentDirectory(Path path)
- throws IOException, AmazonClientException {
+ throws IOException, SdkException {
Path parent = path.getParent();
if (parent != null && !parent.isRoot() && !isUnderMagicCommitPath(parent)) {
createFakeDirectoryIfNecessary(parent);
@@ -3356,11 +3530,11 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @return the statuses of the files/directories in the given patch
* @throws FileNotFoundException when the path does not exist;
* @throws IOException due to an IO problem.
- * @throws AmazonClientException on failures inside the AWS SDK
+ * @throws SdkException on failures inside the AWS SDK
*/
private RemoteIterator<S3AFileStatus> innerListStatus(Path f)
throws FileNotFoundException,
- IOException, AmazonClientException {
+ IOException, SdkException {
Path path = qualify(f);
LOG.debug("List status for path: {}", path);
@@ -3424,15 +3598,15 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
private S3ListRequest createListObjectsRequest(String key,
String delimiter, int limit) {
if (!useListV1) {
- ListObjectsV2Request request =
- getRequestFactory().newListObjectsV2Request(
+ ListObjectsV2Request.Builder requestBuilder =
+ getRequestFactory().newListObjectsV2RequestBuilder(
key, delimiter, limit);
- return S3ListRequest.v2(request);
+ return S3ListRequest.v2(requestBuilder.build());
} else {
- ListObjectsRequest request =
- getRequestFactory().newListObjectsV1Request(
+ ListObjectsRequest.Builder requestBuilder =
+ getRequestFactory().newListObjectsV1RequestBuilder(
key, delimiter, limit);
- return S3ListRequest.v1(request);
+ return S3ListRequest.v1(requestBuilder.build());
}
}
@@ -3720,31 +3894,31 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
&& probes.contains(StatusProbeEnum.Head)) {
try {
// look for the simple file
- ObjectMetadata meta = getObjectMetadata(key);
+ HeadObjectResponse meta = getObjectMetadata(key);
LOG.debug("Found exact file: normal file {}", key);
- long contentLength = meta.getContentLength();
+ long contentLength = meta.contentLength();
// check if CSE is enabled, then strip padded length.
- if (isCSEEnabled
- && meta.getUserMetaDataOf(Headers.CRYPTO_CEK_ALGORITHM) != null
+ if (isCSEEnabled &&
+ meta.metadata().get(AWSHeaders.CRYPTO_CEK_ALGORITHM) != null
&& contentLength >= CSE_PADDING_LENGTH) {
contentLength -= CSE_PADDING_LENGTH;
}
return new S3AFileStatus(contentLength,
- dateToLong(meta.getLastModified()),
+ meta.lastModified().toEpochMilli(),
path,
getDefaultBlockSize(path),
username,
- meta.getETag(),
- meta.getVersionId());
- } catch (AmazonServiceException e) {
+ meta.eTag(),
+ meta.versionId());
+ } catch (AwsServiceException e) {
// if the response is a 404 error, it just means that there is
// no file at that path...the remaining checks will be needed.
// But: an empty bucket is also a 404, so check for that
// and fail.
- if (e.getStatusCode() != SC_404 || isUnknownBucket(e)) {
+ if (e.statusCode() != SC_404_NOT_FOUND || isUnknownBucket(e)) {
throw translateException("getFileStatus", path, e);
}
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw translateException("getFileStatus", path, e);
}
}
@@ -3787,11 +3961,11 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
LOG.debug("Found root directory");
return new S3AFileStatus(Tristate.TRUE, path, username);
}
- } catch (AmazonServiceException e) {
- if (e.getStatusCode() != SC_404 || isUnknownBucket(e)) {
+ } catch (AwsServiceException e) {
+ if (e.statusCode() != SC_404_NOT_FOUND || isUnknownBucket(e)) {
throw translateException("getFileStatus", path, e);
}
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw translateException("getFileStatus", path, e);
}
}
@@ -3835,7 +4009,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @throws IOException IO problem
* @throws FileAlreadyExistsException the destination file exists and
* overwrite==false
- * @throws AmazonClientException failure in the AWS SDK
+ * @throws SdkException failure in the AWS SDK
*/
@Override
@AuditEntryPoint
@@ -3890,13 +4064,12 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
to,
() -> {
final String key = pathToKey(to);
- final ObjectMetadata om = newObjectMetadata(file.length());
Progressable progress = null;
- PutObjectRequest putObjectRequest = newPutObjectRequest(key, om, file);
- S3AFileSystem.this.invoker.retry(
- "putObject(" + "" + ")", to.toString(),
- true,
- () -> executePut(putObjectRequest, progress, putOptionsForPath(to)));
+ PutObjectRequest.Builder putObjectRequestBuilder =
+ newPutObjectRequestBuilder(key, file.length(), false);
+ S3AFileSystem.this.invoker.retry("putObject(" + "" + ")", to.toString(), true,
+ () -> executePut(putObjectRequestBuilder.build(), progress, putOptionsForPath(to),
+ file));
return null;
});
@@ -3921,40 +4094,35 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
/**
* Execute a PUT via the transfer manager, blocking for completion.
- * If the waiting for completion is interrupted, the upload will be
- * aborted before an {@code InterruptedIOException} is thrown.
* @param putObjectRequest request
* @param progress optional progress callback
* @param putOptions put object options
* @return the upload result
- * @throws InterruptedIOException if the blocking was interrupted.
+ * @throws IOException IO failure
*/
@Retries.OnceRaw("For PUT; post-PUT actions are RetrySwallowed")
- UploadResult executePut(
+ PutObjectResponse executePut(
final PutObjectRequest putObjectRequest,
final Progressable progress,
- final PutObjectOptions putOptions)
- throws InterruptedIOException {
- String key = putObjectRequest.getKey();
+ final PutObjectOptions putOptions,
+ final File file)
+ throws IOException {
+ String key = putObjectRequest.key();
long len = getPutRequestLength(putObjectRequest);
- UploadInfo info = putObject(putObjectRequest);
- Upload upload = info.getUpload();
- ProgressableProgressListener listener = new ProgressableProgressListener(
- this, key, upload, progress);
- upload.addProgressListener(listener);
- UploadResult result = waitForUploadCompletion(key, info);
- listener.uploadCompleted();
+ ProgressableProgressListener listener =
+ new ProgressableProgressListener(this, putObjectRequest.key(), progress);
+ UploadInfo info = putObject(putObjectRequest, file, listener);
+ PutObjectResponse result = waitForUploadCompletion(key, info).response();
+ listener.uploadCompleted(info.getFileUpload());
// post-write actions
finishedWrite(key, len,
- result.getETag(), result.getVersionId(), putOptions);
+ result.eTag(), result.versionId(), putOptions);
return result;
}
/**
* Wait for an upload to complete.
- * If the waiting for completion is interrupted, the upload will be
- * aborted before an {@code InterruptedIOException} is thrown.
* If the upload (or its result collection) failed, this is where
* the failure is raised as an AWS exception.
* Calls {@link #incrementPutCompletedStatistics(boolean, long)}
@@ -3962,24 +4130,20 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @param key destination key
* @param uploadInfo upload to wait for
* @return the upload result
- * @throws InterruptedIOException if the blocking was interrupted.
+ * @throws IOException IO failure
*/
@Retries.OnceRaw
- UploadResult waitForUploadCompletion(String key, UploadInfo uploadInfo)
- throws InterruptedIOException {
- Upload upload = uploadInfo.getUpload();
+ CompletedFileUpload waitForUploadCompletion(String key, UploadInfo uploadInfo)
+ throws IOException {
+ FileUpload upload = uploadInfo.getFileUpload();
try {
- UploadResult result = upload.waitForUploadResult();
+ CompletedFileUpload result = upload.completionFuture().join();
incrementPutCompletedStatistics(true, uploadInfo.getLength());
return result;
- } catch (InterruptedException e) {
+ } catch (CompletionException e) {
LOG.info("Interrupted: aborting upload");
incrementPutCompletedStatistics(false, uploadInfo.getLength());
- upload.abort();
- throw (InterruptedIOException)
- new InterruptedIOException("Interrupted in PUT to "
- + keyToQualifiedPath(key))
- .initCause(e);
+ throw extractException("upload", key, e);
}
}
@@ -4072,17 +4236,13 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* both the expected state of this FS and of failures while being stopped.
*/
protected synchronized void stopAllServices() {
- // shutting down the transfer manager also shuts
- // down the S3 client it is bonded to.
- if (transfers != null) {
- try {
- transfers.shutdownNow(true);
- } catch (RuntimeException e) {
- // catch and swallow for resilience.
- LOG.debug("When shutting down", e);
- }
- transfers = null;
- }
+ closeAutocloseables(LOG, transferManager,
+ s3Client,
+ getS3AsyncClient());
+ transferManager = null;
+ s3Client = null;
+ s3AsyncClient = null;
+
// At this point the S3A client is shut down,
// now the executor pools are closed
HadoopExecutors.shutdown(boundedThreadPool, LOG,
@@ -4237,21 +4397,11 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* @throws IOException Other IO problems
*/
@Retries.RetryTranslated
- private CopyResult copyFile(String srcKey, String dstKey, long size,
+ private CopyObjectResponse copyFile(String srcKey, String dstKey, long size,
S3ObjectAttributes srcAttributes, S3AReadOpContext readContext)
- throws IOException, InterruptedIOException {
+ throws IOException {
LOG.debug("copyFile {} -> {} ", srcKey, dstKey);
- ProgressListener progressListener = progressEvent -> {
- switch (progressEvent.getEventType()) {
- case TRANSFER_PART_COMPLETED_EVENT:
- incrementWriteOperations();
- break;
- default:
- break;
- }
- };
-
ChangeTracker changeTracker = new ChangeTracker(
keyToQualifiedPath(srcKey).toString(),
changeDetectionPolicy,
@@ -4263,7 +4413,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
String action = "copyFile(" + srcKey + ", " + dstKey + ")";
Invoker readInvoker = readContext.getReadInvoker();
- ObjectMetadata srcom;
+ HeadObjectResponse srcom;
try {
srcom = once(action, srcKey,
() ->
@@ -4286,33 +4436,32 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
action, srcKey,
true,
() -> {
- CopyObjectRequest copyObjectRequest =
- getRequestFactory().newCopyObjectRequest(srcKey, dstKey, srcom);
- changeTracker.maybeApplyConstraint(copyObjectRequest);
+ CopyObjectRequest.Builder copyObjectRequestBuilder =
+ getRequestFactory().newCopyObjectRequestBuilder(srcKey, dstKey, srcom);
+ changeTracker.maybeApplyConstraint(copyObjectRequestBuilder);
incrementStatistic(OBJECT_COPY_REQUESTS);
- Copy copy = transfers.copy(copyObjectRequest,
- getAuditManager().createStateChangeListener());
- copy.addProgressListener(progressListener);
- CopyOutcome copyOutcome = CopyOutcome.waitForCopy(copy);
- InterruptedException interruptedException =
- copyOutcome.getInterruptedException();
- if (interruptedException != null) {
- // copy interrupted: convert to an IOException.
- throw (IOException)new InterruptedIOException(
- "Interrupted copying " + srcKey
- + " to " + dstKey + ", cancelling")
- .initCause(interruptedException);
- }
- SdkBaseException awsException = copyOutcome.getAwsException();
- if (awsException != null) {
- changeTracker.processException(awsException, "copy");
- throw awsException;
+
+ Copy copy = transferManager.copy(
+ CopyRequest.builder()
+ .copyObjectRequest(copyObjectRequestBuilder.build())
+ .build());
+
+ try {
+ CompletedCopy completedCopy = copy.completionFuture().join();
+ CopyObjectResponse result = completedCopy.response();
+ changeTracker.processResponse(result);
+ incrementWriteOperations();
+ instrumentation.filesCopied(1, size);
+ return result;
+ } catch (CompletionException e) {
+ Throwable cause = e.getCause();
+ if (cause instanceof SdkException) {
+ SdkException awsException = (SdkException)cause;
+ changeTracker.processException(awsException, "copy");
+ throw awsException;
+ }
+ throw extractException(action, srcKey, e);
}
- CopyResult result = copyOutcome.getCopyResult();
- changeTracker.processResponse(result);
- incrementWriteOperations();
- instrumentation.filesCopied(1, size);
- return result;
});
}
@@ -4321,16 +4470,16 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* Retry policy: none + untranslated.
* @param request request to initiate
* @return the result of the call
- * @throws AmazonClientException on failures inside the AWS SDK
+ * @throws SdkException on failures inside the AWS SDK
* @throws IOException Other IO problems
*/
@Retries.OnceRaw
- InitiateMultipartUploadResult initiateMultipartUpload(
- InitiateMultipartUploadRequest request) throws IOException {
- LOG.debug("Initiate multipart upload to {}", request.getKey());
+ CreateMultipartUploadResponse initiateMultipartUpload(
+ CreateMultipartUploadRequest request) throws IOException {
+ LOG.debug("Initiate multipart upload to {}", request.key());
return trackDurationOfSupplier(getDurationTrackerFactory(),
OBJECT_MULTIPART_UPLOAD_INITIATED.getSymbol(),
- () -> getAmazonS3Client().initiateMultipartUpload(request));
+ () -> s3Client.createMultipartUpload(request));
}
/**
@@ -4403,22 +4552,22 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
*/
@Retries.RetryExceptionsSwallowed
private void deleteUnnecessaryFakeDirectories(Path path) {
- List<DeleteObjectsRequest.KeyVersion> keysToRemove = new ArrayList<>();
+ List<ObjectIdentifier> keysToRemove = new ArrayList<>();
while (!path.isRoot()) {
String key = pathToKey(path);
key = (key.endsWith("/")) ? key : (key + "/");
LOG.trace("To delete unnecessary fake directory {} for {}", key, path);
- keysToRemove.add(new DeleteObjectsRequest.KeyVersion(key));
+ keysToRemove.add(ObjectIdentifier.builder().key(key).build());
path = path.getParent();
}
try {
removeKeys(keysToRemove, true);
- } catch(AmazonClientException | IOException e) {
+ } catch (AwsServiceException | IOException e) {
instrumentation.errorIgnored();
if (LOG.isDebugEnabled()) {
StringBuilder sb = new StringBuilder();
- for(DeleteObjectsRequest.KeyVersion kv : keysToRemove) {
- sb.append(kv.getKey()).append(",");
+ for (ObjectIdentifier objectIdentifier : keysToRemove) {
+ sb.append(objectIdentifier.key()).append(",");
}
LOG.debug("While deleting keys {} ", sb.toString(), e);
}
@@ -4451,11 +4600,18 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
@Retries.RetryTranslated
private void createEmptyObject(final String objectName, PutObjectOptions putOptions)
throws IOException {
- invoker.retry("PUT 0-byte object ", objectName,
- true, () ->
- putObjectDirect(getRequestFactory().newDirectoryMarkerRequest(objectName),
- putOptions,
- getDurationTrackerFactory()));
+ final InputStream im = new InputStream() {
+ @Override
+ public int read() throws IOException {
+ return -1;
+ }
+ };
+
+ S3ADataBlocks.BlockUploadData uploadData = new S3ADataBlocks.BlockUploadData(im);
+
+ invoker.retry("PUT 0-byte object ", objectName, true,
+ () -> putObjectDirect(getRequestFactory().newDirectoryMarkerRequest(objectName).build(),
+ putOptions, uploadData, false, getDurationTrackerFactory()));
incrementPutProgressStatistics(objectName, 0);
instrumentation.directoryCreated();
}
@@ -4712,10 +4868,10 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
ETAG_CHECKSUM_ENABLED_DEFAULT)) {
return trackDurationAndSpan(INVOCATION_GET_FILE_CHECKSUM, path, () -> {
LOG.debug("getFileChecksum({})", path);
- ObjectMetadata headers = getObjectMetadata(path, null,
+ HeadObjectResponse headers = getObjectMetadata(path, null,
invoker,
"getFileChecksum are");
- String eTag = headers.getETag();
+ String eTag = headers.eTag();
return eTag != null ? new EtagChecksum(eTag) : null;
});
} else {
@@ -4797,11 +4953,18 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
HeaderProcessing.HeaderProcessingCallbacks {
@Override
- public ObjectMetadata getObjectMetadata(final String key)
+ public HeadObjectResponse getObjectMetadata(final String key)
throws IOException {
return once("getObjectMetadata", key, () ->
S3AFileSystem.this.getObjectMetadata(key));
}
+
+ @Override
+ public HeadBucketResponse getBucketMetadata()
+ throws IOException {
+ return once("getBucketMetadata", bucket, () ->
+ S3AFileSystem.this.getBucketMetadata());
+ }
}
/**
* {@inheritDoc}.
@@ -4914,7 +5077,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
// If we have reached here, it means either there are files
// in this directory or it is empty.
return listFilesAssumingDir;
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw translateException("listFiles", path, e);
}
}
@@ -5012,8 +5175,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
// span is picked up retained in the listing.
return trackDurationAndSpan(MULTIPART_UPLOAD_LIST, prefix, null, () ->
MultipartUtils.listMultipartUploads(
- createStoreContext(),
- s3, prefix, maxKeys
+ createStoreContext(), s3Client, prefix, maxKeys
));
}
@@ -5024,7 +5186,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
* Retry policy: retry, translated.
* @return a listing of multipart uploads.
* @param prefix prefix to scan for, "" for none
- * @throws IOException IO failure, including any uprated AmazonClientException
+ * @throws IOException IO failure, including any uprated SdkException
*/
@InterfaceAudience.Private
@Retries.RetryTranslated
@@ -5036,9 +5198,9 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
}
String p = prefix;
return invoker.retry("listMultipartUploads", p, true, () -> {
- ListMultipartUploadsRequest request = getRequestFactory()
- .newListMultipartUploadsRequest(p);
- return s3.listMultipartUploads(request).getMultipartUploads();
+ ListMultipartUploadsRequest.Builder requestBuilder = getRequestFactory()
+ .newListMultipartUploadsRequestBuilder(p);
+ return s3Client.listMultipartUploads(requestBuilder.build()).uploads();
});
}
@@ -5051,10 +5213,10 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
@Retries.OnceRaw
void abortMultipartUpload(String destKey, String uploadId) {
LOG.info("Aborting multipart upload {} to {}", uploadId, destKey);
- getAmazonS3Client().abortMultipartUpload(
- getRequestFactory().newAbortMultipartUploadRequest(
+ s3Client.abortMultipartUpload(
+ getRequestFactory().newAbortMultipartUploadRequestBuilder(
destKey,
- uploadId));
+ uploadId).build());
}
/**
@@ -5066,18 +5228,18 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
void abortMultipartUpload(MultipartUpload upload) {
String destKey;
String uploadId;
- destKey = upload.getKey();
- uploadId = upload.getUploadId();
+ destKey = upload.key();
+ uploadId = upload.uploadId();
if (LOG.isInfoEnabled()) {
DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
LOG.debug("Aborting multipart upload {} to {} initiated by {} on {}",
- uploadId, destKey, upload.getInitiator(),
- df.format(upload.getInitiated()));
+ uploadId, destKey, upload.initiator(),
+ df.format(Date.from(upload.initiated())));
}
- getAmazonS3Client().abortMultipartUpload(
- getRequestFactory().newAbortMultipartUploadRequest(
+ s3Client.abortMultipartUpload(
+ getRequestFactory().newAbortMultipartUploadRequestBuilder(
destKey,
- uploadId));
+ uploadId).build());
}
/**
@@ -5464,4 +5626,5 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
public boolean isMultipartUploadEnabled() {
return isMultipartUploadEnabled;
}
+
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
index 4b50ab2c04bd..2ed9083efcdd 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
@@ -22,6 +22,7 @@ import javax.annotation.Nullable;
import java.io.Closeable;
import java.io.EOFException;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InterruptedIOException;
import java.net.SocketTimeoutException;
import java.nio.ByteBuffer;
@@ -31,9 +32,9 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.IntFunction;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.S3ObjectInputStream;
+import software.amazon.awssdk.core.ResponseInputStream;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,7 +47,6 @@ import org.apache.hadoop.fs.CanUnbuffer;
import org.apache.hadoop.fs.FSExceptionMessages;
import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.fs.FileRange;
-import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.StreamCapabilities;
import org.apache.hadoop.fs.impl.CombinedFileRange;
import org.apache.hadoop.fs.VectoredReadUtils;
@@ -61,6 +61,7 @@ import org.apache.hadoop.fs.statistics.IOStatisticsSource;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.functional.CallableRaisingIOE;
+
import static java.util.Objects.requireNonNull;
import static org.apache.commons.lang3.StringUtils.isNotEmpty;
import static org.apache.hadoop.fs.VectoredReadUtils.isOrderedDisjoint;
@@ -125,14 +126,9 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
*/
private volatile boolean closed;
/**
- * wrappedStream is associated with an object (instance of S3Object). When
- * the object is garbage collected, the associated wrappedStream will be
- * closed. Keep a reference to this object to prevent the wrapperStream
- * still in use from being closed unexpectedly due to garbage collection.
- * See HADOOP-17338 for details.
+ * Input stream returned by a getObject call.
*/
- private S3Object object;
- private S3ObjectInputStream wrappedStream;
+ private ResponseInputStream<GetObjectResponse> wrappedStream;
private final S3AReadOpContext context;
private final InputStreamCallbacks client;
@@ -271,28 +267,22 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
uri, reason, targetPos, contentRangeFinish, length, pos, nextReadPos,
inputPolicy);
+ GetObjectRequest request = client.newGetRequestBuilder(key)
+ .range(S3AUtils.formatRange(targetPos, contentRangeFinish - 1))
+ .applyMutation(changeTracker::maybeApplyConstraint)
+ .build();
long opencount = streamStatistics.streamOpened();
- GetObjectRequest request = client.newGetRequest(key)
- .withRange(targetPos, contentRangeFinish - 1);
String operation = opencount == 0 ? OPERATION_OPEN : OPERATION_REOPEN;
String text = String.format("%s %s at %d",
operation, uri, targetPos);
- changeTracker.maybeApplyConstraint(request);
-
- object = onceTrackingDuration(text, uri,
+ wrappedStream = onceTrackingDuration(text, uri,
streamStatistics.initiateGetRequest(), () ->
client.getObject(request));
-
- changeTracker.processResponse(object, operation,
+ changeTracker.processResponse(wrappedStream.response(), operation,
targetPos);
- wrappedStream = object.getObjectContent();
- contentRangeStart = targetPos;
- if (wrappedStream == null) {
- throw new PathIOException(uri,
- "Null IO stream from " + operation + " of (" + reason + ") ");
- }
+ contentRangeStart = targetPos;
this.pos = targetPos;
}
@@ -505,14 +495,15 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
*/
@Retries.OnceTranslated
private void onReadFailure(IOException ioe, boolean forceAbort) {
+ GetObjectResponse objectResponse = wrappedStream == null ? null : wrappedStream.response();
if (LOG.isDebugEnabled()) {
LOG.debug("Got exception while trying to read from stream {}, " +
"client: {} object: {}, trying to recover: ",
- uri, client, object, ioe);
+ uri, client, objectResponse, ioe);
} else {
LOG.info("Got exception while trying to read from stream {}, " +
"client: {} object: {}, trying to recover: " + ioe,
- uri, client, object);
+ uri, client, objectResponse);
}
streamStatistics.readException();
closeStream("failure recovery", forceAbort, false);
@@ -672,7 +663,6 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
CompletableFuture<Boolean> operation;
SDKStreamDrainer drainer = new SDKStreamDrainer(
uri,
- object,
wrappedStream,
shouldAbort,
(int) remaining,
@@ -694,7 +684,6 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
// either the stream is closed in the blocking call or the async call is
// submitted with its own copy of the references
wrappedStream = null;
- object = null;
return operation;
}
@@ -910,23 +899,19 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
private void readCombinedRangeAndUpdateChildren(CombinedFileRange combinedFileRange,
IntFunction<ByteBuffer> allocate) {
LOG.debug("Start reading combined range {} from path {} ", combinedFileRange, pathStr);
- // This reference must be kept till all buffers are populated as this is a
- // finalizable object which closes the internal stream when gc triggers.
- S3Object objectRange = null;
- S3ObjectInputStream objectContent = null;
+ ResponseInputStream<GetObjectResponse> rangeContent = null;
try {
- objectRange = getS3ObjectAndValidateNotNull("readCombinedFileRange",
+ rangeContent = getS3ObjectInputStream("readCombinedFileRange",
combinedFileRange.getOffset(),
combinedFileRange.getLength());
- objectContent = objectRange.getObjectContent();
- populateChildBuffers(combinedFileRange, objectContent, allocate);
+ populateChildBuffers(combinedFileRange, rangeContent, allocate);
} catch (Exception ex) {
LOG.debug("Exception while reading a range {} from path {} ", combinedFileRange, pathStr, ex);
for(FileRange child : combinedFileRange.getUnderlying()) {
child.getData().completeExceptionally(ex);
}
} finally {
- IOUtils.cleanupWithLogger(LOG, objectRange, objectContent);
+ IOUtils.cleanupWithLogger(LOG, rangeContent);
}
LOG.debug("Finished reading range {} from path {} ", combinedFileRange, pathStr);
}
@@ -939,7 +924,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
* @throws IOException any IOE.
*/
private void populateChildBuffers(CombinedFileRange combinedFileRange,
- S3ObjectInputStream objectContent,
+ InputStream objectContent,
IntFunction<ByteBuffer> allocate) throws IOException {
// If the combined file range just contains a single child
// range, we only have to fill that one child buffer else
@@ -971,7 +956,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
* @param drainQuantity how many bytes to drain.
* @throws IOException any IOE.
*/
- private void drainUnnecessaryData(S3ObjectInputStream objectContent, long drainQuantity)
+ private void drainUnnecessaryData(InputStream objectContent, long drainQuantity)
throws IOException {
int drainBytes = 0;
int readCount;
@@ -1013,28 +998,24 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
*/
private void readSingleRange(FileRange range, ByteBuffer buffer) {
LOG.debug("Start reading range {} from path {} ", range, pathStr);
- // This reference must be kept till all buffers are populated as this is a
- // finalizable object which closes the internal stream when gc triggers.
- S3Object objectRange = null;
- S3ObjectInputStream objectContent = null;
+ ResponseInputStream<GetObjectResponse> objectRange = null;
try {
long position = range.getOffset();
int length = range.getLength();
- objectRange = getS3ObjectAndValidateNotNull("readSingleRange", position, length);
- objectContent = objectRange.getObjectContent();
- populateBuffer(length, buffer, objectContent);
+ objectRange = getS3ObjectInputStream("readSingleRange", position, length);
+ populateBuffer(length, buffer, objectRange);
range.getData().complete(buffer);
} catch (Exception ex) {
LOG.warn("Exception while reading a range {} from path {} ", range, pathStr, ex);
range.getData().completeExceptionally(ex);
} finally {
- IOUtils.cleanupWithLogger(LOG, objectRange, objectContent);
+ IOUtils.cleanupWithLogger(LOG, objectRange);
}
LOG.debug("Finished reading range {} from path {} ", range, pathStr);
}
/**
- * Get the s3 object for S3 server for a specified range.
+ * Get the s3 object input stream for S3 server for a specified range.
* Also checks if the vectored io operation has been stopped before and after
* the http get request such that we don't waste time populating the buffers.
* @param operationName name of the operation for which get object on S3 is called.
@@ -1043,15 +1024,11 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
* @return result s3 object.
* @throws IOException exception if any.
*/
- private S3Object getS3ObjectAndValidateNotNull(final String operationName,
- final long position,
- final int length) throws IOException {
+ private ResponseInputStream<GetObjectResponse> getS3ObjectInputStream(
+ final String operationName, final long position, final int length) throws IOException {
checkIfVectoredIOStopped();
- S3Object objectRange = getS3Object(operationName, position, length);
- if (objectRange.getObjectContent() == null) {
- throw new PathIOException(uri,
- "Null IO stream received during " + operationName);
- }
+ ResponseInputStream<GetObjectResponse> objectRange =
+ getS3Object(operationName, position, length);
checkIfVectoredIOStopped();
return objectRange;
}
@@ -1066,7 +1043,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
*/
private void populateBuffer(int length,
ByteBuffer buffer,
- S3ObjectInputStream objectContent) throws IOException {
+ InputStream objectContent) throws IOException {
if (buffer.isDirect()) {
VectoredReadUtils.readInDirectBuffer(length, buffer,
@@ -1091,7 +1068,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
* @param length number of bytes to fill in dest.
* @throws IOException any IOE.
*/
- private void readByteArray(S3ObjectInputStream objectContent,
+ private void readByteArray(InputStream objectContent,
byte[] dest,
int offset,
int length) throws IOException {
@@ -1118,13 +1095,16 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
* @return S3Object result s3 object.
* @throws IOException exception if any.
*/
- private S3Object getS3Object(String operationName, long position,
- int length) throws IOException {
- final GetObjectRequest request = client.newGetRequest(key)
- .withRange(position, position + length - 1);
- changeTracker.maybeApplyConstraint(request);
+ private ResponseInputStream<GetObjectResponse> getS3Object(String operationName,
+ long position,
+ int length)
+ throws IOException {
+ final GetObjectRequest request = client.newGetRequestBuilder(key)
+ .range(S3AUtils.formatRange(position, position + length - 1))
+ .applyMutation(changeTracker::maybeApplyConstraint)
+ .build();
DurationTracker tracker = streamStatistics.initiateGetRequest();
- S3Object objectRange;
+ ResponseInputStream<GetObjectResponse> objectRange;
Invoker invoker = context.getReadInvoker();
try {
objectRange = invoker.retry(operationName, pathStr, true,
@@ -1139,7 +1119,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
} finally {
tracker.close();
}
- changeTracker.processResponse(objectRange, operationName,
+ changeTracker.processResponse(objectRange.response(), operationName,
position);
return objectRange;
}
@@ -1293,11 +1273,11 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
public interface InputStreamCallbacks extends Closeable {
/**
- * Create a GET request.
+ * Create a GET request builder.
* @param key object key
- * @return the request
+ * @return the request builder
*/
- GetObjectRequest newGetRequest(String key);
+ GetObjectRequest.Builder newGetRequestBuilder(String key);
/**
* Execute the request.
@@ -1305,7 +1285,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
* @return the response
*/
@Retries.OnceRaw
- S3Object getObject(GetObjectRequest request);
+ ResponseInputStream<GetObjectResponse> getObject(GetObjectRequest request);
/**
* Submit some asynchronous work, for example, draining a stream.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java
new file mode 100644
index 000000000000..23c4d3501206
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.io.IOException;
+import java.nio.file.AccessDeniedException;
+
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.HeadBucketResponse;
+import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.store.audit.AuditEntryPoint;
+
+/**
+ * This is an unstable interface for access to S3A Internal state, S3 operations
+ * and the S3 client connector itself.
+ */
+@InterfaceStability.Unstable
+@InterfaceAudience.LimitedPrivate("testing/diagnostics")
+public interface S3AInternals {
+
+ /**
+ * Returns the S3 client used by this filesystem.
+ * Will log at debug.
+ * <p>
+ * <i>Warning</i>
+ * This bypasses core S3A operations, including auditing.
+ * It is intended for use in testing, diagnostics and for accessing
+ * operations not available through the S3A connector itself.
+ * <p>
+ * Unless audit spans are created through the S3AFileSystem, make
+ * sure that {@code fs.s3a.audit.reject.out.of.span.operations} is
+ * set to false.
+ * <p>
+ * Mocking note: this is the same S3Client as is used by the owning
+ * filesystem; changes to this client will be reflected by changes
+ * in the behavior of that filesystem.
+ * @param reason a justification for requesting access.
+ * @return S3Client
+ */
+ S3Client getAmazonS3Client(String reason);
+
+ /**
+ * Get the region of a bucket.
+ * Invoked from StoreContext; consider an entry point.
+ * @return the region in which a bucket is located
+ * @throws AccessDeniedException if the caller lacks permission.
+ * @throws IOException on any failure.
+ */
+ @Retries.RetryTranslated
+ @AuditEntryPoint
+ String getBucketLocation() throws IOException;
+
+ /**
+ * Get the region of a bucket; fixing up the region so it can be used
+ * in the builders of other AWS clients.
+ * Requires the caller to have the AWS role permission
+ * {@code s3:GetBucketLocation}.
+ * Retry policy: retrying, translated.
+ * @param bucketName the name of the bucket
+ * @return the region in which a bucket is located
+ * @throws AccessDeniedException if the caller lacks permission.
+ * @throws IOException on any failure.
+ */
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ String getBucketLocation(String bucketName) throws IOException;
+
+ /**
+ * Low-level call to get at the object metadata.
+ * Auditing: An audit entry point.
+ * @param path path to the object. This will be qualified.
+ * @return metadata
+ * @throws IOException IO and object access problems.
+ */
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ HeadObjectResponse getObjectMetadata(Path path) throws IOException;
+
+ /**
+ * Get a shared copy of the AWS credentials, with its reference
+ * counter updated.
+ * Caller is required to call {@code close()} on this after
+ * they have finished using it.
+ * @param purpose what is this for? This is for logging
+ * @return a reference to shared credentials.
+ */
+ AWSCredentialProviderList shareCredentials(String purpose);
+
+ /**
+ * Request bucket metadata.
+ * @return the metadata
+ * @throws UnknownStoreException the bucket is absent
+ * @throws IOException any other problem talking to S3
+ */
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ HeadBucketResponse getBucketMetadata() throws IOException;
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
index 528a99f5e096..fdb4591476c5 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
@@ -30,7 +30,8 @@ import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
-import com.amazonaws.AmazonClientException;
+import software.amazon.awssdk.core.exception.SdkException;
+import org.apache.hadoop.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,7 +43,7 @@ import org.apache.hadoop.fs.s3a.auth.NoAuthWithAWSException;
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.net.ConnectTimeoutException;
-import org.apache.hadoop.util.Preconditions;
+
import static org.apache.hadoop.io.retry.RetryPolicies.*;
@@ -68,9 +69,9 @@ import static org.apache.hadoop.fs.s3a.Constants.*;
*
* The retry policy is all built around that of the normal IO exceptions,
* particularly those extracted from
- * {@link S3AUtils#translateException(String, Path, AmazonClientException)}.
+ * {@link S3AUtils#translateException(String, Path, SdkException)}.
* Because the {@link #shouldRetry(Exception, int, int, boolean)} method
- * does this translation if an {@code AmazonClientException} is processed,
+ * does this translation if an {@code SdkException} is processed,
* the policy defined for the IOEs also applies to the original exceptions.
*
* Put differently: this retry policy aims to work for handlers of the
@@ -111,6 +112,11 @@ public class S3ARetryPolicy implements RetryPolicy {
*/
protected final RetryPolicy connectivityFailure;
+ /**
+ * Handling of AWSClientIOException and subclasses.
+ */
+ protected final RetryPolicy retryAwsClientExceptions;
+
/**
* Instantiate.
* @param conf configuration to read.
@@ -138,6 +144,11 @@ public class S3ARetryPolicy implements RetryPolicy {
retryIdempotentCalls = new FailNonIOEs(
new IdempotencyRetryFilter(baseExponentialRetry));
+ // retry on AWSClientIOException and possibly subclasses;
+ // See: HADOOP-18871. S3ARetryPolicy to use sdk exception retryable() if it is valid
+ // currently the normal retryIdempotentCalls policy is used.
+ retryAwsClientExceptions = retryIdempotentCalls;
+
// and a separate policy for throttle requests, which are considered
// repeatable, even for non-idempotent calls, as the service
// rejected the call entirely
@@ -223,9 +234,11 @@ public class S3ARetryPolicy implements RetryPolicy {
// server didn't respond.
policyMap.put(AWSNoResponseException.class, retryIdempotentCalls);
+ // use specific retry policy for aws client exceptions
+ policyMap.put(AWSClientIOException.class, retryAwsClientExceptions);
+ policyMap.put(AWSServiceIOException.class, retryAwsClientExceptions);
+
// other operations
- policyMap.put(AWSClientIOException.class, retryIdempotentCalls);
- policyMap.put(AWSServiceIOException.class, retryIdempotentCalls);
policyMap.put(AWSS3IOException.class, retryIdempotentCalls);
policyMap.put(SocketTimeoutException.class, retryIdempotentCalls);
@@ -242,11 +255,10 @@ public class S3ARetryPolicy implements RetryPolicy {
boolean idempotent) throws Exception {
Preconditions.checkArgument(exception != null, "Null exception");
Exception ex = exception;
- if (exception instanceof AmazonClientException) {
- // uprate the amazon client exception for the purpose of exception
+ if (exception instanceof SdkException) {
+ // update the sdk exception for the purpose of exception
// processing.
- ex = S3AUtils.translateException("", "",
- (AmazonClientException) exception);
+ ex = S3AUtils.translateException("", "", (SdkException) exception);
}
return retryPolicy.shouldRetry(ex, retries, failovers, idempotent);
}
@@ -315,4 +327,28 @@ public class S3ARetryPolicy implements RetryPolicy {
}
}
+ /**
+ * Policy where AWS SDK exceptions are retried if they state that they are retryable.
+ * See HADOOP-18871. S3ARetryPolicy to use sdk exception retryable() if it is valid.
+ */
+ private static final class RetryFromAWSClientExceptionPolicy implements RetryPolicy {
+
+ private final RetryPolicy next;
+
+ private RetryFromAWSClientExceptionPolicy(RetryPolicy next) {
+ this.next = next;
+ }
+
+ @Override
+ public RetryAction shouldRetry(Exception e,
+ int retries,
+ int failovers,
+ boolean isIdempotentOrAtMostOnce) throws Exception {
+ return
+ e instanceof AWSClientIOException ?
+ next.shouldRetry(e, retries, failovers, ((AWSClientIOException)e).retryable())
+ : RetryAction.FAIL;
+ }
+ }
+
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index dc0c211fcab5..62be975b656f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -18,20 +18,12 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.AbortedException;
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.AmazonServiceException;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.Protocol;
-import com.amazonaws.SdkBaseException;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
-import com.amazonaws.retry.RetryUtils;
-import com.amazonaws.services.s3.model.AmazonS3Exception;
-import com.amazonaws.services.s3.model.MultiObjectDeleteException;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
+import software.amazon.awssdk.core.exception.AbortedException;
+import software.amazon.awssdk.core.exception.SdkException;
+import software.amazon.awssdk.core.retry.RetryUtils;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.model.S3Object;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -44,16 +36,12 @@ import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.util.functional.RemoteIterators;
-import org.apache.hadoop.fs.s3a.audit.AuditFailureException;
-import org.apache.hadoop.fs.s3a.audit.AuditIntegration;
import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets;
-import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider;
-import org.apache.hadoop.fs.s3a.impl.NetworkBinding;
-import org.apache.hadoop.fs.s3a.impl.V2Migration;
+import org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteException;
import org.apache.hadoop.fs.s3native.S3xLoginHelper;
import org.apache.hadoop.net.ConnectTimeoutException;
import org.apache.hadoop.security.ProviderUtils;
-import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
import org.slf4j.Logger;
@@ -74,23 +62,24 @@ import java.net.SocketTimeoutException;
import java.net.URI;
import java.nio.file.AccessDeniedException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
import java.util.Date;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
+import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
import static org.apache.commons.lang3.StringUtils.isEmpty;
+import static org.apache.hadoop.fs.s3a.AWSCredentialProviderList.maybeTranslateCredentialException;
import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.audit.AuditIntegration.maybeTranslateAuditException;
import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isUnknownBucket;
-import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH;
-import static org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteSupport.translateDeleteException;
+import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.instantiationException;
+import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.isAbstract;
+import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.isNotInstanceOf;
+import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.unsupportedConstructor;
+import static org.apache.hadoop.fs.s3a.impl.InternalConstants.*;
import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteIterator;
@@ -102,13 +91,7 @@ import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteI
public final class S3AUtils {
private static final Logger LOG = LoggerFactory.getLogger(S3AUtils.class);
- static final String CONSTRUCTOR_EXCEPTION = "constructor exception";
- static final String INSTANTIATION_EXCEPTION
- = "instantiation exception";
- static final String NOT_AWS_PROVIDER =
- "does not implement AWSCredentialsProvider";
- static final String ABSTRACT_PROVIDER =
- "is abstract and therefore cannot be created";
+
static final String ENDPOINT_KEY = "Endpoint";
/** Filesystem is closed; kept here to keep the errors close. */
@@ -143,21 +126,13 @@ public final class S3AUtils {
private static final String BUCKET_PATTERN = FS_S3A_BUCKET_PREFIX + "%s.%s";
- /**
- * Error message when the AWS provider list built up contains a forbidden
- * entry.
- */
- @VisibleForTesting
- public static final String E_FORBIDDEN_AWS_PROVIDER
- = "AWS provider class cannot be used";
-
private S3AUtils() {
}
/**
* Translate an exception raised in an operation into an IOException.
* The specific type of IOException depends on the class of
- * {@link AmazonClientException} passed in, and any status codes included
+ * {@link SdkException} passed in, and any status codes included
* in the operation. That is: HTTP error codes are examined and can be
* used to build a more specific response.
*
@@ -170,14 +145,14 @@ public final class S3AUtils {
*/
public static IOException translateException(String operation,
Path path,
- AmazonClientException exception) {
+ SdkException exception) {
return translateException(operation, path.toString(), exception);
}
/**
* Translate an exception raised in an operation into an IOException.
* The specific type of IOException depends on the class of
- * {@link AmazonClientException} passed in, and any status codes included
+ * {@link SdkException} passed in, and any status codes included
* in the operation. That is: HTTP error codes are examined and can be
* used to build a more specific response.
* @param operation operation
@@ -188,12 +163,14 @@ public final class S3AUtils {
@SuppressWarnings("ThrowableInstanceNeverThrown")
public static IOException translateException(@Nullable String operation,
String path,
- SdkBaseException exception) {
+ SdkException exception) {
String message = String.format("%s%s: %s",
operation,
StringUtils.isNotEmpty(path)? (" on " + path) : "",
exception);
- if (!(exception instanceof AmazonServiceException)) {
+ if (!(exception instanceof AwsServiceException)) {
+ // exceptions raised client-side: connectivity, auth, network problems...
+
Exception innerCause = containsInterruptedException(exception);
if (innerCause != null) {
// interrupted IO, or a socket exception underneath that class
@@ -205,57 +182,60 @@ public final class S3AUtils {
}
// if the exception came from the auditor, hand off translation
// to it.
- if (exception instanceof AuditFailureException) {
- return AuditIntegration.translateAuditException(path, (AuditFailureException) exception);
+ IOException ioe = maybeTranslateAuditException(path, exception);
+ if (ioe != null) {
+ return ioe;
}
- if (exception instanceof CredentialInitializationException) {
- // the exception raised by AWSCredentialProvider list if the
- // credentials were not accepted,
- return (AccessDeniedException)new AccessDeniedException(path, null,
- exception.toString()).initCause(exception);
+ ioe = maybeTranslateCredentialException(path, exception);
+ if (ioe != null) {
+ return ioe;
+ } else {
+ // no custom handling.
+ return new AWSClientIOException(message, exception);
}
- return new AWSClientIOException(message, exception);
} else {
+ // "error response returned by an S3 or other service."
+ // These contain more details and should be translated based
+ // on the HTTP status code and other details.
IOException ioe;
- AmazonServiceException ase = (AmazonServiceException) exception;
+ AwsServiceException ase = (AwsServiceException) exception;
// this exception is non-null if the service exception is an s3 one
- AmazonS3Exception s3Exception = ase instanceof AmazonS3Exception
- ? (AmazonS3Exception) ase
+ S3Exception s3Exception = ase instanceof S3Exception
+ ? (S3Exception) ase
: null;
- int status = ase.getStatusCode();
- message = message + ":" + ase.getErrorCode();
+ int status = ase.statusCode();
+ if (ase.awsErrorDetails() != null) {
+ message = message + ":" + ase.awsErrorDetails().errorCode();
+ }
switch (status) {
- case 301:
- case 307:
+ case SC_301_MOVED_PERMANENTLY:
+ case SC_307_TEMPORARY_REDIRECT:
if (s3Exception != null) {
- if (s3Exception.getAdditionalDetails() != null &&
- s3Exception.getAdditionalDetails().containsKey(ENDPOINT_KEY)) {
- message = String.format("Received permanent redirect response to "
- + "endpoint %s. This likely indicates that the S3 endpoint "
- + "configured in %s does not match the AWS region containing "
- + "the bucket.",
- s3Exception.getAdditionalDetails().get(ENDPOINT_KEY), ENDPOINT);
- }
+ message = String.format("Received permanent redirect response to "
+ + "region %s. This likely indicates that the S3 region "
+ + "configured in %s does not match the AWS region containing " + "the bucket.",
+ s3Exception.awsErrorDetails().sdkHttpResponse().headers().get(BUCKET_REGION_HEADER),
+ AWS_REGION);
ioe = new AWSRedirectException(message, s3Exception);
} else {
ioe = new AWSRedirectException(message, ase);
}
break;
- case 400:
+ case SC_400_BAD_REQUEST:
ioe = new AWSBadRequestException(message, ase);
break;
// permissions
- case 401:
- case 403:
+ case SC_401_UNAUTHORIZED:
+ case SC_403_FORBIDDEN:
ioe = new AccessDeniedException(path, null, message);
ioe.initCause(ase);
break;
// the object isn't there
- case 404:
+ case SC_404_NOT_FOUND:
if (isUnknownBucket(ase)) {
// this is a missing bucket
ioe = new UnknownStoreException(path, message, ase);
@@ -268,20 +248,20 @@ public final class S3AUtils {
// this also surfaces sometimes and is considered to
// be ~ a not found exception.
- case 410:
+ case SC_410_GONE:
ioe = new FileNotFoundException(message);
ioe.initCause(ase);
break;
// method not allowed; seen on S3 Select.
// treated as a bad request
- case 405:
+ case SC_405_METHOD_NOT_ALLOWED:
ioe = new AWSBadRequestException(message, s3Exception);
break;
// out of range. This may happen if an object is overwritten with
// a shorter one while it is being read.
- case 416:
+ case SC_416_RANGE_NOT_SATISFIABLE:
ioe = new EOFException(message);
ioe.initCause(ase);
break;
@@ -289,26 +269,26 @@ public final class S3AUtils {
// this has surfaced as a "no response from server" message.
// so rare we haven't replicated it.
// Treating as an idempotent proxy error.
- case 443:
- case 444:
+ case SC_443_NO_RESPONSE:
+ case SC_444_NO_RESPONSE:
ioe = new AWSNoResponseException(message, ase);
break;
// throttling
- case 503:
+ case SC_503_SERVICE_UNAVAILABLE:
ioe = new AWSServiceThrottledException(message, ase);
break;
// internal error
- case 500:
+ case SC_500_INTERNAL_SERVER_ERROR:
ioe = new AWSStatus500Exception(message, ase);
break;
- case 200:
+ case SC_200_OK:
if (exception instanceof MultiObjectDeleteException) {
// failure during a bulk delete
- return translateDeleteException(message,
- (MultiObjectDeleteException) exception);
+ return ((MultiObjectDeleteException) exception)
+ .translateException(message);
}
// other 200: FALL THROUGH
@@ -334,10 +314,35 @@ public final class S3AUtils {
public static IOException extractException(String operation,
String path,
ExecutionException ee) {
+ return convertExceptionCause(operation, path, ee.getCause());
+ }
+
+ /**
+ * Extract an exception from a failed future, and convert to an IOE.
+ * @param operation operation which failed
+ * @param path path operated on (may be null)
+ * @param ce completion exception
+ * @return an IOE which can be thrown
+ */
+ public static IOException extractException(String operation,
+ String path,
+ CompletionException ce) {
+ return convertExceptionCause(operation, path, ce.getCause());
+ }
+
+ /**
+ * Convert the cause of a concurrent exception to an IOE.
+ * @param operation operation which failed
+ * @param path path operated on (may be null)
+ * @param cause cause of a concurrent exception
+ * @return an IOE which can be thrown
+ */
+ private static IOException convertExceptionCause(String operation,
+ String path,
+ Throwable cause) {
IOException ioe;
- Throwable cause = ee.getCause();
- if (cause instanceof AmazonClientException) {
- ioe = translateException(operation, path, (AmazonClientException) cause);
+ if (cause instanceof SdkException) {
+ ioe = translateException(operation, path, (SdkException) cause);
} else if (cause instanceof IOException) {
ioe = (IOException) cause;
} else {
@@ -375,7 +380,7 @@ public final class S3AUtils {
* @return an IOE which can be rethrown
*/
private static InterruptedIOException translateInterruptedException(
- SdkBaseException exception,
+ SdkException exception,
final Exception innerCause,
String message) {
InterruptedIOException ioe;
@@ -386,6 +391,7 @@ public final class S3AUtils {
if (name.endsWith(".ConnectTimeoutException")
|| name.endsWith(".ConnectionPoolTimeoutException")
|| name.endsWith("$ConnectTimeoutException")) {
+ // TODO: review in v2
// TCP connection http timeout from the shaded or unshaded filenames
// com.amazonaws.thirdparty.apache.http.conn.ConnectTimeoutException
ioe = new ConnectTimeoutException(message);
@@ -409,10 +415,10 @@ public final class S3AUtils {
*/
public static boolean isThrottleException(Exception ex) {
return ex instanceof AWSServiceThrottledException
- || (ex instanceof AmazonServiceException
- && 503 == ((AmazonServiceException)ex).getStatusCode())
- || (ex instanceof SdkBaseException
- && RetryUtils.isThrottlingException((SdkBaseException) ex));
+ || (ex instanceof AwsServiceException
+ && 503 == ((AwsServiceException)ex).statusCode())
+ || (ex instanceof SdkException
+ && RetryUtils.isThrottlingException((SdkException) ex));
}
/**
@@ -422,7 +428,8 @@ public final class S3AUtils {
* @param ex exception
* @return true if this is believed to be a sign the connection was broken.
*/
- public static boolean isMessageTranslatableToEOF(SdkBaseException ex) {
+ public static boolean isMessageTranslatableToEOF(SdkException ex) {
+ // TODO: review in v2
return ex.toString().contains(EOF_MESSAGE_IN_XML_PARSER) ||
ex.toString().contains(EOF_READ_DIFFERENT_LENGTH);
}
@@ -432,47 +439,26 @@ public final class S3AUtils {
* @param e exception
* @return string details
*/
- public static String stringify(AmazonServiceException e) {
+ public static String stringify(AwsServiceException e) {
StringBuilder builder = new StringBuilder(
- String.format("%s: %s error %d: %s; %s%s%n",
- e.getErrorType(),
- e.getServiceName(),
- e.getStatusCode(),
- e.getErrorCode(),
- e.getErrorMessage(),
- (e.isRetryable() ? " (retryable)": "")
+ String.format("%s error %d: %s; %s%s%n",
+ e.awsErrorDetails().serviceName(),
+ e.statusCode(),
+ e.awsErrorDetails().errorCode(),
+ e.awsErrorDetails().errorMessage(),
+ (e.retryable() ? " (retryable)": "")
));
- String rawResponseContent = e.getRawResponseContent();
+ String rawResponseContent = e.awsErrorDetails().rawResponse().asUtf8String();
if (rawResponseContent != null) {
builder.append(rawResponseContent);
}
return builder.toString();
}
- /**
- * Get low level details of an amazon exception for logging; multi-line.
- * @param e exception
- * @return string details
- */
- public static String stringify(AmazonS3Exception e) {
- // get the low level details of an exception,
- StringBuilder builder = new StringBuilder(
- stringify((AmazonServiceException) e));
- Map<String, String> details = e.getAdditionalDetails();
- if (details != null) {
- builder.append('\n');
- for (Map.Entry<String, String> d : details.entrySet()) {
- builder.append(d.getKey()).append('=')
- .append(d.getValue()).append('\n');
- }
- }
- return builder.toString();
- }
-
/**
* Create a files status instance from a listing.
* @param keyPath path to entry
- * @param summary summary from AWS
+ * @param s3Object s3Object entry
* @param blockSize block size to declare.
* @param owner owner of the file
* @param eTag S3 object eTag or null if unavailable
@@ -481,20 +467,20 @@ public final class S3AUtils {
* @return a status entry
*/
public static S3AFileStatus createFileStatus(Path keyPath,
- S3ObjectSummary summary,
+ S3Object s3Object,
long blockSize,
String owner,
String eTag,
String versionId,
boolean isCSEEnabled) {
- long size = summary.getSize();
+ long size = s3Object.size();
// check if cse is enabled; strip out constant padding length.
if (isCSEEnabled && size >= CSE_PADDING_LENGTH) {
size -= CSE_PADDING_LENGTH;
}
return createFileStatus(keyPath,
- objectRepresentsDirectory(summary.getKey()),
- size, summary.getLastModified(), blockSize, owner, eTag, versionId);
+ objectRepresentsDirectory(s3Object.key()),
+ size, Date.from(s3Object.lastModified()), blockSize, owner, eTag, versionId);
}
/**
@@ -556,114 +542,7 @@ public final class S3AUtils {
}
/**
- * The standard AWS provider list for AWS connections.
- */
- @SuppressWarnings("deprecation")
- public static final List<Class<?>>
- STANDARD_AWS_PROVIDERS = Collections.unmodifiableList(
- Arrays.asList(
- TemporaryAWSCredentialsProvider.class,
- SimpleAWSCredentialsProvider.class,
- EnvironmentVariableCredentialsProvider.class,
- IAMInstanceCredentialsProvider.class));
-
- /**
- * Create the AWS credentials from the providers, the URI and
- * the key {@link Constants#AWS_CREDENTIALS_PROVIDER} in the configuration.
- * @param binding Binding URI -may be null
- * @param conf filesystem configuration
- * @return a credentials provider list
- * @throws IOException Problems loading the providers (including reading
- * secrets from credential files).
- */
- public static AWSCredentialProviderList createAWSCredentialProviderSet(
- @Nullable URI binding,
- Configuration conf) throws IOException {
- // this will reject any user:secret entries in the URI
- S3xLoginHelper.rejectSecretsInURIs(binding);
- AWSCredentialProviderList credentials =
- buildAWSProviderList(binding,
- conf,
- AWS_CREDENTIALS_PROVIDER,
- STANDARD_AWS_PROVIDERS,
- new HashSet<>());
- // make sure the logging message strips out any auth details
- LOG.debug("For URI {}, using credentials {}",
- binding, credentials);
- return credentials;
- }
-
- /**
- * Load list of AWS credential provider/credential provider factory classes.
- * @param conf configuration
- * @param key key
- * @param defaultValue list of default values
- * @return the list of classes, possibly empty
- * @throws IOException on a failure to load the list.
- */
- public static List<Class<?>> loadAWSProviderClasses(Configuration conf,
- String key,
- Class<?>... defaultValue) throws IOException {
- try {
- return Arrays.asList(conf.getClasses(key, defaultValue));
- } catch (RuntimeException e) {
- Throwable c = e.getCause() != null ? e.getCause() : e;
- throw new IOException("From option " + key + ' ' + c, c);
- }
- }
-
- /**
- * Load list of AWS credential provider/credential provider factory classes;
- * support a forbidden list to prevent loops, mandate full secrets, etc.
- * @param binding Binding URI -may be null
- * @param conf configuration
- * @param key key
- * @param forbidden a possibly empty set of forbidden classes.
- * @param defaultValues list of default providers.
- * @return the list of classes, possibly empty
- * @throws IOException on a failure to load the list.
- */
- public static AWSCredentialProviderList buildAWSProviderList(
- @Nullable final URI binding,
- final Configuration conf,
- final String key,
- final List<Class<?>> defaultValues,
- final Set<Class<?>> forbidden) throws IOException {
-
- // build up the base provider
- List<Class<?>> awsClasses = loadAWSProviderClasses(conf,
- key,
- defaultValues.toArray(new Class[defaultValues.size()]));
- // and if the list is empty, switch back to the defaults.
- // this is to address the issue that configuration.getClasses()
- // doesn't return the default if the config value is just whitespace.
- if (awsClasses.isEmpty()) {
- awsClasses = defaultValues;
- }
- // iterate through, checking for blacklists and then instantiating
- // each provider
- AWSCredentialProviderList providers = new AWSCredentialProviderList();
- for (Class<?> aClass : awsClasses) {
-
- // List of V1 credential providers that will be migrated with V2 upgrade
- if (!Arrays.asList("EnvironmentVariableCredentialsProvider",
- "EC2ContainerCredentialsProviderWrapper", "InstanceProfileCredentialsProvider")
- .contains(aClass.getSimpleName()) && aClass.getName().contains(AWS_AUTH_CLASS_PREFIX)) {
- V2Migration.v1ProviderReferenced(aClass.getName());
- }
-
- if (forbidden.contains(aClass)) {
- throw new IOException(E_FORBIDDEN_AWS_PROVIDER
- + " in option " + key + ": " + aClass);
- }
- providers.add(createAWSCredentialProvider(conf,
- aClass, binding));
- }
- return providers;
- }
-
- /**
- * Create an AWS credential provider from its class by using reflection. The
+ * Creates an instance of a class using reflection. The
* class must implement one of the following means of construction, which are
* attempted in order:
*
@@ -672,92 +551,87 @@ public final class S3AUtils {
* org.apache.hadoop.conf.Configuration</li>
* <li>a public constructor accepting
* org.apache.hadoop.conf.Configuration</li>
- * <li>a public static method named getInstance that accepts no
+ * <li>a public static method named as per methodName, that accepts no
* arguments and returns an instance of
- * com.amazonaws.auth.AWSCredentialsProvider, or</li>
+ * specified type, or</li>
* <li>a public default constructor.</li>
* </ol>
*
+ * @param className name of class for which instance is to be created
* @param conf configuration
- * @param credClass credential class
* @param uri URI of the FS
- * @return the instantiated class
- * @throws IOException on any instantiation failure.
+ * @param interfaceImplemented interface that this class implements
+ * @param methodName name of factory method to be invoked
+ * @param configKey config key under which this class is specified
+ * @param <InstanceT> Instance of class
+ * @return instance of the specified class
+ * @throws IOException on any problem
*/
- private static AWSCredentialsProvider createAWSCredentialProvider(
+ @SuppressWarnings("unchecked")
+ public static <InstanceT> InstanceT getInstanceFromReflection(String className,
Configuration conf,
- Class<?> credClass,
- @Nullable URI uri) throws IOException {
- AWSCredentialsProvider credentials = null;
- String className = credClass.getName();
- if (!AWSCredentialsProvider.class.isAssignableFrom(credClass)) {
- throw new IOException("Class " + credClass + " " + NOT_AWS_PROVIDER);
- }
- if (Modifier.isAbstract(credClass.getModifiers())) {
- throw new IOException("Class " + credClass + " " + ABSTRACT_PROVIDER);
- }
- LOG.debug("Credential provider class is {}", className);
-
+ @Nullable URI uri,
+ Class<? extends InstanceT> interfaceImplemented,
+ String methodName,
+ String configKey) throws IOException {
try {
- // new X(uri, conf)
- Constructor cons = getConstructor(credClass, URI.class,
- Configuration.class);
- if (cons != null) {
- credentials = (AWSCredentialsProvider)cons.newInstance(uri, conf);
- return credentials;
+ Class<?> instanceClass = S3AUtils.class.getClassLoader().loadClass(className);
+ if (Modifier.isAbstract(instanceClass.getModifiers())) {
+ throw isAbstract(uri, className, configKey);
}
- // new X(conf)
- cons = getConstructor(credClass, Configuration.class);
- if (cons != null) {
- credentials = (AWSCredentialsProvider)cons.newInstance(conf);
- return credentials;
+ if (!interfaceImplemented.isAssignableFrom(instanceClass)) {
+ throw isNotInstanceOf(uri, className, interfaceImplemented.getName(), configKey);
+
}
+ Constructor cons;
+ if (conf != null) {
+ // new X(uri, conf)
+ cons = getConstructor(instanceClass, URI.class, Configuration.class);
- // X.getInstance()
- Method factory = getFactoryMethod(credClass, AWSCredentialsProvider.class,
- "getInstance");
+ if (cons != null) {
+ return (InstanceT) cons.newInstance(uri, conf);
+ }
+ // new X(conf)
+ cons = getConstructor(instanceClass, Configuration.class);
+ if (cons != null) {
+ return (InstanceT) cons.newInstance(conf);
+ }
+ }
+
+ // X.methodName()
+ Method factory = getFactoryMethod(instanceClass, interfaceImplemented, methodName);
if (factory != null) {
- credentials = (AWSCredentialsProvider)factory.invoke(null);
- return credentials;
+ return (InstanceT) factory.invoke(null);
}
// new X()
- cons = getConstructor(credClass);
+ cons = getConstructor(instanceClass);
if (cons != null) {
- credentials = (AWSCredentialsProvider)cons.newInstance();
- return credentials;
+ return (InstanceT) cons.newInstance();
}
// no supported constructor or factory method found
- throw new IOException(String.format("%s " + CONSTRUCTOR_EXCEPTION
- + ". A class specified in %s must provide a public constructor "
- + "of a supported signature, or a public factory method named "
- + "getInstance that accepts no arguments.",
- className, AWS_CREDENTIALS_PROVIDER));
+ throw unsupportedConstructor(uri, className, configKey);
} catch (InvocationTargetException e) {
Throwable targetException = e.getTargetException();
if (targetException == null) {
- targetException = e;
+ targetException = e;
}
if (targetException instanceof IOException) {
throw (IOException) targetException;
- } else if (targetException instanceof SdkBaseException) {
- throw translateException("Instantiate " + className, "",
- (SdkBaseException) targetException);
+ } else if (targetException instanceof SdkException) {
+ throw translateException("Instantiate " + className, "", (SdkException) targetException);
} else {
// supported constructor or factory method found, but the call failed
- throw new IOException(className + " " + INSTANTIATION_EXCEPTION
- + ": " + targetException,
- targetException);
+ throw instantiationException(uri, className, configKey, targetException);
}
} catch (ReflectiveOperationException | IllegalArgumentException e) {
// supported constructor or factory method found, but the call failed
- throw new IOException(className + " " + INSTANTIATION_EXCEPTION
- + ": " + e,
- e);
+ throw instantiationException(uri, className, configKey, e);
}
}
+
/**
* Set a key if the value is non-empty.
* @param config config to patch
@@ -941,13 +815,13 @@ public final class S3AUtils {
/**
* String information about a summary entry for debug messages.
- * @param summary summary object
+ * @param s3Object s3Object entry
* @return string value
*/
- public static String stringify(S3ObjectSummary summary) {
- StringBuilder builder = new StringBuilder(summary.getKey().length() + 100);
- builder.append(summary.getKey()).append(' ');
- builder.append("size=").append(summary.getSize());
+ public static String stringify(S3Object s3Object) {
+ StringBuilder builder = new StringBuilder(s3Object.key().length() + 100);
+ builder.append(s3Object.key()).append(' ');
+ builder.append("size=").append(s3Object.size());
return builder.toString();
}
@@ -1215,215 +1089,6 @@ public final class S3AUtils {
}
}
- /**
- * Create a new AWS {@code ClientConfiguration}.
- * All clients to AWS services <i>MUST</i> use this for consistent setup
- * of connectivity, UA, proxy settings.
- * @param conf The Hadoop configuration
- * @param bucket Optional bucket to use to look up per-bucket proxy secrets
- * @return new AWS client configuration
- * @throws IOException problem creating AWS client configuration
- *
- * @deprecated use {@link #createAwsConf(Configuration, String, String)}
- */
- @Deprecated
- public static ClientConfiguration createAwsConf(Configuration conf,
- String bucket)
- throws IOException {
- return createAwsConf(conf, bucket, null);
- }
-
- /**
- * Create a new AWS {@code ClientConfiguration}. All clients to AWS services
- * <i>MUST</i> use this or the equivalents for the specific service for
- * consistent setup of connectivity, UA, proxy settings.
- *
- * @param conf The Hadoop configuration
- * @param bucket Optional bucket to use to look up per-bucket proxy secrets
- * @param awsServiceIdentifier a string representing the AWS service (S3,
- * etc) for which the ClientConfiguration is being created.
- * @return new AWS client configuration
- * @throws IOException problem creating AWS client configuration
- */
- public static ClientConfiguration createAwsConf(Configuration conf,
- String bucket, String awsServiceIdentifier)
- throws IOException {
- final ClientConfiguration awsConf = new ClientConfiguration();
- initConnectionSettings(conf, awsConf);
- initProxySupport(conf, bucket, awsConf);
- initUserAgent(conf, awsConf);
- if (StringUtils.isNotEmpty(awsServiceIdentifier)) {
- String configKey = null;
- switch (awsServiceIdentifier) {
- case AWS_SERVICE_IDENTIFIER_S3:
- configKey = SIGNING_ALGORITHM_S3;
- break;
- case AWS_SERVICE_IDENTIFIER_STS:
- configKey = SIGNING_ALGORITHM_STS;
- break;
- default:
- // Nothing to do. The original signer override is already setup
- }
- if (configKey != null) {
- String signerOverride = conf.getTrimmed(configKey, "");
- if (!signerOverride.isEmpty()) {
- LOG.debug("Signer override for {}} = {}", awsServiceIdentifier,
- signerOverride);
- awsConf.setSignerOverride(signerOverride);
- }
- }
- }
- return awsConf;
- }
-
- /**
- * Initializes all AWS SDK settings related to connection management.
- *
- * @param conf Hadoop configuration
- * @param awsConf AWS SDK configuration
- *
- * @throws IOException if there was an error initializing the protocol
- * settings
- */
- public static void initConnectionSettings(Configuration conf,
- ClientConfiguration awsConf) throws IOException {
- awsConf.setMaxConnections(intOption(conf, MAXIMUM_CONNECTIONS,
- DEFAULT_MAXIMUM_CONNECTIONS, 1));
- awsConf.setConnectionTTL(longOption(conf, CONNECTION_TTL,
- DEFAULT_CONNECTION_TTL, -1));
- initProtocolSettings(conf, awsConf);
- awsConf.setMaxErrorRetry(intOption(conf, MAX_ERROR_RETRIES,
- DEFAULT_MAX_ERROR_RETRIES, 0));
- awsConf.setConnectionTimeout(intOption(conf, ESTABLISH_TIMEOUT,
- DEFAULT_ESTABLISH_TIMEOUT, 0));
- awsConf.setSocketTimeout(intOption(conf, SOCKET_TIMEOUT,
- DEFAULT_SOCKET_TIMEOUT, 0));
- int sockSendBuffer = intOption(conf, SOCKET_SEND_BUFFER,
- DEFAULT_SOCKET_SEND_BUFFER, 2048);
- int sockRecvBuffer = intOption(conf, SOCKET_RECV_BUFFER,
- DEFAULT_SOCKET_RECV_BUFFER, 2048);
- long requestTimeoutMillis = conf.getTimeDuration(REQUEST_TIMEOUT,
- DEFAULT_REQUEST_TIMEOUT, TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
-
- if (requestTimeoutMillis > Integer.MAX_VALUE) {
- LOG.debug("Request timeout is too high({} ms). Setting to {} ms instead",
- requestTimeoutMillis, Integer.MAX_VALUE);
- requestTimeoutMillis = Integer.MAX_VALUE;
- }
- awsConf.setRequestTimeout((int) requestTimeoutMillis);
- awsConf.setSocketBufferSizeHints(sockSendBuffer, sockRecvBuffer);
- String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM, "");
- if (!signerOverride.isEmpty()) {
- LOG.debug("Signer override = {}", signerOverride);
- awsConf.setSignerOverride(signerOverride);
- }
- }
-
- /**
- * Initializes the connection protocol settings when connecting to S3 (e.g.
- * either HTTP or HTTPS). If secure connections are enabled, this method
- * will load the configured SSL providers.
- *
- * @param conf Hadoop configuration
- * @param awsConf AWS SDK configuration
- *
- * @throws IOException if there is an error initializing the configured
- * {@link javax.net.ssl.SSLSocketFactory}
- */
- private static void initProtocolSettings(Configuration conf,
- ClientConfiguration awsConf) throws IOException {
- boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS,
- DEFAULT_SECURE_CONNECTIONS);
- awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);
- if (secureConnections) {
- NetworkBinding.bindSSLChannelMode(conf, awsConf);
- }
- }
-
- /**
- * Initializes AWS SDK proxy support in the AWS client configuration
- * if the S3A settings enable it.
- *
- * @param conf Hadoop configuration
- * @param bucket Optional bucket to use to look up per-bucket proxy secrets
- * @param awsConf AWS SDK configuration to update
- * @throws IllegalArgumentException if misconfigured
- * @throws IOException problem getting username/secret from password source.
- */
- public static void initProxySupport(Configuration conf,
- String bucket,
- ClientConfiguration awsConf) throws IllegalArgumentException,
- IOException {
- String proxyHost = conf.getTrimmed(PROXY_HOST, "");
- int proxyPort = conf.getInt(PROXY_PORT, -1);
- if (!proxyHost.isEmpty()) {
- awsConf.setProxyHost(proxyHost);
- if (proxyPort >= 0) {
- awsConf.setProxyPort(proxyPort);
- } else {
- if (conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS)) {
- LOG.warn("Proxy host set without port. Using HTTPS default 443");
- awsConf.setProxyPort(443);
- } else {
- LOG.warn("Proxy host set without port. Using HTTP default 80");
- awsConf.setProxyPort(80);
- }
- }
- final String proxyUsername = lookupPassword(bucket, conf, PROXY_USERNAME,
- null, null);
- final String proxyPassword = lookupPassword(bucket, conf, PROXY_PASSWORD,
- null, null);
- if ((proxyUsername == null) != (proxyPassword == null)) {
- String msg = "Proxy error: " + PROXY_USERNAME + " or " +
- PROXY_PASSWORD + " set without the other.";
- LOG.error(msg);
- throw new IllegalArgumentException(msg);
- }
- boolean isProxySecured = conf.getBoolean(PROXY_SECURED, false);
- awsConf.setProxyUsername(proxyUsername);
- awsConf.setProxyPassword(proxyPassword);
- awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN));
- awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION));
- awsConf.setProxyProtocol(isProxySecured ? Protocol.HTTPS : Protocol.HTTP);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Using proxy server {}://{}:{} as user {} with password {} "
- + "on domain {} as workstation {}",
- awsConf.getProxyProtocol(),
- awsConf.getProxyHost(),
- awsConf.getProxyPort(),
- String.valueOf(awsConf.getProxyUsername()),
- awsConf.getProxyPassword(), awsConf.getProxyDomain(),
- awsConf.getProxyWorkstation());
- }
- } else if (proxyPort >= 0) {
- String msg =
- "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST;
- LOG.error(msg);
- throw new IllegalArgumentException(msg);
- }
- }
-
- /**
- * Initializes the User-Agent header to send in HTTP requests to AWS
- * services. We always include the Hadoop version number. The user also
- * may set an optional custom prefix to put in front of the Hadoop version
- * number. The AWS SDK internally appends its own information, which seems
- * to include the AWS SDK version, OS and JVM version.
- *
- * @param conf Hadoop configuration
- * @param awsConf AWS SDK configuration to update
- */
- private static void initUserAgent(Configuration conf,
- ClientConfiguration awsConf) {
- String userAgent = "Hadoop " + VersionInfo.getVersion();
- String userAgentPrefix = conf.getTrimmed(USER_AGENT_PREFIX, "");
- if (!userAgentPrefix.isEmpty()) {
- userAgent = userAgentPrefix + ", " + userAgent;
- }
- LOG.debug("Using User-Agent: {}", userAgent);
- awsConf.setUserAgentPrefix(userAgent);
- }
-
/**
* Convert the data of an iterator of {@link S3AFileStatus} to
* an array.
@@ -1917,4 +1582,15 @@ public final class S3AUtils {
}
};
+ /**
+ * Format a byte range for a request header.
+ * See https://www.rfc-editor.org/rfc/rfc9110.html#section-14.1.2
+ *
+ * @param rangeStart the start byte offset
+ * @param rangeEnd the end byte offset (inclusive)
+ * @return a formatted byte range
+ */
+ public static String formatRange(long rangeStart, long rangeEnd) {
+ return String.format("bytes=%d-%d", rangeStart, rangeEnd);
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
index 9010f34dc259..d4504cd08d74 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
@@ -24,11 +24,14 @@ import java.net.URI;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.Executor;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.handlers.RequestHandler2;
-import com.amazonaws.monitoring.MonitoringListener;
-import com.amazonaws.services.s3.AmazonS3;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.transfer.s3.S3TransferManager;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -37,7 +40,7 @@ import org.apache.hadoop.fs.s3a.statistics.StatisticsFromAwsSdk;
import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_ENDPOINT;
/**
- * Factory for creation of {@link AmazonS3} client instances.
+ * Factory for creation of {@link S3Client} client instances.
* Important: HBase's HBoss module implements this interface in its
* tests.
* Take care when updating this interface to ensure that a client
@@ -49,20 +52,44 @@ import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_ENDPOINT;
*/
@InterfaceAudience.LimitedPrivate("HBoss")
@InterfaceStability.Evolving
-@Deprecated
public interface S3ClientFactory {
/**
- * Creates a new {@link AmazonS3} client.
+ * Creates a new {@link S3Client}.
+ * The client returned supports synchronous operations. For
+ * asynchronous operations, use
+ * {@link #createS3AsyncClient(URI, S3ClientCreationParameters)}.
*
* @param uri S3A file system URI
* @param parameters parameter object
* @return S3 client
- * @throws IOException IO problem
+ * @throws IOException on any IO problem
*/
- AmazonS3 createS3Client(URI uri,
+ S3Client createS3Client(URI uri,
S3ClientCreationParameters parameters) throws IOException;
+ /**
+ * Creates a new {@link S3AsyncClient}.
+ * The client returned supports asynchronous operations. For
+ * synchronous operations, use
+ * {@link #createS3Client(URI, S3ClientCreationParameters)}.
+ *
+ * @param uri S3A file system URI
+ * @param parameters parameter object
+ * @return Async S3 client
+ * @throws IOException on any IO problem
+ */
+ S3AsyncClient createS3AsyncClient(URI uri,
+ S3ClientCreationParameters parameters) throws IOException;
+
+ /**
+ * Creates a new {@link S3TransferManager}.
+ *
+ * @param s3AsyncClient the async client to be used by the TM.
+ * @return S3 transfer manager
+ */
+ S3TransferManager createS3TransferManager(S3AsyncClient s3AsyncClient);
+
/**
* Settings for the S3 Client.
* Implemented as a class to pass in so that adding
@@ -74,7 +101,7 @@ public interface S3ClientFactory {
/**
* Credentials.
*/
- private AWSCredentialsProvider credentialSet;
+ private AwsCredentialsProvider credentialSet;
/**
* Endpoint.
@@ -86,11 +113,6 @@ public interface S3ClientFactory {
*/
private final Map<String, String> headers = new HashMap<>();
- /**
- * Monitoring listener.
- */
- private MonitoringListener monitoringListener;
-
/**
* RequestMetricCollector metrics...if not-null will be wrapped
* with an {@code AwsStatisticsCollector} and passed to
@@ -109,9 +131,9 @@ public interface S3ClientFactory {
private boolean requesterPays;
/**
- * Request handlers; used for auditing, X-Ray etc.
- */
- private List<RequestHandler2> requestHandlers;
+ * Execution interceptors; used for auditing, X-Ray etc.
+ * */
+ private List<ExecutionInterceptor> executionInterceptors;
/**
* Suffix to UA.
@@ -125,37 +147,43 @@ public interface S3ClientFactory {
private URI pathUri;
/**
- * List of request handlers to include in the chain
- * of request execution in the SDK.
- * @return the handler list
+ * Minimum part size for transfer parts.
*/
- public List<RequestHandler2> getRequestHandlers() {
- return requestHandlers;
- }
+ private long minimumPartSize;
/**
- * List of request handlers.
- * @param handlers handler list.
- * @return this object
+ * Threshold for multipart operations.
*/
- public S3ClientCreationParameters withRequestHandlers(
- @Nullable final List<RequestHandler2> handlers) {
- requestHandlers = handlers;
- return this;
- }
+ private long multiPartThreshold;
- public MonitoringListener getMonitoringListener() {
- return monitoringListener;
+ /**
+ * Executor that the transfer manager will use to execute background tasks.
+ */
+ private Executor transferManagerExecutor;
+
+ /**
+ * Region of the S3 bucket.
+ */
+ private Region region;
+
+
+ /**
+ * List of execution interceptors to include in the chain
+ * of interceptors in the SDK.
+ * @return the interceptors list
+ */
+ public List<ExecutionInterceptor> getExecutionInterceptors() {
+ return executionInterceptors;
}
/**
- * listener for AWS monitoring events.
- * @param listener listener
+ * List of execution interceptors.
+ * @param interceptors interceptors list.
* @return this object
*/
- public S3ClientCreationParameters withMonitoringListener(
- @Nullable final MonitoringListener listener) {
- monitoringListener = listener;
+ public S3ClientCreationParameters withExecutionInterceptors(
+ @Nullable final List<ExecutionInterceptor> interceptors) {
+ executionInterceptors = interceptors;
return this;
}
@@ -191,7 +219,7 @@ public interface S3ClientFactory {
return requesterPays;
}
- public AWSCredentialsProvider getCredentialSet() {
+ public AwsCredentialsProvider getCredentialSet() {
return credentialSet;
}
@@ -202,7 +230,7 @@ public interface S3ClientFactory {
*/
public S3ClientCreationParameters withCredentialSet(
- final AWSCredentialsProvider value) {
+ final AwsCredentialsProvider value) {
credentialSet = value;
return this;
}
@@ -294,5 +322,82 @@ public interface S3ClientFactory {
pathUri = value;
return this;
}
+
+ /**
+ * Get the minimum part size for transfer parts.
+ * @return part size
+ */
+ public long getMinimumPartSize() {
+ return minimumPartSize;
+ }
+
+ /**
+ * Set the minimum part size for transfer parts.
+ * @param value new value
+ * @return the builder
+ */
+ public S3ClientCreationParameters withMinimumPartSize(
+ final long value) {
+ minimumPartSize = value;
+ return this;
+ }
+
+ /**
+ * Get the threshold for multipart operations.
+ * @return multipart threshold
+ */
+ public long getMultiPartThreshold() {
+ return multiPartThreshold;
+ }
+
+ /**
+ * Set the threshold for multipart operations.
+ * @param value new value
+ * @return the builder
+ */
+ public S3ClientCreationParameters withMultipartThreshold(
+ final long value) {
+ multiPartThreshold = value;
+ return this;
+ }
+
+ /**
+ * Get the executor that the transfer manager will use to execute background tasks.
+ * @return part size
+ */
+ public Executor getTransferManagerExecutor() {
+ return transferManagerExecutor;
+ }
+
+ /**
+ * Set the executor that the transfer manager will use to execute background tasks.
+ * @param value new value
+ * @return the builder
+ */
+ public S3ClientCreationParameters withTransferManagerExecutor(
+ final Executor value) {
+ transferManagerExecutor = value;
+ return this;
+ }
+
+ /**
+ * Set region.
+ *
+ * @param value new value
+ * @return the builder
+ */
+ public S3ClientCreationParameters withRegion(
+ final Region value) {
+ region = value;
+ return this;
+ }
+
+ /**
+ * Get the region.
+ * @return invoker
+ */
+ public Region getRegion() {
+ return region;
+ }
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java
index d51211516f25..c729f3de15f0 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java
@@ -18,8 +18,8 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
/**
* API version-independent container for S3 List requests.
@@ -78,14 +78,14 @@ public final class S3ListRequest {
public String toString() {
if (isV1()) {
return String.format(DESCRIPTION,
- v1Request.getBucketName(), v1Request.getPrefix(),
- v1Request.getDelimiter(), v1Request.getMaxKeys(),
- v1Request.isRequesterPays());
+ v1Request.bucket(), v1Request.prefix(),
+ v1Request.delimiter(), v1Request.maxKeys(),
+ v1Request.requestPayerAsString());
} else {
return String.format(DESCRIPTION,
- v2Request.getBucketName(), v2Request.getPrefix(),
- v2Request.getDelimiter(), v2Request.getMaxKeys(),
- v2Request.isRequesterPays());
+ v2Request.bucket(), v2Request.prefix(),
+ v2Request.delimiter(), v2Request.maxKeys(),
+ v2Request.requestPayerAsString());
}
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java
index 69c42bfe1471..c77311211abc 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java
@@ -22,19 +22,21 @@ import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;
-import com.amazonaws.services.s3.model.ListObjectsV2Result;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
+import software.amazon.awssdk.services.s3.model.CommonPrefix;
+import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
import org.slf4j.Logger;
/**
* API version-independent container for S3 List responses.
*/
public class S3ListResult {
- private ObjectListing v1Result;
- private ListObjectsV2Result v2Result;
+ private ListObjectsResponse v1Result;
+ private ListObjectsV2Response v2Result;
- protected S3ListResult(ObjectListing v1, ListObjectsV2Result v2) {
+ protected S3ListResult(ListObjectsResponse v1, ListObjectsV2Response v2) {
v1Result = v1;
v2Result = v2;
}
@@ -44,7 +46,7 @@ public class S3ListResult {
* @param result v1 result
* @return new list result container
*/
- public static S3ListResult v1(ObjectListing result) {
+ public static S3ListResult v1(ListObjectsResponse result) {
return new S3ListResult(result, null);
}
@@ -53,7 +55,7 @@ public class S3ListResult {
* @param result v2 result
* @return new list result container
*/
- public static S3ListResult v2(ListObjectsV2Result result) {
+ public static S3ListResult v2(ListObjectsV2Response result) {
return new S3ListResult(null, result);
}
@@ -65,19 +67,19 @@ public class S3ListResult {
return v1Result != null;
}
- public ObjectListing getV1() {
+ public ListObjectsResponse getV1() {
return v1Result;
}
- public ListObjectsV2Result getV2() {
+ public ListObjectsV2Response getV2() {
return v2Result;
}
- public List<S3ObjectSummary> getObjectSummaries() {
+ public List<S3Object> getS3Objects() {
if (isV1()) {
- return v1Result.getObjectSummaries();
+ return v1Result.contents();
} else {
- return v2Result.getObjectSummaries();
+ return v2Result.contents();
}
}
@@ -89,21 +91,21 @@ public class S3ListResult {
}
}
- public List<String> getCommonPrefixes() {
+ public List<CommonPrefix> getCommonPrefixes() {
if (isV1()) {
- return v1Result.getCommonPrefixes();
+ return v1Result.commonPrefixes();
} else {
- return v2Result.getCommonPrefixes();
+ return v2Result.commonPrefixes();
}
}
/**
- * Get the list of keys in the object summary.
+ * Get the list of keys in the list result.
* @return a possibly empty list
*/
- private List<String> objectSummaryKeys() {
- return getObjectSummaries().stream()
- .map(S3ObjectSummary::getKey)
+ private List<String> objectKeys() {
+ return getS3Objects().stream()
+ .map(S3Object::key)
.collect(Collectors.toList());
}
@@ -112,9 +114,8 @@ public class S3ListResult {
* @return true if the result is non-empty
*/
public boolean hasPrefixesOrObjects() {
-
return !(getCommonPrefixes()).isEmpty()
- || !getObjectSummaries().isEmpty();
+ || !getS3Objects().isEmpty();
}
/**
@@ -128,7 +129,7 @@ public class S3ListResult {
// no children.
// So the listing must contain the marker entry only as an object,
// and prefixes is null
- List<String> keys = objectSummaryKeys();
+ List<String> keys = objectKeys();
return keys.size() == 1 && keys.contains(dirKey)
&& getCommonPrefixes().isEmpty();
}
@@ -138,15 +139,15 @@ public class S3ListResult {
* @param log log to use
*/
public void logAtDebug(Logger log) {
- Collection<String> prefixes = getCommonPrefixes();
- Collection<S3ObjectSummary> summaries = getObjectSummaries();
+ Collection<CommonPrefix> prefixes = getCommonPrefixes();
+ Collection<S3Object> s3Objects = getS3Objects();
log.debug("Prefix count = {}; object count={}",
- prefixes.size(), summaries.size());
- for (S3ObjectSummary summary : summaries) {
- log.debug("Summary: {} {}", summary.getKey(), summary.getSize());
+ prefixes.size(), s3Objects.size());
+ for (S3Object s3Object : s3Objects) {
+ log.debug("Summary: {} {}", s3Object.key(), s3Object.size());
}
- for (String prefix : prefixes) {
- log.debug("Prefix: {}", prefix);
+ for (CommonPrefix prefix : prefixes) {
+ log.debug("Prefix: {}", prefix.prefix());
}
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java
index 0a0454854b2a..4fc5b8658b60 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.services.s3.transfer.model.CopyResult;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.Path;
@@ -74,31 +72,6 @@ public class S3ObjectAttributes {
this.len = len;
}
- /**
- * Construct from the result of a copy and those parameters
- * which aren't included in an AWS SDK response.
- * @param path path
- * @param copyResult copy result.
- * @param serverSideEncryptionAlgorithm current encryption algorithm
- * @param serverSideEncryptionKey any server side encryption key?
- * @param len object length
- */
- public S3ObjectAttributes(
- final Path path,
- final CopyResult copyResult,
- final S3AEncryptionMethods serverSideEncryptionAlgorithm,
- final String serverSideEncryptionKey,
- final long len) {
- this.bucket = copyResult.getDestinationBucketName();
- this.key = copyResult.getDestinationKey();
- this.path = path;
- this.serverSideEncryptionAlgorithm = serverSideEncryptionAlgorithm;
- this.serverSideEncryptionKey = serverSideEncryptionKey;
- this.eTag = copyResult.getETag();
- this.versionId = copyResult.getVersionId();
- this.len = len;
- }
-
public String getBucket() {
return bucket;
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java
index 6579a2bc3e7d..b7de937f5137 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java
@@ -39,6 +39,5 @@ import org.apache.hadoop.fs.s3a.auth.NoAwsCredentialsException;
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
-@SuppressWarnings("deprecation")
public final class SharedInstanceCredentialProvider extends IAMInstanceCredentialsProvider {
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
index 657bd437b5be..13c951e83b9c 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
@@ -18,9 +18,11 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
+
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang3.StringUtils;
@@ -42,13 +44,10 @@ import static org.apache.hadoop.fs.s3a.S3AUtils.getAWSAccessKeys;
* property fs.s3a.aws.credentials.provider. Therefore, changing the class name
* would be a backward-incompatible change.
*
- * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
- * as part of upgrading S3A to SDK V2. See HADOOP-18073.
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
-@Deprecated
-public class SimpleAWSCredentialsProvider implements AWSCredentialsProvider {
+public class SimpleAWSCredentialsProvider implements AwsCredentialsProvider {
public static final String NAME
= "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider";
@@ -63,7 +62,7 @@ public class SimpleAWSCredentialsProvider implements AWSCredentialsProvider {
*/
public SimpleAWSCredentialsProvider(final URI uri, final Configuration conf)
throws IOException {
- this(getAWSAccessKeys(uri, conf));
+ this(getAWSAccessKeys(uri, conf));
}
/**
@@ -75,25 +74,25 @@ public class SimpleAWSCredentialsProvider implements AWSCredentialsProvider {
@VisibleForTesting
SimpleAWSCredentialsProvider(final S3xLoginHelper.Login login)
throws IOException {
- this.accessKey = login.getUser();
- this.secretKey = login.getPassword();
+ this.accessKey = login.getUser();
+ this.secretKey = login.getPassword();
}
@Override
- public AWSCredentials getCredentials() {
+ public AwsCredentials resolveCredentials() {
if (!StringUtils.isEmpty(accessKey) && !StringUtils.isEmpty(secretKey)) {
- return new BasicAWSCredentials(accessKey, secretKey);
+ return AwsBasicCredentials.create(accessKey, secretKey);
}
throw new NoAwsCredentialsException("SimpleAWSCredentialsProvider",
"No AWS credentials in the Hadoop configuration");
}
- @Override
- public void refresh() {}
-
@Override
public String toString() {
- return getClass().getSimpleName();
+ return "SimpleAWSCredentialsProvider{" +
+ "accessKey.empty=" + accessKey.isEmpty() +
+ ", secretKey.empty'" + secretKey.isEmpty() +
+ '}';
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
index f9694871d730..b5f7fcbcd016 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
@@ -548,6 +548,11 @@ public enum Statistic {
StoreStatisticNames.STORE_IO_THROTTLE_RATE,
"Rate of S3 request throttling",
TYPE_QUANTILE),
+ STORE_REGION_PROBE(
+ StoreStatisticNames.STORE_REGION_PROBE,
+ "Store Region Probe",
+ TYPE_DURATION
+ ),
/*
* Delegation Token Operations.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
index db3d0bb13297..5587b11a36f8 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.fs.s3a;
import javax.annotation.Nullable;
import java.io.IOException;
-import com.amazonaws.auth.AWSCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
import java.net.URI;
@@ -44,12 +44,9 @@ import org.apache.hadoop.fs.s3a.auth.NoAwsCredentialsException;
* This credential provider must not fail in creation because that will
* break a chain of credential providers.
*
- * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
- * as part of upgrading S3A to SDK V2. See HADOOP-18073.
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
-@Deprecated
public class TemporaryAWSCredentialsProvider extends AbstractSessionCredentialsProvider {
public static final String NAME
@@ -92,7 +89,7 @@ public class TemporaryAWSCredentialsProvider extends AbstractSessionCredentialsP
* @throws NoAwsCredentialsException the credentials are actually empty.
*/
@Override
- protected AWSCredentials createCredentials(Configuration config)
+ protected AwsCredentials createCredentials(Configuration config)
throws IOException {
MarshalledCredentials creds = MarshalledCredentialBinding.fromFileSystem(
getUri(), config);
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Tristate.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Tristate.java
index 0462ccfd7cbb..d7123787735a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Tristate.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Tristate.java
@@ -18,15 +18,58 @@
package org.apache.hadoop.fs.s3a;
+import java.util.Optional;
+
+import static java.util.Optional.empty;
+import static java.util.Optional.of;
+
/**
* Simple enum to express {true, false, don't know}.
*/
public enum Tristate {
+
// Do not add additional values here. Logic will assume there are exactly
// three possibilities.
- TRUE, FALSE, UNKNOWN;
+ TRUE(of(Boolean.TRUE)),
+ FALSE(of(Boolean.FALSE)),
+ UNKNOWN(empty());
+
+ /**
+ * Mapping to an optional boolean.
+ */
+ @SuppressWarnings("NonSerializableFieldInSerializableClass")
+ private final Optional<Boolean> mapping;
+
+ Tristate(final Optional<Boolean> t) {
+ mapping = t;
+ }
+
+ /**
+ * Get the boolean mapping, if present.
+ * @return the boolean value, if present.
+ */
+ public Optional<Boolean> getMapping() {
+ return mapping;
+ }
+
+ /**
+ * Does this value map to a boolean.
+ * @return true if the state is one of true or false.
+ */
+ public boolean isBoolean() {
+ return mapping.isPresent();
+ }
public static Tristate fromBool(boolean v) {
return v ? TRUE : FALSE;
}
+
+ /**
+ * Build a tristate from a boolean.
+ * @param b source optional
+ * @return a tristate derived from the argument.
+ */
+ public static Tristate from(Optional<Boolean> b) {
+ return b.map(Tristate::fromBool).orElse(UNKNOWN);
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/UploadInfo.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/UploadInfo.java
index 238cd97ed8f4..85ed69e14bc7 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/UploadInfo.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/UploadInfo.java
@@ -18,22 +18,22 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.services.s3.transfer.Upload;
+import software.amazon.awssdk.transfer.s3.model.FileUpload;
/**
* Simple struct that contains information about a S3 upload.
*/
public class UploadInfo {
- private final Upload upload;
+ private final FileUpload fileUpload;
private final long length;
- public UploadInfo(Upload upload, long length) {
- this.upload = upload;
+ public UploadInfo(FileUpload upload, long length) {
+ this.fileUpload = upload;
this.length = length;
}
- public Upload getUpload() {
- return upload;
+ public FileUpload getFileUpload() {
+ return fileUpload;
}
public long getLength() {
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java
index 0bbb8a35f51c..1eacc4cbe07b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java
@@ -19,29 +19,28 @@
package org.apache.hadoop.fs.s3a;
import javax.annotation.Nullable;
-import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.InputStream;
import java.util.List;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.AtomicInteger;
-import com.amazonaws.services.s3.model.AmazonS3Exception;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
-import com.amazonaws.services.s3.model.MultipartUpload;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PartETag;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.PutObjectResult;
-import com.amazonaws.services.s3.model.SelectObjectContentRequest;
-import com.amazonaws.services.s3.model.SelectObjectContentResult;
-import com.amazonaws.services.s3.model.UploadPartRequest;
-import com.amazonaws.services.s3.model.UploadPartResult;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse;
+import software.amazon.awssdk.services.s3.model.CompletedPart;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.MultipartUpload;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
+import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest;
+import software.amazon.awssdk.services.s3.model.SelectObjectContentResponseHandler;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -50,6 +49,8 @@ import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.s3a.api.RequestFactory;
import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
import org.apache.hadoop.fs.s3a.impl.StoreContext;
+import org.apache.hadoop.fs.s3a.select.SelectEventStreamPublisher;
+import org.apache.hadoop.fs.s3a.select.SelectObjectContentHelper;
import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext;
import org.apache.hadoop.fs.s3a.select.SelectBinding;
import org.apache.hadoop.fs.statistics.DurationTrackerFactory;
@@ -239,49 +240,25 @@ public class WriteOperationHelper implements WriteOperations {
/**
* Create a {@link PutObjectRequest} request against the specific key.
* @param destKey destination key
- * @param inputStream source data.
* @param length size, if known. Use -1 for not known
* @param options options for the request
+ * @param isFile is data to be uploaded a file
* @return the request
*/
@Retries.OnceRaw
public PutObjectRequest createPutObjectRequest(String destKey,
- InputStream inputStream,
long length,
- final PutObjectOptions options) {
- activateAuditSpan();
- ObjectMetadata objectMetadata = newObjectMetadata(length);
- return getRequestFactory().newPutObjectRequest(
- destKey,
- objectMetadata,
- options,
- inputStream);
- }
+ final PutObjectOptions options,
+ boolean isFile) {
- /**
- * Create a {@link PutObjectRequest} request to upload a file.
- * @param dest key to PUT to.
- * @param sourceFile source file
- * @param options options for the request
- * @return the request
- */
- @Retries.OnceRaw
- public PutObjectRequest createPutObjectRequest(
- String dest,
- File sourceFile,
- final PutObjectOptions options) {
activateAuditSpan();
- final ObjectMetadata objectMetadata =
- newObjectMetadata((int) sourceFile.length());
-
- PutObjectRequest putObjectRequest = getRequestFactory().
- newPutObjectRequest(dest,
- objectMetadata,
- options,
- sourceFile);
- return putObjectRequest;
+
+ return getRequestFactory()
+ .newPutObjectRequestBuilder(destKey, options, length, false)
+ .build();
}
+
/**
* Callback on a successful write.
* @param length length of the write
@@ -297,17 +274,6 @@ public class WriteOperationHelper implements WriteOperations {
LOG.debug("Write to {} failed", this, ex);
}
- /**
- * Create a new object metadata instance.
- * Any standard metadata headers are added here, for example:
- * encryption.
- * @param length size, if known. Use -1 for not known
- * @return a new metadata instance
- */
- public ObjectMetadata newObjectMetadata(long length) {
- return getRequestFactory().newObjectMetadata(length);
- }
-
/**
* {@inheritDoc}
*/
@@ -320,11 +286,11 @@ public class WriteOperationHelper implements WriteOperations {
try (AuditSpan span = activateAuditSpan()) {
return retry("initiate MultiPartUpload", destKey, true,
() -> {
- final InitiateMultipartUploadRequest initiateMPURequest =
- getRequestFactory().newMultipartUploadRequest(
+ final CreateMultipartUploadRequest.Builder initiateMPURequestBuilder =
+ getRequestFactory().newMultipartUploadRequestBuilder(
destKey, options);
- return owner.initiateMultipartUpload(initiateMPURequest)
- .getUploadId();
+ return owner.initiateMultipartUpload(initiateMPURequestBuilder.build())
+ .uploadId();
});
}
}
@@ -345,10 +311,10 @@ public class WriteOperationHelper implements WriteOperations {
* @throws IOException on problems.
*/
@Retries.RetryTranslated
- private CompleteMultipartUploadResult finalizeMultipartUpload(
+ private CompleteMultipartUploadResponse finalizeMultipartUpload(
String destKey,
String uploadId,
- List<PartETag> partETags,
+ List<CompletedPart> partETags,
long length,
PutObjectOptions putOptions,
Retried retrying) throws IOException {
@@ -357,18 +323,18 @@ public class WriteOperationHelper implements WriteOperations {
"No upload parts in multipart upload");
}
try (AuditSpan span = activateAuditSpan()) {
- CompleteMultipartUploadResult uploadResult;
+ CompleteMultipartUploadResponse uploadResult;
uploadResult = invoker.retry("Completing multipart upload", destKey,
true,
retrying,
() -> {
- final CompleteMultipartUploadRequest request =
- getRequestFactory().newCompleteMultipartUploadRequest(
+ final CompleteMultipartUploadRequest.Builder requestBuilder =
+ getRequestFactory().newCompleteMultipartUploadRequestBuilder(
destKey, uploadId, partETags);
- return writeOperationHelperCallbacks.completeMultipartUpload(request);
+ return writeOperationHelperCallbacks.completeMultipartUpload(requestBuilder.build());
});
- owner.finishedWrite(destKey, length, uploadResult.getETag(),
- uploadResult.getVersionId(),
+ owner.finishedWrite(destKey, length, uploadResult.eTag(),
+ uploadResult.versionId(),
putOptions);
return uploadResult;
}
@@ -391,10 +357,10 @@ public class WriteOperationHelper implements WriteOperations {
* the retry count was exceeded
*/
@Retries.RetryTranslated
- public CompleteMultipartUploadResult completeMPUwithRetries(
+ public CompleteMultipartUploadResponse completeMPUwithRetries(
String destKey,
String uploadId,
- List<PartETag> partETags,
+ List<CompletedPart> partETags,
long length,
AtomicInteger errorCount,
PutObjectOptions putOptions)
@@ -452,7 +418,7 @@ public class WriteOperationHelper implements WriteOperations {
@Retries.RetryTranslated
public void abortMultipartUpload(MultipartUpload upload)
throws IOException {
- invoker.retry("Aborting multipart commit", upload.getKey(), true,
+ invoker.retry("Aborting multipart commit", upload.key(), true,
withinAuditSpan(getAuditSpan(),
() -> owner.abortMultipartUpload(upload)));
}
@@ -477,7 +443,7 @@ public class WriteOperationHelper implements WriteOperations {
abortMultipartUpload(upload);
count++;
} catch (FileNotFoundException e) {
- LOG.debug("Already aborted: {}", upload.getKey(), e);
+ LOG.debug("Already aborted: {}", upload.key(), e);
}
}
return count;
@@ -506,45 +472,31 @@ public class WriteOperationHelper implements WriteOperations {
}
/**
- * Create and initialize a part request of a multipart upload.
- * Exactly one of: {@code uploadStream} or {@code sourceFile}
- * must be specified.
- * A subset of the file may be posted, by providing the starting point
- * in {@code offset} and a length of block in {@code size} equal to
- * or less than the remaining bytes.
+ * Create and initialize a part request builder of a multipart upload.
* The part number must be less than 10000.
* Retry policy is once-translated; to much effort
* @param destKey destination key of ongoing operation
* @param uploadId ID of ongoing upload
* @param partNumber current part number of the upload
* @param size amount of data
- * @param uploadStream source of data to upload
- * @param sourceFile optional source file.
- * @param offset offset in file to start reading.
- * @return the request.
+ * @return the request builder.
* @throws IllegalArgumentException if the parameters are invalid.
* @throws PathIOException if the part number is out of range.
*/
@Override
@Retries.OnceTranslated
- public UploadPartRequest newUploadPartRequest(
+ public UploadPartRequest.Builder newUploadPartRequestBuilder(
String destKey,
String uploadId,
int partNumber,
- long size,
- InputStream uploadStream,
- File sourceFile,
- Long offset) throws IOException {
+ long size) throws IOException {
return once("upload part request", destKey,
withinAuditSpan(getAuditSpan(), () ->
- getRequestFactory().newUploadPartRequest(
+ getRequestFactory().newUploadPartRequestBuilder(
destKey,
uploadId,
partNumber,
- size,
- uploadStream,
- sourceFile,
- offset)));
+ size)));
}
/**
@@ -566,18 +518,20 @@ public class WriteOperationHelper implements WriteOperations {
* @param putObjectRequest the request
* @param putOptions put object options
* @param durationTrackerFactory factory for duration tracking
+ * @param uploadData data to be uploaded
+ * @param isFile is data to be uploaded a file
+ *
* @return the upload initiated
* @throws IOException on problems
*/
@Retries.RetryTranslated
- public PutObjectResult putObject(PutObjectRequest putObjectRequest,
- PutObjectOptions putOptions,
+ public PutObjectResponse putObject(PutObjectRequest putObjectRequest,
+ PutObjectOptions putOptions, S3ADataBlocks.BlockUploadData uploadData, boolean isFile,
DurationTrackerFactory durationTrackerFactory)
throws IOException {
- return retry("Writing Object",
- putObjectRequest.getKey(), true,
- withinAuditSpan(getAuditSpan(), () ->
- owner.putObjectDirect(putObjectRequest, putOptions, durationTrackerFactory)));
+ return retry("Writing Object", putObjectRequest.key(), true, withinAuditSpan(getAuditSpan(),
+ () -> owner.putObjectDirect(putObjectRequest, putOptions, uploadData, isFile,
+ durationTrackerFactory)));
}
/**
@@ -613,10 +567,10 @@ public class WriteOperationHelper implements WriteOperations {
* the retry count was exceeded
*/
@Retries.RetryTranslated
- public CompleteMultipartUploadResult commitUpload(
+ public CompleteMultipartUploadResponse commitUpload(
String destKey,
String uploadId,
- List<PartETag> partETags,
+ List<CompletedPart> partETags,
long length)
throws IOException {
checkNotNull(uploadId);
@@ -635,19 +589,21 @@ public class WriteOperationHelper implements WriteOperations {
* Upload part of a multi-partition file.
* @param request request
* @param durationTrackerFactory duration tracker factory for operation
+ * @param request the upload part request.
+ * @param body the request body.
* @return the result of the operation.
* @throws IOException on problems
*/
@Retries.RetryTranslated
- public UploadPartResult uploadPart(UploadPartRequest request,
+ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody body,
final DurationTrackerFactory durationTrackerFactory)
throws IOException {
- return retry("upload part #" + request.getPartNumber()
- + " upload ID " + request.getUploadId(),
- request.getKey(),
+ return retry("upload part #" + request.partNumber()
+ + " upload ID " + request.uploadId(),
+ request.key(),
true,
withinAuditSpan(getAuditSpan(),
- () -> owner.uploadPart(request, durationTrackerFactory)));
+ () -> owner.uploadPart(request, body, durationTrackerFactory)));
}
/**
@@ -659,15 +615,9 @@ public class WriteOperationHelper implements WriteOperations {
return conf;
}
- /**
- * Create a S3 Select request for the destination path.
- * This does not build the query.
- * @param path pre-qualified path for query
- * @return the request
- */
- public SelectObjectContentRequest newSelectRequest(Path path) {
+ public SelectObjectContentRequest.Builder newSelectRequestBuilder(Path path) {
try (AuditSpan span = getAuditSpan()) {
- return getRequestFactory().newSelectRequest(
+ return getRequestFactory().newSelectRequestBuilder(
storeContext.pathToKey(path));
}
}
@@ -676,26 +626,27 @@ public class WriteOperationHelper implements WriteOperations {
* Execute an S3 Select operation.
* On a failure, the request is only logged at debug to avoid the
* select exception being printed.
- * @param source source for selection
+ *
+ * @param source source for selection
* @param request Select request to issue.
- * @param action the action for use in exception creation
+ * @param action the action for use in exception creation
* @return response
* @throws IOException failure
*/
@Retries.RetryTranslated
- public SelectObjectContentResult select(
+ public SelectEventStreamPublisher select(
final Path source,
final SelectObjectContentRequest request,
final String action)
throws IOException {
// no setting of span here as the select binding is (statically) created
// without any span.
- String bucketName = request.getBucketName();
+ String bucketName = request.bucket();
Preconditions.checkArgument(bucket.equals(bucketName),
"wrong bucket: %s", bucketName);
if (LOG.isDebugEnabled()) {
LOG.debug("Initiating select call {} {}",
- source, request.getExpression());
+ source, request.expression());
LOG.debug(SelectBinding.toString(request));
}
return invoker.retry(
@@ -706,8 +657,9 @@ public class WriteOperationHelper implements WriteOperations {
try (DurationInfo ignored =
new DurationInfo(LOG, "S3 Select operation")) {
try {
- return writeOperationHelperCallbacks.selectObjectContent(request);
- } catch (AmazonS3Exception e) {
+ return SelectObjectContentHelper.select(
+ writeOperationHelperCallbacks, source, request, action);
+ } catch (Throwable e) {
LOG.error("Failure of S3 Select request against {}",
source);
LOG.debug("S3 Select request against {}:\n{}",
@@ -756,16 +708,18 @@ public class WriteOperationHelper implements WriteOperations {
/**
* Initiates a select request.
* @param request selectObjectContent request
+ * @param t selectObjectContent request handler
* @return selectObjectContentResult
*/
- SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request);
+ CompletableFuture<Void> selectObjectContent(SelectObjectContentRequest request,
+ SelectObjectContentResponseHandler t);
/**
* Initiates a complete multi-part upload request.
* @param request Complete multi-part upload request
* @return completeMultipartUploadResult
*/
- CompleteMultipartUploadResult completeMultipartUpload(CompleteMultipartUploadRequest request);
+ CompleteMultipartUploadResponse completeMultipartUpload(CompleteMultipartUploadRequest request);
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java
index 1c3d36885758..0fda4921a30d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java
@@ -20,29 +20,27 @@ package org.apache.hadoop.fs.s3a;
import javax.annotation.Nullable;
import java.io.Closeable;
-import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.InputStream;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
-import com.amazonaws.services.s3.model.MultipartUpload;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PartETag;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.PutObjectResult;
-import com.amazonaws.services.s3.model.SelectObjectContentRequest;
-import com.amazonaws.services.s3.model.SelectObjectContentResult;
-import com.amazonaws.services.s3.model.UploadPartRequest;
-import com.amazonaws.services.s3.model.UploadPartResult;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse;
+import software.amazon.awssdk.services.s3.model.CompletedPart;
+import software.amazon.awssdk.services.s3.model.MultipartUpload;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
+import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartResponse;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
import org.apache.hadoop.fs.statistics.DurationTrackerFactory;
+import org.apache.hadoop.fs.s3a.select.SelectEventStreamPublisher;
import org.apache.hadoop.fs.store.audit.AuditSpanSource;
import org.apache.hadoop.util.functional.CallableRaisingIOE;
@@ -77,27 +75,15 @@ public interface WriteOperations extends AuditSpanSource, Closeable {
/**
* Create a {@link PutObjectRequest} request against the specific key.
* @param destKey destination key
- * @param inputStream source data.
* @param length size, if known. Use -1 for not known
* @param options options for the request
+ * @param isFile is data to be uploaded a file
* @return the request
*/
PutObjectRequest createPutObjectRequest(String destKey,
- InputStream inputStream,
long length,
- @Nullable PutObjectOptions options);
-
- /**
- * Create a {@link PutObjectRequest} request to upload a file.
- * @param dest key to PUT to.
- * @param sourceFile source file
- * @param options options for the request
- * @return the request
- */
- PutObjectRequest createPutObjectRequest(
- String dest,
- File sourceFile,
- @Nullable PutObjectOptions options);
+ @Nullable PutObjectOptions options,
+ boolean isFile);
/**
* Callback on a successful write.
@@ -111,15 +97,6 @@ public interface WriteOperations extends AuditSpanSource, Closeable {
*/
void writeFailed(Exception ex);
- /**
- * Create a new object metadata instance.
- * Any standard metadata headers are added here, for example:
- * encryption.
- * @param length size, if known. Use -1 for not known
- * @return a new metadata instance
- */
- ObjectMetadata newObjectMetadata(long length);
-
/**
* Start the multipart upload process.
* Retry policy: retrying, translated.
@@ -148,10 +125,10 @@ public interface WriteOperations extends AuditSpanSource, Closeable {
* the retry count was exceeded
*/
@Retries.RetryTranslated
- CompleteMultipartUploadResult completeMPUwithRetries(
+ CompleteMultipartUploadResponse completeMPUwithRetries(
String destKey,
String uploadId,
- List<PartETag> partETags,
+ List<CompletedPart> partETags,
long length,
AtomicInteger errorCount,
PutObjectOptions putOptions)
@@ -213,31 +190,20 @@ public interface WriteOperations extends AuditSpanSource, Closeable {
throws IOException;
/**
- * Create and initialize a part request of a multipart upload.
- * Exactly one of: {@code uploadStream} or {@code sourceFile}
- * must be specified.
- * A subset of the file may be posted, by providing the starting point
- * in {@code offset} and a length of block in {@code size} equal to
- * or less than the remaining bytes.
+ * Create and initialize a part request builder of a multipart upload.
* @param destKey destination key of ongoing operation
* @param uploadId ID of ongoing upload
* @param partNumber current part number of the upload
* @param size amount of data
- * @param uploadStream source of data to upload
- * @param sourceFile optional source file.
- * @param offset offset in file to start reading.
- * @return the request.
+ * @return the request builder.
* @throws IllegalArgumentException if the parameters are invalid
* @throws PathIOException if the part number is out of range.
*/
- UploadPartRequest newUploadPartRequest(
+ UploadPartRequest.Builder newUploadPartRequestBuilder(
String destKey,
String uploadId,
int partNumber,
- long size,
- InputStream uploadStream,
- File sourceFile,
- Long offset) throws IOException;
+ long size) throws IOException;
/**
* PUT an object directly (i.e. not via the transfer manager).
@@ -246,12 +212,14 @@ public interface WriteOperations extends AuditSpanSource, Closeable {
* @param putObjectRequest the request
* @param putOptions put object options
* @param durationTrackerFactory factory for duration tracking
+ * @param uploadData data to be uploaded
+ * @param isFile is data to be uploaded a file
* @return the upload initiated
* @throws IOException on problems
*/
@Retries.RetryTranslated
- PutObjectResult putObject(PutObjectRequest putObjectRequest,
- PutObjectOptions putOptions,
+ PutObjectResponse putObject(PutObjectRequest putObjectRequest,
+ PutObjectOptions putOptions, S3ADataBlocks.BlockUploadData uploadData, boolean isFile,
DurationTrackerFactory durationTrackerFactory)
throws IOException;
@@ -279,22 +247,23 @@ public interface WriteOperations extends AuditSpanSource, Closeable {
* the retry count was exceeded
*/
@Retries.RetryTranslated
- CompleteMultipartUploadResult commitUpload(
+ CompleteMultipartUploadResponse commitUpload(
String destKey,
String uploadId,
- List<PartETag> partETags,
+ List<CompletedPart> partETags,
long length)
throws IOException;
/**
* Upload part of a multi-partition file.
- * @param request request
+ * @param request the upload part request.
+ * @param body the request body.
* @param durationTrackerFactory factory for duration tracking
* @return the result of the operation.
* @throws IOException on problems
*/
@Retries.RetryTranslated
- UploadPartResult uploadPart(UploadPartRequest request,
+ UploadPartResponse uploadPart(UploadPartRequest request, RequestBody body,
DurationTrackerFactory durationTrackerFactory)
throws IOException;
@@ -306,25 +275,26 @@ public interface WriteOperations extends AuditSpanSource, Closeable {
Configuration getConf();
/**
- * Create a S3 Select request for the destination path.
+ * Create a S3 Select request builder for the destination path.
* This does not build the query.
* @param path pre-qualified path for query
- * @return the request
+ * @return the request builder
*/
- SelectObjectContentRequest newSelectRequest(Path path);
+ SelectObjectContentRequest.Builder newSelectRequestBuilder(Path path);
/**
* Execute an S3 Select operation.
* On a failure, the request is only logged at debug to avoid the
* select exception being printed.
- * @param source source for selection
+ *
+ * @param source source for selection
* @param request Select request to issue.
- * @param action the action for use in exception creation
+ * @param action the action for use in exception creation
* @return response
* @throws IOException failure
*/
@Retries.RetryTranslated
- SelectObjectContentResult select(
+ SelectEventStreamPublisher select(
Path source,
SelectObjectContentRequest request,
String action)
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/AwsV1BindingSupport.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/AwsV1BindingSupport.java
new file mode 100644
index 000000000000..177952cb9d22
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/AwsV1BindingSupport.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.adapter;
+
+import java.io.IOException;
+import java.net.URI;
+import javax.annotation.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.s3a.impl.InstantiationIOException;
+
+import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.unavailable;
+
+/**
+ * Binding support; the sole way which the rest of the code should instantiate v1 SDK libraries.
+ * Uses this class's Classloader for its analysis/loading.
+ */
+@SuppressWarnings("StaticNonFinalField")
+public final class AwsV1BindingSupport {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ AwsV1BindingSupport.class);
+
+ /**
+ * V1 credential provider classname: {@code}.
+ */
+ public static final String CREDENTIAL_PROVIDER_CLASSNAME =
+ "com.amazonaws.auth.AWSCredentialsProvider";
+
+ /**
+ * SDK availability.
+ */
+ private static final boolean SDK_V1_FOUND = checkForAwsV1Sdk();
+
+ private AwsV1BindingSupport() {
+ }
+
+ /**
+ * Probe for the AWS v1 SDK being available by looking for
+ * the class {@link #CREDENTIAL_PROVIDER_CLASSNAME}.
+ * @return true if it was found in the classloader
+ */
+ private static boolean checkForAwsV1Sdk() {
+
+ try {
+ ClassLoader cl = AwsV1BindingSupport.class.getClassLoader();
+ cl.loadClass(CREDENTIAL_PROVIDER_CLASSNAME);
+ LOG.debug("v1 SDK class {} found", CREDENTIAL_PROVIDER_CLASSNAME);
+ return true;
+ } catch (Exception e) {
+ LOG.debug("v1 SDK class {} not found", CREDENTIAL_PROVIDER_CLASSNAME, e);
+ return false;
+ }
+ }
+
+ /**
+ * Is the AWS v1 SDK available?
+ * @return true if it was found in the classloader
+ */
+ public static synchronized boolean isAwsV1SdkAvailable() {
+ return SDK_V1_FOUND;
+ }
+
+
+ /**
+ * Create an AWS credential provider from its class by using reflection. The
+ * class must implement one of the following means of construction, which are
+ * attempted in order:
+ *
+ * <ol>
+ * <li>a public constructor accepting java.net.URI and
+ * org.apache.hadoop.conf.Configuration</li>
+ * <li>a public constructor accepting
+ * org.apache.hadoop.conf.Configuration</li>
+ * <li>a public static method named getInstance that accepts no
+ * arguments and returns an instance of
+ * com.amazonaws.auth.AWSCredentialsProvider, or</li>
+ * <li>a public default constructor.</li>
+ * </ol>
+ * @param conf configuration
+ * @param className credential classname
+ * @param uri URI of the FS
+ * @param key configuration key to use
+ * @return the instantiated class
+ * @throws InstantiationIOException on any instantiation failure, including v1 SDK not found
+ * @throws IOException anything else.
+ */
+ public static AwsCredentialsProvider createAWSV1CredentialProvider(
+ Configuration conf,
+ String className,
+ @Nullable URI uri,
+ final String key) throws IOException {
+ if (!isAwsV1SdkAvailable()) {
+ throw unavailable(uri, className, key, "No AWS v1 SDK available");
+ }
+ return V1ToV2AwsCredentialProviderAdapter.create(conf, className, uri);
+ }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/V1ToV2AwsCredentialProviderAdapter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/V1ToV2AwsCredentialProviderAdapter.java
new file mode 100644
index 000000000000..2cbd9836d3a4
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/V1ToV2AwsCredentialProviderAdapter.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.adapter;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.URI;
+import javax.annotation.Nullable;
+
+import com.amazonaws.SdkClientException;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.AWSSessionCredentials;
+import com.amazonaws.auth.AnonymousAWSCredentials;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.s3a.CredentialInitializationException;
+import org.apache.hadoop.fs.s3a.S3AUtils;
+import org.apache.hadoop.fs.s3a.impl.InstantiationIOException;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.s3a.Constants.AWS_CREDENTIALS_PROVIDER;
+
+/**
+ * Adapts a V1 {@link AWSCredentialsProvider} to the V2 {@link AwsCredentialsProvider} interface.
+ */
+public final class V1ToV2AwsCredentialProviderAdapter
+ implements AwsCredentialsProvider, Closeable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ V1ToV2AwsCredentialProviderAdapter.class);
+
+ /**
+ * The V1 credential provider constructed.
+ */
+ private final AWSCredentialsProvider v1CredentialsProvider;
+
+
+ private V1ToV2AwsCredentialProviderAdapter(AWSCredentialsProvider v1CredentialsProvider) {
+ this.v1CredentialsProvider = requireNonNull(v1CredentialsProvider);
+ }
+
+
+ /**
+ * Collect v1 credentials and convert to v2.
+ * @return v2 credentials
+ * @throws CredentialInitializationException if the inner retrieval raised an exception
+ */
+ @Override
+ public AwsCredentials resolveCredentials() {
+ try {
+ // get the wrapped credentials
+ AWSCredentials toAdapt = v1CredentialsProvider.getCredentials();
+ return convertToV2Credentials(toAdapt);
+ } catch (SdkClientException e) {
+ // wrap with a v2 exception so that code which adds a try/catch for v2 sdk exceptions
+ // gets a compatible exception.
+ throw new CredentialInitializationException(e.toString(), e);
+ }
+ }
+
+ /**
+ * Close the wrapped provider if it implements Closeable/AutoCloseable.
+ * @throws IOException failure
+ */
+ @Override
+ public void close() throws IOException {
+ if (v1CredentialsProvider instanceof Closeable) {
+ ((Closeable) v1CredentialsProvider).close();
+ } else if (v1CredentialsProvider instanceof AutoCloseable) {
+ S3AUtils.closeAutocloseables(LOG, (AutoCloseable) v1CredentialsProvider);
+ }
+ }
+
+ /**
+ * Convert v1 credentials to v2, including support for session and anonymous
+ * credentials.
+ * @param toAdapt credentials to adapt.
+ * @return v2 credentials.
+ */
+ static AwsCredentials convertToV2Credentials(final AWSCredentials toAdapt) {
+ if (toAdapt instanceof AWSSessionCredentials) {
+ return AwsSessionCredentials.create(toAdapt.getAWSAccessKeyId(),
+ toAdapt.getAWSSecretKey(),
+ ((AWSSessionCredentials) toAdapt).getSessionToken());
+ } else if (toAdapt instanceof AnonymousAWSCredentials) {
+ return AnonymousCredentialsProvider.create().resolveCredentials();
+ } else {
+ return AwsBasicCredentials.create(toAdapt.getAWSAccessKeyId(), toAdapt.getAWSSecretKey());
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "V1ToV2AwsCredentialProviderAdapter{" +
+ "v1CredentialsProvider=" + v1CredentialsProvider +
+ '}';
+ }
+
+ /**
+ * @param v1CredentialsProvider V1 credential provider to adapt.
+ * @return A new instance of the credentials provider adapter.
+ */
+ static AwsCredentialsProvider create(AWSCredentialsProvider v1CredentialsProvider) {
+ return new V1ToV2AwsCredentialProviderAdapter(v1CredentialsProvider);
+ }
+
+ /**
+ * Create an AWS credential provider from its class by using reflection. The
+ * class must implement one of the following means of construction, which are
+ * attempted in order:
+ *
+ * <ol>
+ * <li>a public constructor accepting java.net.URI and
+ * org.apache.hadoop.conf.Configuration</li>
+ * <li>a public constructor accepting
+ * org.apache.hadoop.conf.Configuration</li>
+ * <li>a public static method named getInstance that accepts no
+ * arguments and returns an instance of
+ * com.amazonaws.auth.AWSCredentialsProvider, or</li>
+ * <li>a public default constructor.</li>
+ * </ol>
+ * @param conf configuration
+ * @param className classname
+ * @param uri URI of the FS
+ * @return the instantiated class
+ * @throws InstantiationIOException on construction and instantiation failures,
+ * including v1 SDK exceptions.
+ * @throws IOException if raised by a constructor/factory method.
+ */
+ static AwsCredentialsProvider create(
+ Configuration conf,
+ String className,
+ @Nullable URI uri) throws InstantiationIOException, IOException {
+
+ final AWSCredentialsProvider instance =
+ S3AUtils.getInstanceFromReflection(className, conf, uri, AWSCredentialsProvider.class,
+ "getInstance", AWS_CREDENTIALS_PROVIDER);
+ return create(instance);
+ }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/UploadInfo.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/package-info.java
similarity index 59%
copy from hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/UploadInfo.java
copy to hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/package-info.java
index 238cd97ed8f4..124534188bcc 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/UploadInfo.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/package-info.java
@@ -16,28 +16,16 @@
* limitations under the License.
*/
-package org.apache.hadoop.fs.s3a;
-
-import com.amazonaws.services.s3.transfer.Upload;
-
/**
- * Simple struct that contains information about a S3 upload.
+ * Adapter classes for allowing V1 credential providers to be used with SDKV2.
+ * This is the only package where use of aws v1 classes are permitted;
+ * all instantiations of objects here must use reflection to probe for
+ * availability or be prepared to catch exceptions which may be raised
+ * if the v1 SDK isn't found on the classpath
*/
-public class UploadInfo {
- private final Upload upload;
- private final long length;
-
- public UploadInfo(Upload upload, long length) {
- this.upload = upload;
- this.length = length;
- }
-
- public Upload getUpload() {
- return upload;
- }
-
- public long getLength() {
- return length;
- }
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.fs.s3a.adapter;
-}
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java
index 2a4771925f08..99a898f72816 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java
@@ -19,41 +19,37 @@
package org.apache.hadoop.fs.s3a.api;
import javax.annotation.Nullable;
-import java.io.File;
-import java.io.InputStream;
import java.util.List;
-import java.util.Optional;
-
-import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CannedAccessControlList;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CopyObjectRequest;
-import com.amazonaws.services.s3.model.DeleteObjectRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
-import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
-import com.amazonaws.services.s3.model.ListNextBatchOfObjectsRequest;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ListObjectsV2Request;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PartETag;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams;
-import com.amazonaws.services.s3.model.SSECustomerKey;
-import com.amazonaws.services.s3.model.SelectObjectContentRequest;
-import com.amazonaws.services.s3.model.StorageClass;
-import com.amazonaws.services.s3.model.UploadPartRequest;
+import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CompletedPart;
+import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
+import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest;
+import software.amazon.awssdk.services.s3.model.StorageClass;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.s3a.S3AEncryptionMethods;
import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets;
import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
/**
- * Factory for S3 objects.
+ * Factory for S3 request objects.
*
* This is where the owner FS's {@code prepareRequest()}
* callback is invoked to mark up a request for this span.
@@ -67,6 +63,8 @@ import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
* as there are no guarantees how they are processed.
* That is: no guarantees of retry or translation.
*/
+@InterfaceStability.Unstable
+@InterfaceAudience.LimitedPrivate("testing/diagnostics")
public interface RequestFactory {
/**
@@ -79,22 +77,7 @@ public interface RequestFactory {
* Get the canned ACL of this FS.
* @return an ACL, if any
*/
- CannedAccessControlList getCannedACL();
-
- /**
- * Create the AWS SDK structure used to configure SSE,
- * if the encryption secrets contain the information/settings for this.
- * @return an optional set of KMS Key settings
- */
- Optional<SSEAwsKeyManagementParams> generateSSEAwsKeyParams();
-
- /**
- * Create the SSE-C structure for the AWS SDK, if the encryption secrets
- * contain the information/settings for this.
- * This will contain a secret extracted from the bucket/configuration.
- * @return an optional customer key.
- */
- Optional<SSECustomerKey> generateSSECustomerKey();
+ String getCannedACL();
/**
* Get the encryption algorithm of this endpoint.
@@ -115,79 +98,58 @@ public interface RequestFactory {
StorageClass getStorageClass();
/**
- * Create a new object metadata instance.
- * Any standard metadata headers are added here, for example:
- * encryption.
- *
- * @param length length of data to set in header; Ignored if negative
- * @return a new metadata instance
- */
- ObjectMetadata newObjectMetadata(long length);
-
- /**
- * Create a copy request.
+ * Create a copy request builder.
* This includes the work of copying the relevant parts
* of the metadata from the source
* @param srcKey source
* @param dstKey destination
* @param srcom source object metadata.
- * @return the request
+ * @return the request builder
*/
- CopyObjectRequest newCopyObjectRequest(String srcKey,
+ CopyObjectRequest.Builder newCopyObjectRequestBuilder(String srcKey,
String dstKey,
- ObjectMetadata srcom);
+ HeadObjectResponse srcom);
- /**
- * Create a putObject request.
- * Adds the ACL and metadata
- * @param key key of object
- * @param metadata metadata header
- * @param options options for the request
- * @param srcfile source file
- * @return the request
- */
- PutObjectRequest newPutObjectRequest(String key,
- ObjectMetadata metadata, PutObjectOptions options, File srcfile);
/**
- * Create a {@link PutObjectRequest} request.
+ * Create a {@link PutObjectRequest} request builder.
* The metadata is assumed to have been configured with the size of the
* operation.
* @param key key of object
- * @param metadata metadata header
* @param options options for the request
- * @param inputStream source data.
- * @return the request
+ * @param length length of object to be uploaded
+ * @param isDirectoryMarker true if object to be uploaded is a directory marker
+ * @return the request builder
*/
- PutObjectRequest newPutObjectRequest(String key,
- ObjectMetadata metadata,
+ PutObjectRequest.Builder newPutObjectRequestBuilder(String key,
PutObjectOptions options,
- InputStream inputStream);
+ long length,
+ boolean isDirectoryMarker);
/**
* Create a {@link PutObjectRequest} request for creating
* an empty directory.
*
* @param directory destination directory.
- * @return request for a zero byte upload.
+ * @return request builder for a zero byte upload.
*/
- PutObjectRequest newDirectoryMarkerRequest(String directory);
+ PutObjectRequest.Builder newDirectoryMarkerRequest(String directory);
/**
* List all multipart uploads under a prefix.
* @param prefix prefix to list under
- * @return the request.
+ * @return the request builder.
*/
- ListMultipartUploadsRequest newListMultipartUploadsRequest(
+ ListMultipartUploadsRequest.Builder newListMultipartUploadsRequestBuilder(
@Nullable String prefix);
/**
* Abort a multipart upload.
* @param destKey destination object key
* @param uploadId ID of initiated upload
- * @return the request.
+ * @return the request builder.
*/
- AbortMultipartUploadRequest newAbortMultipartUploadRequest(
+ AbortMultipartUploadRequest.Builder newAbortMultipartUploadRequestBuilder(
String destKey,
String uploadId);
@@ -195,10 +157,10 @@ public interface RequestFactory {
* Start a multipart upload.
* @param destKey destination object key
* @param options options for the request
- * @return the request.
+ * @return the request builder.
* @throws PathIOException if multipart uploads are disabled
*/
- InitiateMultipartUploadRequest newMultipartUploadRequest(
+ CreateMultipartUploadRequest.Builder newMultipartUploadRequestBuilder(
String destKey,
@Nullable PutObjectOptions options) throws PathIOException;
@@ -207,107 +169,95 @@ public interface RequestFactory {
* @param destKey destination object key
* @param uploadId ID of initiated upload
* @param partETags ordered list of etags
- * @return the request.
+ * @return the request builder.
*/
- CompleteMultipartUploadRequest newCompleteMultipartUploadRequest(
+ CompleteMultipartUploadRequest.Builder newCompleteMultipartUploadRequestBuilder(
String destKey,
String uploadId,
- List<PartETag> partETags);
+ List<CompletedPart> partETags);
/**
- * Create a HEAD request.
+ * Create a HEAD object request builder.
* @param key key, may have trailing /
- * @return the request.
+ * @return the request builder.
+ */
+ HeadObjectRequest.Builder newHeadObjectRequestBuilder(String key);
+
+ /**
+ * Create a HEAD bucket request builder.
+ * @param bucket bucket to get metadata for
+ * @return the request builder.
*/
- GetObjectMetadataRequest newGetObjectMetadataRequest(String key);
+ HeadBucketRequest.Builder newHeadBucketRequestBuilder(String bucket);
+
/**
- * Create a GET request.
+ * Create a GET request builder.
* @param key object key
- * @return the request.
+ * @return the request builder.
*/
- GetObjectRequest newGetObjectRequest(String key);
+ GetObjectRequest.Builder newGetObjectRequestBuilder(String key);
/**
- * Create and initialize a part request of a multipart upload.
- * Exactly one of: {@code uploadStream} or {@code sourceFile}
- * must be specified.
- * A subset of the file may be posted, by providing the starting point
- * in {@code offset} and a length of block in {@code size} equal to
- * or less than the remaining bytes.
- * @param destKey destination key of ongoing operation
- * @param uploadId ID of ongoing upload
- * @param partNumber current part number of the upload
- * @param size amount of data
- * @param uploadStream source of data to upload
- * @param sourceFile optional source file.
- * @param offset offset in file to start reading.
- * @return the request.
+ * Create and initialize a part request builder of a multipart upload.
+ *
+ * @param destKey destination key of ongoing operation
+ * @param uploadId ID of ongoing upload
+ * @param partNumber current part number of the upload
+ * @param size amount of data
+ * @return the request builder.
* @throws PathIOException if the part number is out of range.
*/
- UploadPartRequest newUploadPartRequest(
+ UploadPartRequest.Builder newUploadPartRequestBuilder(
String destKey,
String uploadId,
int partNumber,
- long size,
- InputStream uploadStream,
- File sourceFile,
- long offset) throws PathIOException;
+ long size) throws PathIOException;
/**
- * Create a S3 Select request for the destination object.
+ * Create a S3 Select request builder for the destination object.
* This does not build the query.
* @param key object key
- * @return the request
+ * @return the request builder
*/
- SelectObjectContentRequest newSelectRequest(String key);
+ SelectObjectContentRequest.Builder newSelectRequestBuilder(String key);
/**
- * Create the (legacy) V1 list request.
+ * Create the (legacy) V1 list request builder.
* @param key key to list under
* @param delimiter delimiter for keys
* @param maxKeys maximum number in a list page.
- * @return the request
+ * @return the request builder.
*/
- ListObjectsRequest newListObjectsV1Request(String key,
+ ListObjectsRequest.Builder newListObjectsV1RequestBuilder(String key,
String delimiter,
int maxKeys);
/**
- * Create the next V1 page list request, following
- * on from the previous response.
- * @param prev previous response
- * @return the request
- */
-
- ListNextBatchOfObjectsRequest newListNextBatchOfObjectsRequest(
- ObjectListing prev);
-
- /**
- * Create a V2 list request.
+ * Create a V2 list request builder.
* This will be recycled for any subsequent requests.
* @param key key to list under
* @param delimiter delimiter for keys
* @param maxKeys maximum number in a list page.
- * @return the request
+ * @return the request builder.
*/
- ListObjectsV2Request newListObjectsV2Request(String key,
+ ListObjectsV2Request.Builder newListObjectsV2RequestBuilder(String key,
String delimiter,
int maxKeys);
/**
- * Create a request to delete a single object.
+ * Create a request builder to delete a single object.
* @param key object to delete
- * @return the request
+ * @return the request builder.
*/
- DeleteObjectRequest newDeleteObjectRequest(String key);
+ DeleteObjectRequest.Builder newDeleteObjectRequestBuilder(String key);
/**
- * Bulk delete request.
+ * Create a request builder to delete objects in bulk.
* @param keysToDelete list of keys to delete.
- * @return the request
+ * @return the request builder.
*/
- DeleteObjectsRequest newBulkDeleteRequest(
- List<DeleteObjectsRequest.KeyVersion> keysToDelete);
+ DeleteObjectsRequest.Builder newBulkDeleteRequestBuilder(
+ List<ObjectIdentifier> keysToDelete);
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSAuditEventCallbacks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSAuditEventCallbacks.java
index 8134d5cea942..712246c21961 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSAuditEventCallbacks.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSAuditEventCallbacks.java
@@ -18,15 +18,9 @@
package org.apache.hadoop.fs.s3a.audit;
-import com.amazonaws.AmazonWebServiceRequest;
-import com.amazonaws.Request;
-import com.amazonaws.Response;
-import com.amazonaws.SdkBaseException;
-import com.amazonaws.handlers.HandlerAfterAttemptContext;
-import com.amazonaws.handlers.HandlerBeforeAttemptContext;
-import com.amazonaws.http.HttpResponse;
+import software.amazon.awssdk.core.SdkRequest;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
-import org.apache.hadoop.fs.s3a.Retries;
/**
* Callbacks for audit spans. This is implemented
@@ -37,10 +31,10 @@ import org.apache.hadoop.fs.s3a.Retries;
* detect this and raise an exception.
*
* Look at the documentation for
- * {@code com.amazonaws.handlers.IRequestHandler2} for details
+ * {@code ExecutionInterceptor} for details
* on the callbacks.
*/
-public interface AWSAuditEventCallbacks {
+public interface AWSAuditEventCallbacks extends ExecutionInterceptor {
/**
* Return a span ID which must be unique for all spans within
@@ -66,95 +60,8 @@ public interface AWSAuditEventCallbacks {
* It is not invoked on any AWS requests created in the SDK.
* Avoid raising exceptions or talking to any remote service;
* this callback is for annotation rather than validation.
- * @param request request request.
- * @param <T> type of request
- * @return the request, possibly modified.
+ * @param builder the request builder.
*/
- default <T extends AmazonWebServiceRequest> T requestCreated(T request) {
- return request;
- }
+ default void requestCreated(SdkRequest.Builder builder) {}
- /**
- * Preflight preparation of AWS request.
- * @param request request
- * @param <T> type of request
- * @return an updated request.
- * @throws AuditFailureException for generic audit failures
- * @throws SdkBaseException for other reasons.
- */
- @Retries.OnceRaw
- default <T extends AmazonWebServiceRequest> T beforeExecution(T request)
- throws AuditFailureException, SdkBaseException {
- return request;
- }
-
- /**
- * Callback after S3 responded to a request.
- * @param request request
- * @param response response.
- * @throws AuditFailureException for generic audit failures
- * @throws SdkBaseException for other reasons.
- */
- default void afterResponse(Request<?> request,
- Response<?> response)
- throws AuditFailureException, SdkBaseException {
- }
-
- /**
- * Callback after a request resulted in an error.
- * @param request request
- * @param response response.
- * @param exception exception raised.
- * @throws AuditFailureException for generic audit failures
- * @throws SdkBaseException for other reasons.
- */
- default void afterError(Request<?> request,
- Response<?> response,
- Exception exception)
- throws AuditFailureException, SdkBaseException {
- }
-
- /**
- * Request before marshalling.
- * @param request request
- * @return possibly modified request.
- */
- default AmazonWebServiceRequest beforeMarshalling(
- AmazonWebServiceRequest request) {
- return request;
- }
-
- /**
- * Request before marshalling.
- * @param request request
- */
- default void beforeRequest(Request<?> request) {
- }
-
- /**
- * Before any attempt is made.
- * @param context full context, including the request.
- */
- default void beforeAttempt(HandlerBeforeAttemptContext context) {
- }
-
- /**
- * After any attempt is made.
- * @param context full context, including the request.
- */
- default void afterAttempt(
- HandlerAfterAttemptContext context) {
- }
-
- /**
- * Before unmarshalling the response.
- * @param request request made.
- * @param httpResponse response received
- * @return updated response.
- */
- default HttpResponse beforeUnmarshalling(
- final Request<?> request,
- final HttpResponse httpResponse) {
- return httpResponse;
- }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java
index b4be341c912e..8a24a4e14db1 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java
@@ -20,24 +20,23 @@ package org.apache.hadoop.fs.s3a.audit;
import java.util.List;
-import com.amazonaws.AmazonWebServiceRequest;
-import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CopyPartRequest;
-import com.amazonaws.services.s3.model.DeleteObjectRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.GetBucketLocationRequest;
-import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
-import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
-import com.amazonaws.services.s3.model.ListNextBatchOfObjectsRequest;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ListObjectsV2Request;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.SelectObjectContentRequest;
-import com.amazonaws.services.s3.model.UploadPartRequest;
+import software.amazon.awssdk.core.SdkRequest;
+import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.GetBucketLocationRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartCopyRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
import static org.apache.hadoop.fs.statistics.StoreStatisticNames.ACTION_HTTP_GET_REQUEST;
import static org.apache.hadoop.fs.statistics.StoreStatisticNames.ACTION_HTTP_HEAD_REQUEST;
@@ -64,102 +63,85 @@ public class AWSRequestAnalyzer {
* read/write and path.
* @param request request.
* @return information about the request.
- * @param <T> type of request.
*/
- public <T extends AmazonWebServiceRequest> RequestInfo analyze(T request) {
+ public RequestInfo analyze(SdkRequest request) {
// this is where Scala's case statement would massively
// simplify life.
// Please Keep in Alphabetical Order.
if (request instanceof AbortMultipartUploadRequest) {
return writing(MULTIPART_UPLOAD_ABORTED,
- ((AbortMultipartUploadRequest) request).getKey(),
+ ((AbortMultipartUploadRequest) request).key(),
0);
} else if (request instanceof CompleteMultipartUploadRequest) {
CompleteMultipartUploadRequest r
= (CompleteMultipartUploadRequest) request;
return writing(MULTIPART_UPLOAD_COMPLETED,
- r.getKey(),
- r.getPartETags().size());
+ r.key(),
+ r.multipartUpload().parts().size());
+ } else if (request instanceof CreateMultipartUploadRequest) {
+ return writing(MULTIPART_UPLOAD_STARTED,
+ ((CreateMultipartUploadRequest) request).key(),
+ 0);
} else if (request instanceof DeleteObjectRequest) {
// DeleteObject: single object
return writing(OBJECT_DELETE_REQUEST,
- ((DeleteObjectRequest) request).getKey(),
+ ((DeleteObjectRequest) request).key(),
1);
} else if (request instanceof DeleteObjectsRequest) {
// DeleteObjects: bulk delete
// use first key as the path
DeleteObjectsRequest r = (DeleteObjectsRequest) request;
- List<DeleteObjectsRequest.KeyVersion> keys
- = r.getKeys();
+ List<ObjectIdentifier> objectIdentifiers
+ = r.delete().objects();
return writing(OBJECT_BULK_DELETE_REQUEST,
- keys.isEmpty() ? null : keys.get(0).getKey(),
- keys.size());
+ objectIdentifiers.isEmpty() ? null : objectIdentifiers.get(0).key(),
+ objectIdentifiers.size());
} else if (request instanceof GetBucketLocationRequest) {
GetBucketLocationRequest r = (GetBucketLocationRequest) request;
return reading(STORE_EXISTS_PROBE,
- r.getBucketName(),
+ r.bucket(),
0);
- } else if (request instanceof GetObjectMetadataRequest) {
- return reading(ACTION_HTTP_HEAD_REQUEST,
- ((GetObjectMetadataRequest) request).getKey(), 0);
} else if (request instanceof GetObjectRequest) {
GetObjectRequest r = (GetObjectRequest) request;
- long[] range = r.getRange();
- long size = range == null
- ? -1
- : range[1] - range[0];
return reading(ACTION_HTTP_GET_REQUEST,
- r.getKey(),
- size);
- } else if (request instanceof InitiateMultipartUploadRequest) {
- return writing(MULTIPART_UPLOAD_STARTED,
- ((InitiateMultipartUploadRequest) request).getKey(),
- 0);
+ r.key(),
+ sizeFromRangeHeader(r.range()));
+ } else if (request instanceof HeadObjectRequest) {
+ return reading(ACTION_HTTP_HEAD_REQUEST,
+ ((HeadObjectRequest) request).key(), 0);
} else if (request instanceof ListMultipartUploadsRequest) {
ListMultipartUploadsRequest r
= (ListMultipartUploadsRequest) request;
return reading(MULTIPART_UPLOAD_LIST,
- r.getPrefix(),
- r.getMaxUploads());
+ r.prefix(),
+ r.maxUploads());
} else if (request instanceof ListObjectsRequest) {
ListObjectsRequest r = (ListObjectsRequest) request;
return reading(OBJECT_LIST_REQUEST,
- r.getPrefix(),
- r.getMaxKeys());
- } else if (request instanceof ListNextBatchOfObjectsRequest) {
- ListNextBatchOfObjectsRequest r = (ListNextBatchOfObjectsRequest) request;
- ObjectListing l = r.getPreviousObjectListing();
- String prefix = "";
- int size = 0;
- if (l != null) {
- prefix = l.getPrefix();
- size = l.getMaxKeys();
- }
- return reading(OBJECT_LIST_REQUEST,
- prefix,
- size);
+ r.prefix(),
+ r.maxKeys());
} else if (request instanceof ListObjectsV2Request) {
ListObjectsV2Request r = (ListObjectsV2Request) request;
return reading(OBJECT_LIST_REQUEST,
- r.getPrefix(),
- r.getMaxKeys());
+ r.prefix(),
+ r.maxKeys());
} else if (request instanceof PutObjectRequest) {
PutObjectRequest r = (PutObjectRequest) request;
return writing(OBJECT_PUT_REQUEST,
- r.getKey(),
+ r.key(),
0);
} else if (request instanceof SelectObjectContentRequest) {
SelectObjectContentRequest r =
(SelectObjectContentRequest) request;
return reading(OBJECT_SELECT_REQUESTS,
- r.getKey(),
+ r.key(),
1);
} else if (request instanceof UploadPartRequest) {
UploadPartRequest r = (UploadPartRequest) request;
return writing(MULTIPART_UPLOAD_PART_PUT,
- r.getKey(),
- r.getPartSize());
+ r.key(),
+ r.contentLength());
}
// no explicit support, return classname
return writing(request.getClass().getName(), null, 0);
@@ -212,7 +194,7 @@ public class AWSRequestAnalyzer {
*/
public static boolean
isRequestNotAlwaysInSpan(final Object request) {
- return request instanceof CopyPartRequest
+ return request instanceof UploadPartCopyRequest
|| request instanceof CompleteMultipartUploadRequest
|| request instanceof GetBucketLocationRequest;
}
@@ -225,9 +207,9 @@ public class AWSRequestAnalyzer {
* @return true if the transfer manager creates them.
*/
public static boolean isRequestMultipartIO(final Object request) {
- return request instanceof CopyPartRequest
+ return request instanceof UploadPartCopyRequest
|| request instanceof CompleteMultipartUploadRequest
- || request instanceof InitiateMultipartUploadRequest
+ || request instanceof CreateMultipartUploadRequest
|| request instanceof UploadPartRequest;
}
@@ -307,4 +289,23 @@ public class AWSRequestAnalyzer {
private static long toSafeLong(final Number size) {
return size != null ? size.longValue() : 0;
}
+
+ private static final String BYTES_PREFIX = "bytes=";
+
+ private static Number sizeFromRangeHeader(String rangeHeader) {
+ if (rangeHeader != null && rangeHeader.startsWith(BYTES_PREFIX)) {
+ String[] values = rangeHeader
+ .substring(BYTES_PREFIX.length())
+ .split("-");
+ if (values.length == 2) {
+ try {
+ long start = Long.parseUnsignedLong(values[0]);
+ long end = Long.parseUnsignedLong(values[0]);
+ return end - start;
+ } catch(NumberFormatException e) {
+ }
+ }
+ }
+ return -1;
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditIntegration.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditIntegration.java
index c66f45eb309f..dfbe8e1a1350 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditIntegration.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditIntegration.java
@@ -23,7 +23,7 @@ import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.nio.file.AccessDeniedException;
-import com.amazonaws.HandlerContextAware;
+import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,10 +34,11 @@ import org.apache.hadoop.fs.s3a.audit.impl.LoggingAuditor;
import org.apache.hadoop.fs.s3a.audit.impl.NoopAuditManagerS3A;
import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+
import static java.util.Objects.requireNonNull;
import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_ENABLED;
import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_ENABLED_DEFAULT;
-import static org.apache.hadoop.fs.s3a.audit.impl.S3AInternalAuditConstants.AUDIT_SPAN_HANDLER_CONTEXT;
+import static org.apache.hadoop.fs.s3a.audit.impl.S3AInternalAuditConstants.AUDIT_SPAN_EXECUTION_ATTRIBUTE;
/**
* Support for integrating auditing within the S3A code.
@@ -123,25 +124,24 @@ public final class AuditIntegration {
}
/**
- * Get the span from a handler context.
- * @param request request
- * @param <T> type of request.
+ * Get the span from the execution attributes.
+ * @param executionAttributes the execution attributes
* @return the span callbacks or null
*/
- public static <T extends HandlerContextAware> AWSAuditEventCallbacks
- retrieveAttachedSpan(final T request) {
- return request.getHandlerContext(AUDIT_SPAN_HANDLER_CONTEXT);
+ public static AuditSpanS3A
+ retrieveAttachedSpan(final ExecutionAttributes executionAttributes) {
+ return executionAttributes.getAttribute(AUDIT_SPAN_EXECUTION_ATTRIBUTE);
}
/**
- * Attach a span to a handler context.
- * @param request request
+ * Attach a span to the execution attributes.
+ * @param executionAttributes the execution attributes
* @param span span to attach
- * @param <T> type of request.
*/
- public static <T extends HandlerContextAware> void attachSpanToRequest(
- final T request, final AWSAuditEventCallbacks span) {
- request.addHandlerContext(AUDIT_SPAN_HANDLER_CONTEXT, span);
+ public static void attachSpanToRequest(
+ final ExecutionAttributes executionAttributes,
+ final AuditSpanS3A span) {
+ executionAttributes.putAttribute(AUDIT_SPAN_EXECUTION_ATTRIBUTE, span);
}
/**
@@ -160,4 +160,30 @@ public final class AuditIntegration {
return (AccessDeniedException)new AccessDeniedException(path, null,
exception.toString()).initCause(exception);
}
+
+ /**
+ * Translate an exception if it or its inner exception is an
+ * {@link AuditFailureException}.
+ * If this condition is not met, null is returned.
+ * @param path path of operation.
+ * @param exception exception
+ * @return a translated exception or null.
+ */
+ public static IOException maybeTranslateAuditException(String path,
+ Exception exception) {
+ if (exception instanceof AuditFailureException) {
+ return translateAuditException(path, (AuditFailureException) exception);
+ } else if (exception.getCause() instanceof AuditFailureException) {
+ return translateAuditException(path,
+ (AuditFailureException) exception.getCause());
+ } else {
+ return null;
+ }
+ }
+
+ public static boolean containsAuditException(Exception exception) {
+ return exception instanceof AuditFailureException
+ || exception.getCause() instanceof AuditFailureException;
+ }
+
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditManagerS3A.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditManagerS3A.java
index c1302d57454b..bb7f94cfc206 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditManagerS3A.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditManagerS3A.java
@@ -21,8 +21,8 @@ package org.apache.hadoop.fs.s3a.audit;
import java.io.IOException;
import java.util.List;
-import com.amazonaws.handlers.RequestHandler2;
-import com.amazonaws.services.s3.transfer.internal.TransferStateChangeListener;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
+import software.amazon.awssdk.transfer.s3.progress.TransferListener;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.Path;
@@ -32,6 +32,7 @@ import org.apache.hadoop.fs.store.audit.ActiveThreadSpanSource;
import org.apache.hadoop.fs.store.audit.AuditSpanSource;
import org.apache.hadoop.service.Service;
+
/**
* Interface for Audit Managers auditing operations through the
* AWS libraries.
@@ -56,24 +57,24 @@ public interface AuditManagerS3A extends Service,
OperationAuditor getAuditor();
/**
- * Create the request handler(s) for this audit service.
- * The list returned is mutable; new handlers may be added.
- * @return list of handlers for the SDK.
+ * Create the execution interceptor(s) for this audit service.
+ * The list returned is mutable; new interceptors may be added.
+ * @return list of interceptors for the SDK.
* @throws IOException failure.
*/
- List<RequestHandler2> createRequestHandlers() throws IOException;
+ List<ExecutionInterceptor> createExecutionInterceptors() throws IOException;
/**
- * Return a transfer state change callback which
+ * Return a transfer callback which
* fixes the active span context to be that in which
- * the state change listener was created.
+ * the transfer listener was created.
* This can be used to audit the creation of the multipart
* upload initiation request which the transfer manager
* makes when a file to be copied is split up.
* This must be invoked/used within the active span.
- * @return a state change listener.
+ * @return a transfer listener.
*/
- TransferStateChangeListener createStateChangeListener();
+ TransferListener createTransferListener();
/**
* Check for permission to access a path.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3AAuditConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3AAuditConstants.java
index 1d76833f8cea..55deb0a1989d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3AAuditConstants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3AAuditConstants.java
@@ -66,13 +66,20 @@ public final class S3AAuditConstants {
"org.apache.hadoop.fs.s3a.audit.impl.NoopAuditor";
/**
- * List of extra AWS SDK request handlers: {@value}.
- * These are added to the SDK request chain <i>after</i>
- * any audit service.
+ * Deprecated list of extra AWS SDK request handlers: {@value}.
+ * Use {@link #AUDIT_EXECUTION_INTERCEPTORS} instead.
*/
public static final String AUDIT_REQUEST_HANDLERS =
"fs.s3a.audit.request.handlers";
+ /**
+ * List of extra AWS SDK execution interceptors: {@value}.
+ * These are added to the SDK request chain <i>after</i>
+ * any audit service.
+ */
+ public static final String AUDIT_EXECUTION_INTERCEPTORS =
+ "fs.s3a.audit.execution.interceptors";
+
/**
* Should operations outside spans be rejected?
* This is for testing coverage of the span code; if used
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/ActiveAuditManagerS3A.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/ActiveAuditManagerS3A.java
index ffd08d37e4aa..1da203dc8024 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/ActiveAuditManagerS3A.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/ActiveAuditManagerS3A.java
@@ -25,16 +25,16 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
-import com.amazonaws.AmazonWebServiceRequest;
-import com.amazonaws.HandlerContextAware;
-import com.amazonaws.Request;
-import com.amazonaws.Response;
-import com.amazonaws.SdkBaseException;
-import com.amazonaws.handlers.HandlerAfterAttemptContext;
-import com.amazonaws.handlers.HandlerBeforeAttemptContext;
-import com.amazonaws.handlers.RequestHandler2;
-import com.amazonaws.http.HttpResponse;
-import com.amazonaws.services.s3.transfer.internal.TransferStateChangeListener;
+import software.amazon.awssdk.core.SdkRequest;
+import software.amazon.awssdk.core.SdkResponse;
+import software.amazon.awssdk.core.interceptor.Context;
+import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
+import software.amazon.awssdk.http.SdkHttpRequest;
+import software.amazon.awssdk.http.SdkHttpResponse;
+import software.amazon.awssdk.transfer.s3.progress.TransferListener;
+
+import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,6 +56,7 @@ import org.apache.hadoop.fs.s3a.audit.AuditSpanS3A;
import org.apache.hadoop.fs.s3a.audit.OperationAuditor;
import org.apache.hadoop.fs.s3a.audit.OperationAuditorOptions;
import org.apache.hadoop.fs.s3a.audit.S3AAuditConstants;
+import org.apache.hadoop.fs.s3a.impl.V2Migration;
import org.apache.hadoop.fs.store.LogExactlyOnce;
import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
import org.apache.hadoop.service.CompositeService;
@@ -66,6 +67,7 @@ import static org.apache.hadoop.fs.s3a.Statistic.AUDIT_FAILURE;
import static org.apache.hadoop.fs.s3a.Statistic.AUDIT_REQUEST_EXECUTION;
import static org.apache.hadoop.fs.s3a.audit.AuditIntegration.attachSpanToRequest;
import static org.apache.hadoop.fs.s3a.audit.AuditIntegration.retrieveAttachedSpan;
+import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_EXECUTION_INTERCEPTORS;
import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_REQUEST_HANDLERS;
/**
@@ -82,10 +84,11 @@ import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_REQUEST_HAN
* will deactivate the wrapped span and then
* switch the active span to the unbounded span.
*
- * The inner class {@link AWSAuditEventCallbacks} is returned
- * as a request handler in {@link #createRequestHandlers()};
- * this forwards all requests to the outer {@code ActiveAuditManagerS3A},
- * which then locates the active span and forwards the request.
+ * This class also implements {@link ExecutionInterceptor} and
+ * returns itself in {@link #createExecutionInterceptors()};
+ * once registered with the S3 client, the implemented methods
+ * will be called during different parts of an SDK request lifecycle,
+ * which then locate the active span and forward the request.
* If any such invocation raises an {@link AuditFailureException}
* then the IOStatistics counter for {@code AUDIT_FAILURE}
* is incremented.
@@ -390,25 +393,39 @@ public final class ActiveAuditManagerS3A
}
/**
- * Return a request handler for the AWS SDK which
+ * Return a list of execution interceptors for the AWS SDK which
* relays to this class.
- * @return a request handler.
+ * @return a list of execution interceptors.
*/
@Override
- public List<RequestHandler2> createRequestHandlers()
+ public List<ExecutionInterceptor> createExecutionInterceptors()
throws IOException {
// wire up the AWS SDK To call back into this class when
// preparing to make S3 calls.
- List<RequestHandler2> requestHandlers = new ArrayList<>();
- requestHandlers.add(new SdkRequestHandler());
- // now look for any more handlers
- final Class<?>[] handlers = getConfig().getClasses(AUDIT_REQUEST_HANDLERS);
- if (handlers != null) {
- for (Class<?> handler : handlers) {
+ List<ExecutionInterceptor> executionInterceptors = new ArrayList<>();
+ executionInterceptors.add(this);
+
+ final String handlers = getConfig().getTrimmed(AUDIT_REQUEST_HANDLERS, "");
+ if (!handlers.isEmpty()) {
+ // warn and ignore v1 handlers.
+ V2Migration.v1RequestHandlersUsed(handlers);
+ }
+
+ // V2 SDK supports global/service interceptors, but they need to be configured on the
+ // classpath and don't get the filesystem/job configuration passed down.
+ final Class<?>[] interceptors = getConfig().getClasses(AUDIT_EXECUTION_INTERCEPTORS);
+ if (interceptors != null) {
+ for (Class<?> handler : interceptors) {
try {
+ LOG.debug("Adding intercept of class {}", handler);
Constructor<?> ctor = handler.getConstructor();
- requestHandlers.add((RequestHandler2)ctor.newInstance());
+ final ExecutionInterceptor interceptor = (ExecutionInterceptor) ctor.newInstance();
+ if (interceptor instanceof Configurable) {
+ // pass in the configuration.
+ ((Configurable) interceptor).setConf(getConfig());
+ }
+ executionInterceptors.add(interceptor);
} catch (ExceptionInInitializerError e) {
throw FutureIO.unwrapInnerException(e);
} catch (Exception e) {
@@ -416,13 +433,18 @@ public final class ActiveAuditManagerS3A
}
}
}
- return requestHandlers;
+ return executionInterceptors;
}
@Override
- public TransferStateChangeListener createStateChangeListener() {
+ public TransferListener createTransferListener() {
final WrappingAuditSpan span = activeSpan();
- return (transfer, state) -> switchToActiveSpan(span);
+ return new TransferListener() {
+ @Override
+ public void transferInitiated(Context.TransferInitiated context) {
+ switchToActiveSpan(span);
+ }
+ };
}
@Override
@@ -434,20 +456,18 @@ public final class ActiveAuditManagerS3A
}
/**
- * Attach a reference to the active thread span, then
- * invoke the same callback on that active thread.
+ * Audit the creation of a request and retrieve
+ * a reference to the active thread span.
*/
@Override
- public <T extends AmazonWebServiceRequest> T requestCreated(
- final T request) {
+ public void requestCreated(final SdkRequest.Builder builder) {
AuditSpanS3A span = getActiveAuditSpan();
if (LOG.isTraceEnabled()) {
LOG.trace("Created Request {} in span {}",
- analyzer.analyze(request), span);
+ analyzer.analyze(builder.build()), span);
}
- attachSpanToRequest(request, span);
try {
- return span.requestCreated(request);
+ span.requestCreated(builder);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -463,14 +483,13 @@ public final class ActiveAuditManagerS3A
* {@inheritDoc}
*/
@Override
- public <T extends AmazonWebServiceRequest> T beforeExecution(
- final T request) {
+ public void beforeExecution(Context.BeforeExecution context,
+ ExecutionAttributes executionAttributes) {
ioStatisticsStore.incrementCounter(AUDIT_REQUEST_EXECUTION.getSymbol());
-
- // identify the span and invoke the callback
+ AuditSpanS3A span = getActiveAuditSpan();
+ attachSpanToRequest(executionAttributes, span);
try {
- return extractAndActivateSpanFromRequest(request)
- .beforeExecution(request);
+ span.beforeExecution(context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -479,16 +498,14 @@ public final class ActiveAuditManagerS3A
/**
* Forward to active span.
- * @param request request
- * @param response response.
+ * {@inheritDoc}
*/
@Override
- public void afterResponse(final Request<?> request,
- final Response<?> response)
- throws AuditFailureException, SdkBaseException {
+ public void afterExecution(Context.AfterExecution context,
+ ExecutionAttributes executionAttributes) {
try {
- extractAndActivateSpanFromRequest(request)
- .afterResponse(request, response);
+ extractAndActivateSpanFromRequest(context.request(), executionAttributes)
+ .afterExecution(context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -496,18 +513,19 @@ public final class ActiveAuditManagerS3A
}
/**
- * Get the active span from the handler context,
+ * Get the active span from the execution attributes,
* falling back to the active thread span if there
- * is nothing in the context.
- * Provided the span is a wrapped span, the
+ * is nothing in the attributes.
+ * Provided the span is a wrapped span, the span is
+ * activated.
* @param request request
- * @param <T> type of request.
- * @return the callbacks
+ * @param executionAttributes the execution attributes
+ * @return the active span
*/
- private <T extends HandlerContextAware> AWSAuditEventCallbacks
- extractAndActivateSpanFromRequest(final T request) {
- AWSAuditEventCallbacks span;
- span = retrieveAttachedSpan(request);
+ private AuditSpanS3A extractAndActivateSpanFromRequest(
+ final SdkRequest request,
+ final ExecutionAttributes executionAttributes) {
+ AuditSpanS3A span = retrieveAttachedSpan(executionAttributes);
if (span == null) {
// no span is attached. Not unusual for the copy operations,
// or for calls to GetBucketLocation made by the AWS client
@@ -530,18 +548,16 @@ public final class ActiveAuditManagerS3A
/**
* Forward to active span.
- * @param request request
- * @param response response.
- * @param exception exception raised.
+ * @param context execution context
+ * @param executionAttributes the execution attributes
+ * {@inheritDoc}
*/
@Override
- public void afterError(final Request<?> request,
- final Response<?> response,
- final Exception exception)
- throws AuditFailureException, SdkBaseException {
+ public void onExecutionFailure(Context.FailedExecution context,
+ ExecutionAttributes executionAttributes) {
try {
- extractAndActivateSpanFromRequest(request)
- .afterError(request, response, exception);
+ extractAndActivateSpanFromRequest(context.request(), executionAttributes).onExecutionFailure(
+ context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -549,11 +565,12 @@ public final class ActiveAuditManagerS3A
}
@Override
- public AmazonWebServiceRequest beforeMarshalling(
- final AmazonWebServiceRequest request) {
+ public SdkRequest modifyRequest(Context.ModifyRequest context,
+ ExecutionAttributes executionAttributes) {
try {
- return extractAndActivateSpanFromRequest(request)
- .beforeMarshalling(request);
+ return extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .modifyRequest(context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -561,10 +578,12 @@ public final class ActiveAuditManagerS3A
}
@Override
- public void beforeRequest(final Request<?> request) {
+ public void beforeMarshalling(Context.BeforeMarshalling context,
+ ExecutionAttributes executionAttributes) {
try {
- extractAndActivateSpanFromRequest(request)
- .beforeRequest(request);
+ extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .beforeMarshalling(context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -572,10 +591,12 @@ public final class ActiveAuditManagerS3A
}
@Override
- public void beforeAttempt(final HandlerBeforeAttemptContext context) {
+ public void afterMarshalling(Context.AfterMarshalling context,
+ ExecutionAttributes executionAttributes) {
try {
- extractAndActivateSpanFromRequest(context.getRequest())
- .beforeAttempt(context);
+ extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .afterMarshalling(context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -583,10 +604,12 @@ public final class ActiveAuditManagerS3A
}
@Override
- public void afterAttempt(final HandlerAfterAttemptContext context) {
+ public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context,
+ ExecutionAttributes executionAttributes) {
try {
- extractAndActivateSpanFromRequest(context.getRequest())
- .afterAttempt(context);
+ return extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .modifyHttpRequest(context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -594,73 +617,80 @@ public final class ActiveAuditManagerS3A
}
@Override
- public HttpResponse beforeUnmarshalling(final Request<?> request,
- final HttpResponse httpResponse) {
+ public void beforeTransmission(Context.BeforeTransmission context,
+ ExecutionAttributes executionAttributes) {
try {
- extractAndActivateSpanFromRequest(request.getOriginalRequest())
- .beforeUnmarshalling(request, httpResponse);
+ extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .beforeTransmission(context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
}
- return httpResponse;
}
- /**
- * Callbacks from the AWS SDK; all forward to the ActiveAuditManagerS3A.
- * We need a separate class because the SDK requires the handler list
- * to be list of {@code RequestHandler2} instances.
- */
- private class SdkRequestHandler extends RequestHandler2 {
-
- @Override
- public AmazonWebServiceRequest beforeExecution(
- final AmazonWebServiceRequest request) {
- return ActiveAuditManagerS3A.this.beforeExecution(request);
- }
-
- @Override
- public void afterResponse(final Request<?> request,
- final Response<?> response) {
- ActiveAuditManagerS3A.this.afterResponse(request, response);
- }
-
- @Override
- public void afterError(final Request<?> request,
- final Response<?> response,
- final Exception e) {
- ActiveAuditManagerS3A.this.afterError(request, response, e);
- }
-
- @Override
- public AmazonWebServiceRequest beforeMarshalling(
- final AmazonWebServiceRequest request) {
- return ActiveAuditManagerS3A.this.beforeMarshalling(request);
+ @Override
+ public void afterTransmission(Context.AfterTransmission context,
+ ExecutionAttributes executionAttributes) {
+ try {
+ extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .afterTransmission(context, executionAttributes);
+ } catch (AuditFailureException e) {
+ ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
+ throw e;
}
+ }
- @Override
- public void beforeRequest(final Request<?> request) {
- ActiveAuditManagerS3A.this.beforeRequest(request);
+ @Override
+ public SdkHttpResponse modifyHttpResponse(Context.ModifyHttpResponse context,
+ ExecutionAttributes executionAttributes) {
+ try {
+ return extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .modifyHttpResponse(context, executionAttributes);
+ } catch (AuditFailureException e) {
+ ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
+ throw e;
}
+ }
- @Override
- public void beforeAttempt(
- final HandlerBeforeAttemptContext context) {
- ActiveAuditManagerS3A.this.beforeAttempt(context);
+ @Override
+ public void beforeUnmarshalling(Context.BeforeUnmarshalling context,
+ ExecutionAttributes executionAttributes) {
+ try {
+ extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .beforeUnmarshalling(context, executionAttributes);
+ } catch (AuditFailureException e) {
+ ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
+ throw e;
}
+ }
- @Override
- public HttpResponse beforeUnmarshalling(
- final Request<?> request,
- final HttpResponse httpResponse) {
- return ActiveAuditManagerS3A.this.beforeUnmarshalling(request,
- httpResponse);
+ @Override
+ public void afterUnmarshalling(Context.AfterUnmarshalling context,
+ ExecutionAttributes executionAttributes) {
+ try {
+ extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .afterUnmarshalling(context, executionAttributes);
+ } catch (AuditFailureException e) {
+ ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
+ throw e;
}
+ }
- @Override
- public void afterAttempt(
- final HandlerAfterAttemptContext context) {
- ActiveAuditManagerS3A.this.afterAttempt(context);
+ @Override
+ public SdkResponse modifyResponse(Context.ModifyResponse context,
+ ExecutionAttributes executionAttributes) {
+ try {
+ return extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .modifyResponse(context, executionAttributes);
+ } catch (AuditFailureException e) {
+ ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
+ throw e;
}
}
@@ -748,9 +778,8 @@ public final class ActiveAuditManagerS3A
* {@inheritDoc}
*/
@Override
- public <T extends AmazonWebServiceRequest> T requestCreated(
- final T request) {
- return span.requestCreated(request);
+ public void requestCreated(final SdkRequest.Builder builder) {
+ span.requestCreated(builder);
}
/**
@@ -774,79 +803,132 @@ public final class ActiveAuditManagerS3A
/**
* Forward to the inner span.
- * @param request request
- * @param <T> type of request
- * @return an updated request.
+ * {@inheritDoc}
*/
@Override
- public <T extends AmazonWebServiceRequest> T beforeExecution(
- final T request) {
- return span.beforeExecution(request);
+ public void beforeExecution(Context.BeforeExecution context,
+ ExecutionAttributes executionAttributes) {
+ span.beforeExecution(context, executionAttributes);
}
/**
* Forward to the inner span.
- * @param request request
- * @param response response.
+ * {@inheritDoc}
*/
@Override
- public void afterResponse(final Request<?> request,
- final Response<?> response) {
- span.afterResponse(request, response);
+ public void afterExecution(Context.AfterExecution context,
+ ExecutionAttributes executionAttributes) {
+ span.afterExecution(context, executionAttributes);
}
/**
* Forward to the inner span.
- * @param request request
- * @param response response.
- * @param exception exception raised.
+ * {@inheritDoc}
*/
@Override
- public void afterError(final Request<?> request,
- final Response<?> response,
- final Exception exception) {
- span.afterError(request, response, exception);
+ public void onExecutionFailure(Context.FailedExecution context,
+ ExecutionAttributes executionAttributes) {
+ span.onExecutionFailure(context, executionAttributes);
}
/**
* Forward to the inner span.
- * @param request request
- * @return request to marshall
+ * {@inheritDoc}
*/
@Override
- public AmazonWebServiceRequest beforeMarshalling(
- final AmazonWebServiceRequest request) {
- return span.beforeMarshalling(request);
+ public void beforeMarshalling(Context.BeforeMarshalling context,
+ ExecutionAttributes executionAttributes) {
+ span.beforeMarshalling(context, executionAttributes);
}
/**
* Forward to the inner span.
- * @param request request
+ * {@inheritDoc}
*/
@Override
- public void beforeRequest(final Request<?> request) {
- span.beforeRequest(request);
+ public SdkRequest modifyRequest(Context.ModifyRequest context,
+ ExecutionAttributes executionAttributes) {
+ return span.modifyRequest(context, executionAttributes);
}
/**
* Forward to the inner span.
- * @param context full context, including the request.
+ * {@inheritDoc}
*/
@Override
- public void beforeAttempt(
- final HandlerBeforeAttemptContext context) {
- span.beforeAttempt(context);
+ public void afterMarshalling(Context.AfterMarshalling context,
+ ExecutionAttributes executionAttributes) {
+ span.afterMarshalling(context, executionAttributes);
}
/**
* Forward to the inner span.
- *
- * @param context full context, including the request.
+ * {@inheritDoc}
*/
@Override
- public void afterAttempt(
- final HandlerAfterAttemptContext context) {
- span.afterAttempt(context);
+ public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context,
+ ExecutionAttributes executionAttributes) {
+ return span.modifyHttpRequest(context, executionAttributes);
+ }
+
+ /**
+ * Forward to the inner span.
+ * {@inheritDoc}
+ */
+ @Override
+ public void beforeTransmission(Context.BeforeTransmission context,
+ ExecutionAttributes executionAttributes) {
+ span.beforeTransmission(context, executionAttributes);
+ }
+
+ /**
+ * Forward to the inner span.
+ * {@inheritDoc}
+ */
+ @Override
+ public void afterTransmission(Context.AfterTransmission context,
+ ExecutionAttributes executionAttributes) {
+ span.afterTransmission(context, executionAttributes);
+ }
+
+ /**
+ * Forward to the inner span.
+ * {@inheritDoc}
+ */
+ @Override
+ public SdkHttpResponse modifyHttpResponse(Context.ModifyHttpResponse context,
+ ExecutionAttributes executionAttributes) {
+ return span.modifyHttpResponse(context, executionAttributes);
+ }
+
+ /**
+ * Forward to the inner span.
+ * {@inheritDoc}
+ */
+ @Override
+ public void beforeUnmarshalling(Context.BeforeUnmarshalling context,
+ ExecutionAttributes executionAttributes) {
+ span.beforeUnmarshalling(context, executionAttributes);
+ }
+
+ /**
+ * Forward to the inner span.
+ * {@inheritDoc}
+ */
+ @Override
+ public void afterUnmarshalling(Context.AfterUnmarshalling context,
+ ExecutionAttributes executionAttributes) {
+ span.afterUnmarshalling(context, executionAttributes);
+ }
+
+ /**
+ * Forward to the inner span.
+ * {@inheritDoc}
+ */
+ @Override
+ public SdkResponse modifyResponse(Context.ModifyResponse context,
+ ExecutionAttributes executionAttributes) {
+ return span.modifyResponse(context, executionAttributes);
}
@Override
@@ -859,5 +941,4 @@ public final class ActiveAuditManagerS3A
return sb.toString();
}
}
-
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java
index fcf2015487c4..3a2d9d7f823e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java
@@ -24,10 +24,14 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
-import com.amazonaws.AmazonWebServiceRequest;
-import com.amazonaws.services.s3.model.DeleteObjectRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.awscore.AwsExecutionAttribute;
+import software.amazon.awssdk.core.SdkRequest;
+import software.amazon.awssdk.core.interceptor.Context;
+import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
+import software.amazon.awssdk.http.SdkHttpRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -258,21 +262,22 @@ public class LoggingAuditor
/**
* Attach Range of data for GetObject Request.
- * @param request given get object request
+ * @param request the sdk request to be modified
+ * @param executionAttributes execution attributes for this request
*/
- private void attachRangeFromRequest(AmazonWebServiceRequest request) {
- if (request instanceof GetObjectRequest) {
- long[] rangeValue = ((GetObjectRequest) request).getRange();
- if (rangeValue == null || rangeValue.length == 0) {
- return;
- }
- if (rangeValue.length != 2) {
- WARN_INCORRECT_RANGE.warn("Expected range to contain 0 or 2 elements."
- + " Got {} elements. Ignoring.", rangeValue.length);
- return;
+ private void attachRangeFromRequest(SdkHttpRequest request,
+ ExecutionAttributes executionAttributes) {
+
+ String operationName = executionAttributes.getAttribute(AwsExecutionAttribute.OPERATION_NAME);
+
+ if (operationName != null && operationName.equals("GetObject")) {
+ if (request.headers() != null && request.headers().get("Range") != null) {
+ String[] rangeHeader = request.headers().get("Range").get(0).split("=");
+ // only set header if range unit is bytes
+ if (rangeHeader[0].equals("bytes")) {
+ referrer.set(AuditConstants.PARAM_RANGE, rangeHeader[1]);
+ }
}
- String combinedRangeValue = String.format("%d-%d", rangeValue[0], rangeValue[1]);
- referrer.set(AuditConstants.PARAM_RANGE, combinedRangeValue);
}
}
@@ -346,64 +351,78 @@ public class LoggingAuditor
referrer.set(key, value);
}
+
+
/**
- * Before execution, the logging auditor always builds
- * the referrer header, saves to the outer class
- * (where {@link #getLastHeader()} can retrieve it,
+ * Before transmitting a request, the logging auditor
+ * always builds the referrer header, saves to the outer
+ * class (where {@link #getLastHeader()} can retrieve it,
* and logs at debug.
* If configured to add the header to the S3 logs, it will
* be set as the HTTP referrer.
- * @param request request
- * @param <T> type of request.
- * @return the request with any extra headers.
+ * @param context The current state of the execution,
+ * including the SDK and current HTTP request.
+ * @param executionAttributes A mutable set of attributes scoped
+ * to one specific request/response
+ * cycle that can be used to give data
+ * to future lifecycle methods.
+ * @return The potentially-modified HTTP request that should be
+ * sent to the service. Must not be null.
*/
@Override
- public <T extends AmazonWebServiceRequest> T beforeExecution(
- final T request) {
+ public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context,
+ ExecutionAttributes executionAttributes) {
+ SdkHttpRequest httpRequest = context.httpRequest();
+ SdkRequest sdkRequest = context.request();
+
// attach range for GetObject requests
- attachRangeFromRequest(request);
+ attachRangeFromRequest(httpRequest, executionAttributes);
+
// for delete op, attach the number of files to delete
- attachDeleteKeySizeAttribute(request);
+ attachDeleteKeySizeAttribute(sdkRequest);
+
// build the referrer header
final String header = referrer.buildHttpReferrer();
// update the outer class's field.
setLastHeader(header);
if (headerEnabled) {
// add the referrer header
- request.putCustomRequestHeader(HEADER_REFERRER,
- header);
+ httpRequest = httpRequest.toBuilder()
+ .appendHeader(HEADER_REFERRER, header)
+ .build();
}
if (LOG.isDebugEnabled()) {
LOG.debug("[{}] {} Executing {} with {}; {}",
currentThreadID(),
getSpanId(),
getOperationName(),
- analyzer.analyze(request),
+ analyzer.analyze(context.request()),
header);
}
+
// now see if the request is actually a blocked multipart request
- if (!isMultipartUploadEnabled && isRequestMultipartIO(request)) {
+ if (!isMultipartUploadEnabled && isRequestMultipartIO(sdkRequest)) {
throw new AuditOperationRejectedException("Multipart IO request "
- + request + " rejected " + header);
+ + sdkRequest + " rejected " + header);
}
- return request;
+ return httpRequest;
}
/**
* For delete requests, attach delete key size as a referrer attribute.
*
* @param request the request object.
- * @param <T> type of the request.
*/
- private <T extends AmazonWebServiceRequest> void attachDeleteKeySizeAttribute(T request) {
+ private void attachDeleteKeySizeAttribute(SdkRequest request) {
+
if (request instanceof DeleteObjectsRequest) {
- int keySize = ((DeleteObjectsRequest) request).getKeys().size();
- this.set(DELETE_KEYS_SIZE, String.valueOf(keySize));
+ int keySize = ((DeleteObjectsRequest) request).delete().objects().size();
+ referrer.set(DELETE_KEYS_SIZE, String.valueOf(keySize));
} else if (request instanceof DeleteObjectRequest) {
- String key = ((DeleteObjectRequest) request).getKey();
+ String key = ((DeleteObjectRequest) request).key();
if (key != null && key.length() > 0) {
- this.set(DELETE_KEYS_SIZE, "1");
+ referrer.set(DELETE_KEYS_SIZE, "1");
}
}
}
@@ -460,15 +479,13 @@ public class LoggingAuditor
}
@Override
- public <T extends AmazonWebServiceRequest> T requestCreated(
- final T request) {
+ public void requestCreated(final SdkRequest.Builder builder) {
String error = "Creating a request outside an audit span "
- + analyzer.analyze(request);
+ + analyzer.analyze(builder.build());
LOG.info(error);
if (LOG.isDebugEnabled()) {
LOG.debug(error, new AuditFailureException("unaudited"));
}
- return request;
}
/**
@@ -476,20 +493,22 @@ public class LoggingAuditor
* increment the failure count.
* Some requests (e.g. copy part) are not expected in spans due
* to how they are executed; these do not trigger failures.
- * @param request request
- * @param <T> type of request
- * @return an updated request.
- * @throws AuditFailureException if failure is enabled.
+ * @param context The current state of the execution, including
+ * the unmodified SDK request from the service
+ * client call.
+ * @param executionAttributes A mutable set of attributes scoped
+ * to one specific request/response
+ * cycle that can be used to give data
+ * to future lifecycle methods.
*/
@Override
- public <T extends AmazonWebServiceRequest> T beforeExecution(
- final T request) {
-
+ public void beforeExecution(Context.BeforeExecution context,
+ ExecutionAttributes executionAttributes) {
String error = "executing a request outside an audit span "
- + analyzer.analyze(request);
+ + analyzer.analyze(context.request());
final String unaudited = getSpanId() + " "
+ UNAUDITED_OPERATION + " " + error;
- if (isRequestNotAlwaysInSpan(request)) {
+ if (isRequestNotAlwaysInSpan(context.request())) {
// can get by auditing during a copy, so don't overreact
LOG.debug(unaudited);
} else {
@@ -500,7 +519,7 @@ public class LoggingAuditor
}
}
// now hand off to the superclass for its normal preparation
- return super.beforeExecution(request);
+ super.beforeExecution(context, executionAttributes);
}
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditManagerS3A.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditManagerS3A.java
index d1ebd922e073..e58c906460da 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditManagerS3A.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditManagerS3A.java
@@ -24,9 +24,8 @@ import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
-import com.amazonaws.handlers.RequestHandler2;
-import com.amazonaws.services.s3.transfer.Transfer;
-import com.amazonaws.services.s3.transfer.internal.TransferStateChangeListener;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
+import software.amazon.awssdk.transfer.s3.progress.TransferListener;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
@@ -39,6 +38,7 @@ import org.apache.hadoop.fs.s3a.audit.OperationAuditor;
import org.apache.hadoop.fs.s3a.audit.OperationAuditorOptions;
import org.apache.hadoop.service.CompositeService;
+
import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.iostatisticsStore;
/**
@@ -121,17 +121,13 @@ public class NoopAuditManagerS3A extends CompositeService
}
@Override
- public List<RequestHandler2> createRequestHandlers() throws IOException {
+ public List<ExecutionInterceptor> createExecutionInterceptors() throws IOException {
return new ArrayList<>();
}
@Override
- public TransferStateChangeListener createStateChangeListener() {
- return new TransferStateChangeListener() {
- public void transferStateChanged(final Transfer transfer,
- final Transfer.TransferState state) {
- }
- };
+ public TransferListener createTransferListener() {
+ return new TransferListener() {};
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/S3AInternalAuditConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/S3AInternalAuditConstants.java
index f82e3d7f1e5e..c170a2be6611 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/S3AInternalAuditConstants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/S3AInternalAuditConstants.java
@@ -18,10 +18,10 @@
package org.apache.hadoop.fs.s3a.audit.impl;
-import com.amazonaws.handlers.HandlerContextKey;
+import software.amazon.awssdk.core.interceptor.ExecutionAttribute;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.fs.s3a.audit.AWSAuditEventCallbacks;
+import org.apache.hadoop.fs.s3a.audit.AuditSpanS3A;
/**
* Internal constants; not intended for public use, or
@@ -34,11 +34,11 @@ public final class S3AInternalAuditConstants {
}
/**
- * Handler key for audit span callbacks.
- * This is used to bind the handler in the AWS code.
+ * Exceution attribute for audit span callbacks.
+ * This is used to retrieve the span in the AWS code.
*/
- public static final HandlerContextKey<AWSAuditEventCallbacks>
- AUDIT_SPAN_HANDLER_CONTEXT =
- new HandlerContextKey<>(
- "org.apache.hadoop.fs.s3a.audit.AWSAuditEventCallbacks");
+ public static final ExecutionAttribute<AuditSpanS3A>
+ AUDIT_SPAN_EXECUTION_ATTRIBUTE =
+ new ExecutionAttribute<>(
+ "org.apache.hadoop.fs.s3a.audit.AuditSpanS3A");
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java
index 1815285738b0..4754427a4b11 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.fs.s3a.auth;
import javax.annotation.Nullable;
import java.net.URI;
... 15606 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org