You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by da...@apache.org on 2025/10/16 09:12:17 UTC
(jackrabbit-oak) branch trunk updated: OAK-11935 : updating aws sdk from 1.x to 2.x (#2558)
This is an automated email from the ASF dual-hosted git repository.
daim pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 1bcc171b04 OAK-11935 : updating aws sdk from 1.x to 2.x (#2558)
1bcc171b04 is described below
commit 1bcc171b04c1fdadbd5be86661582fa71c543cfe
Author: Rishabh Kumar <ri...@gmail.com>
AuthorDate: Thu Oct 16 14:42:11 2025 +0530
OAK-11935 : updating aws sdk from 1.x to 2.x (#2558)
* OAK-11935 : updating aws sdk from 1.x to 2.x
* OAK-11935 : added more unit cases
* OAK-11935 : added unit cases for S3RequestDecorator
* OAK-11935 : fixed issue when running ITs with different encryption mode
* OAK-11935 : removed unused imports
* OAK-11935 : incorporated review comments
* OAK-11935 : added review comments
* OAK-11935 : added properties for cross Region access
* OAK-11935 : removed additional list for removing keys in case of GCP mode
* OAK-11935 : moved out CRUD operations from Utils to S3CrudHelper
* OAK-11935 : added unit cases for waitForBucket method
* OAK-11935 : make total attemps and delay configurable to use lower values in unit cases
* OAK-11935 : remaned S3CrudHelper to S3BackendHelper
---
oak-blob-cloud/pom.xml | 21 +-
.../oak/blob/cloud/s3/DataEncryption.java} | 25 +-
.../jackrabbit/oak/blob/cloud/s3/S3Backend.java | 799 ++++++++++-----------
.../oak/blob/cloud/s3/S3BackendHelper.java | 226 ++++++
.../jackrabbit/oak/blob/cloud/s3/S3Constants.java | 10 +-
.../oak/blob/cloud/s3/S3RequestDecorator.java | 178 ++---
.../apache/jackrabbit/oak/blob/cloud/s3/Utils.java | 506 +++++++++----
.../oak/blob/cloud/s3/S3BackendHelperTest.java | 137 ++++
.../cloud/s3/S3DataRecordAccessProviderTest.java | 7 +
.../oak/blob/cloud/s3/S3DataStoreUtils.java | 100 ++-
.../oak/blob/cloud/s3/S3RequestDecoratorTest.java | 509 +++++++++++++
.../blob/cloud/s3/TestS3DSWithSSECustomerKey.java | 32 +-
.../oak/blob/cloud/s3/TestS3DSWithSSEKMS.java | 26 +-
.../blob/cloud/s3/TestS3DSWithSSEKMSwithKey.java | 30 +-
.../oak/blob/cloud/s3/TestS3DSWithSSES3.java | 4 +
.../jackrabbit/oak/blob/cloud/s3/TestS3Ds.java | 186 +++--
.../jackrabbit/oak/blob/cloud/s3/UtilsTest.java | 199 +++++
oak-blob-cloud/src/test/resources/aws.properties | 9 +-
.../AbstractDataRecordAccessProviderTest.java | 8 +
.../fixtures/datastore/S3DataStoreFixture.java | 84 +--
oak-parent/pom.xml | 7 +
.../jackrabbit/oak/fixture/DataStoreUtils.java | 49 +-
oak-run-elastic/pom.xml | 3 +-
oak-run/pom.xml | 3 +-
oak-segment-aws/pom.xml | 5 +
25 files changed, 2230 insertions(+), 933 deletions(-)
diff --git a/oak-blob-cloud/pom.xml b/oak-blob-cloud/pom.xml
index 43d4769a28..d8e14737a5 100644
--- a/oak-blob-cloud/pom.xml
+++ b/oak-blob-cloud/pom.xml
@@ -30,10 +30,6 @@
<name>Oak Cloud Blob Store</name>
<packaging>bundle</packaging>
- <properties>
- <aws.version>1.12.791</aws.version>
- </properties>
-
<build>
<plugins>
<plugin>
@@ -132,9 +128,20 @@
<!-- Amazon AWS dependency -->
<dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-s3</artifactId>
- <version>${aws.version}</version>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>s3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>apache-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>s3-transfer-manager</artifactId>
+ </dependency>
+ <dependency>
+ <artifactId>netty-nio-client</artifactId>
+ <groupId>software.amazon.awssdk</groupId>
</dependency>
<dependency>
diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSES3.java b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/DataEncryption.java
similarity index 54%
copy from oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSES3.java
copy to oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/DataEncryption.java
index 7fd2dd4f69..a9e0116b2d 100644
--- a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSES3.java
+++ b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/DataEncryption.java
@@ -14,28 +14,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.jackrabbit.oak.blob.cloud.s3;
-import org.junit.Before;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.jackrabbit.oak.commons.annotations.Internal;
/**
- * Test S3DataStore operation with SSE_S3 encryption.
- * It requires to pass aws config file via system property or system properties by prefixing with 'ds.'.
- * See details @ {@link S3DataStoreUtils}.
- * For e.g. -Dconfig=/opt/cq/aws.properties. Sample aws properties located at
- * src/test/resources/aws.properties
+ * Enum to indicate S3 encryption mode
+ *
*/
-public class TestS3DSWithSSES3 extends TestS3Ds {
-
- protected static final Logger LOG = LoggerFactory.getLogger(TestS3DSWithSSES3.class);
-
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- props.setProperty(S3Constants.S3_ENCRYPTION, S3Constants.S3_ENCRYPTION_SSE_S3);
- }
+@Internal
+public enum DataEncryption {
+ SSE_S3, SSE_KMS, SSE_C, NONE;
}
diff --git a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
index db5c759845..3ff34375de 100644
--- a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
+++ b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
@@ -23,6 +23,7 @@ import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
+import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
@@ -30,20 +31,16 @@ import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
-import java.util.Queue;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
-import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
-import com.amazonaws.services.s3.model.GetObjectRequest;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
@@ -51,8 +48,7 @@ import org.apache.jackrabbit.core.data.DataIdentifier;
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.commons.PropertiesUtil;
-import org.apache.jackrabbit.oak.commons.collections.IterableUtils;
-import org.apache.jackrabbit.oak.commons.collections.ListUtils;
+import org.apache.jackrabbit.oak.commons.collections.IteratorUtils;
import org.apache.jackrabbit.oak.commons.collections.MapUtils;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions;
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUpload;
@@ -64,44 +60,47 @@ import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.AmazonServiceException;
-import com.amazonaws.HttpMethod;
-import com.amazonaws.SdkClientException;
-import com.amazonaws.regions.Regions;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.S3ClientOptions;
-import com.amazonaws.services.s3.model.BucketAccelerateConfiguration;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CopyObjectRequest;
-import com.amazonaws.services.s3.model.CreateBucketRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsResult;
-import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
-import com.amazonaws.services.s3.model.GetBucketAccelerateConfigurationRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ListPartsRequest;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PartETag;
-import com.amazonaws.services.s3.model.PartListing;
-import com.amazonaws.services.s3.model.PartSummary;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.Region;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.amazonaws.services.s3.transfer.Copy;
-import com.amazonaws.services.s3.transfer.TransferManager;
-import com.amazonaws.services.s3.transfer.Upload;
-
import org.apache.jackrabbit.guava.common.cache.Cache;
import org.apache.jackrabbit.guava.common.cache.CacheBuilder;
-import org.apache.jackrabbit.oak.commons.collections.AbstractIterator;
+import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration;
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
+import software.amazon.awssdk.core.async.AsyncRequestBody;
+import software.amazon.awssdk.core.exception.SdkClientException;
+import software.amazon.awssdk.core.sync.ResponseTransformer;
+import software.amazon.awssdk.http.SdkHttpMethod;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.BucketAccelerateStatus;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
+import software.amazon.awssdk.services.s3.model.CompletedPart;
+import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
+import software.amazon.awssdk.services.s3.model.Delete;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse;
+import software.amazon.awssdk.services.s3.model.GetBucketAccelerateConfigurationResponse;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
+import software.amazon.awssdk.services.s3.model.ListPartsResponse;
+import software.amazon.awssdk.services.s3.model.MetadataDirective;
+import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
+import software.amazon.awssdk.services.s3.model.Part;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable;
+import software.amazon.awssdk.services.s3.presigner.S3Presigner;
+import software.amazon.awssdk.transfer.s3.S3TransferManager;
+import software.amazon.awssdk.transfer.s3.model.Copy;
+import software.amazon.awssdk.transfer.s3.model.CopyRequest;
+import software.amazon.awssdk.transfer.s3.model.FileUpload;
+import software.amazon.awssdk.transfer.s3.model.Upload;
import static org.apache.jackrabbit.oak.commons.conditions.Validate.checkArgument;
-import static java.lang.Thread.currentThread;
/**
* A data store backend that stores data on Amazon S3.
@@ -121,28 +120,26 @@ public class S3Backend extends AbstractSharedBackend {
private static final String REF_KEY = "reference.key";
- private static final int MAX_UNIQUE_RECORD_TRIES = 10;
-
static final String PART_NUMBER = "partNumber";
static final String UPLOAD_ID = "uploadId";
- private static final int ONE_MB = 1024*1024;
- static final long MIN_MULTIPART_UPLOAD_PART_SIZE = 1024 * 1024 * 10; // 10MB
- static final long MAX_MULTIPART_UPLOAD_PART_SIZE = 1024 * 1024 * 256; // 256MB
+ static final long MIN_MULTIPART_UPLOAD_PART_SIZE = 1024 * 1024 * 10L; // 10MB
+ static final long MAX_MULTIPART_UPLOAD_PART_SIZE = 1024 * 1024 * 256L; // 256MB
static final long MAX_SINGLE_PUT_UPLOAD_SIZE = 1024L * 1024L * 1024L * 5L; // 5GB, AWS limitation
static final long MAX_BINARY_UPLOAD_SIZE = 1024L * 1024L * 1024L * 1024L * 5L; // 5TB, AWS limitation
private static final int MAX_ALLOWABLE_UPLOAD_URIS = 10000; // AWS limitation
- private AmazonS3Client s3service;
+ private S3Client s3Client;
+ private S3AsyncClient s3AsyncClient;
// needed only in case of transfer acceleration is enabled for presigned URIs
- private AmazonS3Client s3PresignService;
+ private S3Presigner s3PresignService;
private String bucket;
private byte[] secret;
- private TransferManager tmx;
+ private S3TransferManager tmx;
private Properties properties;
@@ -167,32 +164,24 @@ public class S3Backend extends AbstractSharedBackend {
LOG.debug("init");
+ final String region = Utils.getRegion(properties);
+
+ s3Client = Utils.openService(properties, false);
+
+ setBinaryTransferAccelerationEnabled(
+ Boolean.parseBoolean(
+ properties.getProperty(S3Constants.PRESIGNED_URI_ENABLE_ACCELERATION, "false")));
+
+ s3AsyncClient = Utils.openAsyncService(properties);
s3ReqDecorator = new S3RequestDecorator(properties);
- s3service = Utils.openService(properties);
- s3PresignService = s3service;
- if (bucket == null || "".equals(bucket.trim())) {
+ if (bucket == null || bucket.trim().isEmpty()) {
bucket = properties.getProperty(S3Constants.S3_BUCKET);
// Alternately check if the 'container' property is set
if (StringUtils.isEmpty(bucket)) {
bucket = properties.getProperty(S3Constants.S3_CONTAINER);
}
}
- String region = properties.getProperty(S3Constants.S3_REGION);
-
- if (StringUtils.isEmpty(region)) {
- com.amazonaws.regions.Region ec2Region = Regions.getCurrentRegion();
- if (ec2Region != null) {
- region = ec2Region.getName();
- } else {
- throw new AmazonClientException(
- "parameter ["
- + S3Constants.S3_REGION
- + "] not configured and cannot be derived from environment");
- }
- } else if (Utils.DEFAULT_AWS_BUCKET_REGION.equals(region)) {
- region = Region.US_Standard.toString();
- }
createBucketIfNeeded(region);
@@ -202,13 +191,15 @@ public class S3Backend extends AbstractSharedBackend {
writeThreads = Integer.parseInt(writeThreadsStr);
}
LOG.info("Using thread pool of [{}] threads in S3 transfer manager.", writeThreads);
- tmx = new TransferManager(s3service, Executors.newFixedThreadPool(writeThreads,
- BasicThreadFactory.builder().namingPattern("s3-transfer-manager-worker-%d").build()));
+ tmx = S3TransferManager.builder()
+ .s3Client(s3AsyncClient)
+ .executor(Executors.
+ newFixedThreadPool(writeThreads,
+ BasicThreadFactory.builder().namingPattern("s3-transfer-manager-worker-%d").build()))
+ .build();
String renameKeyProp = properties.getProperty(S3Constants.S3_RENAME_KEYS);
- boolean renameKeyBool = (renameKeyProp == null || "".equals(renameKeyProp))
- ? false
- : Boolean.parseBoolean(renameKeyProp);
+ boolean renameKeyBool = renameKeyProp != null && !renameKeyProp.isEmpty() && Boolean.parseBoolean(renameKeyProp);
LOG.info("Rename keys [{}]", renameKeyBool);
if (renameKeyBool) {
renameKeys();
@@ -235,19 +226,15 @@ public class S3Backend extends AbstractSharedBackend {
setHttpDownloadURICacheSize(cacheMaxSize);
}
- String enablePresignedAccelerationStr = properties.getProperty(S3Constants.PRESIGNED_URI_ENABLE_ACCELERATION);
- setBinaryTransferAccelerationEnabled(enablePresignedAccelerationStr != null && "true".equals(enablePresignedAccelerationStr));
-
presignedDownloadURIVerifyExists =
PropertiesUtil.toBoolean(properties.get(S3Constants.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS), true);
// Initialize reference key secret
getOrCreateReferenceKey();
- LOG.debug("S3 Backend initialized in [{}] ms",
- +(System.currentTimeMillis() - startTime.getTime()));
+ LOG.debug("S3 Backend initialized in [{}] ms", (System.currentTimeMillis() - startTime.getTime()));
} catch (Exception e) {
- LOG.error("Error ", e);
+ LOG.error("Error while initializing S3 Backend", e);
Map<String, Object> filteredMap = new HashMap<>();
if (properties != null) {
filteredMap = MapUtils.filterKeys(Utils.asMap(properties),
@@ -263,18 +250,24 @@ public class S3Backend extends AbstractSharedBackend {
private void createBucketIfNeeded(final String region) {
try {
- if (!s3service.doesBucketExist(bucket)) {
- String bucketRegion = region;
+ if (!S3BackendHelper.bucketExists(s3Client, bucket)) {
+ String bucketRegion;
if (Utils.US_EAST_1_AWS_BUCKET_REGION.equals(region)) {
// The SDK has changed such that if the region is us-east-1
// the region value should not be provided in the
// request to create the bucket.
// See https://stackoverflow.com/questions/51912072/invalidlocationconstraint-error-while-creating-s3-bucket-when-the-used-command-i
bucketRegion = null;
+ } else {
+ bucketRegion = region;
}
- CreateBucketRequest req = new CreateBucketRequest(bucket, bucketRegion);
- s3service.createBucket(req);
- if (Utils.waitForBucket(s3service, bucket)) {
+ s3Client.createBucket(createBucket ->
+ createBucket.bucket(bucket).
+ createBucketConfiguration(config ->
+ config.locationConstraint(bucketRegion)
+ .build())
+ .build());
+ if (S3BackendHelper.waitForBucket(s3Client, bucket, 20, 100L)) {
LOG.error("Bucket [{}] does not exist in [{}] and was not automatically created",
bucket, region);
return;
@@ -285,29 +278,26 @@ public class S3Backend extends AbstractSharedBackend {
}
}
catch (SdkClientException awsException) {
- LOG.error("Attempt to create S3 bucket [{}] in [{}] failed",
- bucket, region, awsException);
+ LOG.error("Attempt to create S3 bucket [{}] in [{}] failed", bucket, region, awsException);
}
}
- void setBinaryTransferAccelerationEnabled(boolean enabled) {
- if (enabled) {
- // verify acceleration is enabled on the bucket
- BucketAccelerateConfiguration accelerateConfig = s3service.getBucketAccelerateConfiguration(new GetBucketAccelerateConfigurationRequest(bucket));
- if (accelerateConfig.isAccelerateEnabled()) {
- // If transfer acceleration is enabled for presigned URIs, we need a separate AmazonS3Client
- // instance with the acceleration mode enabled, because we don't want the requests from the
- // data store itself to S3 to use acceleration
- s3PresignService = Utils.openService(properties);
- s3PresignService.setS3ClientOptions(S3ClientOptions.builder().setAccelerateModeEnabled(true).build());
- LOG.info("S3 Transfer Acceleration enabled for presigned URIs.");
+ void setBinaryTransferAccelerationEnabled(final boolean enabled) {
+ if (!enabled) {
+ s3PresignService = Utils.createPresigner(s3Client, properties);
+ return;
+ }
- } else {
- LOG.warn("S3 Transfer Acceleration is not enabled on the bucket {}. Will create normal, non-accelerated presigned URIs.",
- bucket, S3Constants.PRESIGNED_URI_ENABLE_ACCELERATION);
- }
+ GetBucketAccelerateConfigurationResponse accelerateConfig = s3Client.getBucketAccelerateConfiguration(
+ b -> b.bucket(bucket).build());
+
+ if (Objects.equals(BucketAccelerateStatus.ENABLED, accelerateConfig.status())) {
+ s3PresignService = Utils.createPresigner(Utils.openService(properties, true), properties);
+ LOG.info("S3 Transfer Acceleration enabled for presigned URIs.");
} else {
- s3PresignService = s3service;
+ LOG.warn("S3 Transfer Acceleration is not enabled on the bucket {}. Will create normal, non-accelerated presigned URIs. To enable set {}",
+ bucket, S3Constants.PRESIGNED_URI_ENABLE_ACCELERATION);
+ s3PresignService = Utils.createPresigner(s3Client, properties);
}
}
@@ -316,57 +306,72 @@ public class S3Backend extends AbstractSharedBackend {
* method uses parallel concurrent connections to upload.
*/
@Override
- public void write(DataIdentifier identifier, File file)
- throws DataStoreException {
+ public void write(DataIdentifier identifier, File file) throws DataStoreException {
String key = getKeyName(identifier);
- ObjectMetadata objectMetaData = null;
long start = System.currentTimeMillis();
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
- Thread.currentThread().setContextClassLoader(
- getClass().getClassLoader());
+ Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
// check if the same record already exists
+ HeadObjectResponse headMeta = null;
try {
- objectMetaData = s3service.getObjectMetadata(s3ReqDecorator.decorate(new GetObjectMetadataRequest(bucket, key)));
- } catch (AmazonServiceException ase) {
- if (!(ase.getStatusCode() == 404 || ase.getStatusCode() == 403)) {
- throw ase;
+ headMeta = s3Client.headObject(s3ReqDecorator.decorate(
+ HeadObjectRequest.builder()
+ .bucket(bucket)
+ .key(key)
+ .build()));
+ } catch (S3Exception se) {
+ if (!(se instanceof NoSuchKeyException || se.statusCode() == 404 || se.statusCode() == 403)) {
+ throw se;
}
}
- if (objectMetaData != null) {
- long l = objectMetaData.getContentLength();
+ if (headMeta != null) {
+ long l = headMeta.contentLength();
if (l != file.length()) {
- throw new DataStoreException("Collision: " + key
- + " new length: " + file.length() + " old length: " + l);
+ throw new DataStoreException("Collision: " + key + " new length: " + file.length() + " old length: " + l);
}
- LOG.debug("[{}]'s exists, lastmodified = [{}]", key,
- objectMetaData.getLastModified().getTime());
- CopyObjectRequest copReq = new CopyObjectRequest(bucket, key,
- bucket, key);
- LOG.warn("Object MetaData before copy: {}", objectMetaData.getRawMetadata());
+ LOG.debug("[{}]'s exists, lastmodified = [{}]", key, headMeta.lastModified().toEpochMilli());
+ CopyObjectRequest copyReq =
+ CopyObjectRequest.builder()
+ .sourceBucket(bucket)
+ .sourceKey(key)
+ .destinationBucket(bucket)
+ .destinationKey(key)
+ .metadataDirective(MetadataDirective.REPLACE) // Required to update metadata and reset creation time
+ .build();
+ LOG.warn("Object MetaData before copy: {}", headMeta.metadata());
if (Objects.equals(RemoteStorageMode.S3, properties.get(S3Constants.MODE))) {
- copReq.setNewObjectMetadata(objectMetaData);
+ copyReq = copyReq.toBuilder().metadata(headMeta.metadata()).build();
}
- Copy copy = tmx.copy(s3ReqDecorator.decorate(copReq));
try {
- copy.waitForCopyResult();
+ Copy copy = tmx.copy(CopyRequest.builder()
+ .copyObjectRequest(
+ s3ReqDecorator.decorate(copyReq))
+ .build());
+ copy.completionFuture().join();
LOG.debug("lastModified of [{}] updated successfully.", identifier);
- }catch (Exception e2) {
- throw new DataStoreException("Could not upload " + key, e2);
+ } catch (Exception e2) {
+ throw new DataStoreException("Could not upload " + key, e2.getCause());
}
}
- if (objectMetaData == null) {
+ if (headMeta == null) {
try {
// start multipart parallel upload using amazon sdk
- Upload up = tmx.upload(s3ReqDecorator.decorate(new PutObjectRequest(
- bucket, key, file)));
+ FileUpload up = tmx.uploadFile(uploadReq ->
+ uploadReq.source(file).
+ putObjectRequest(
+ s3ReqDecorator.decorate(
+ PutObjectRequest.builder().bucket(bucket).key(key)
+ .build()))
+ .build());
+
if (LOG_STREAMS_UPLOAD.isDebugEnabled()) {
// Log message, with exception so we can get a trace to see where the call came from
LOG_STREAMS_UPLOAD.debug("Binary uploaded to S3 - identifier={}", key, new Exception());
}
// wait for upload to finish
- up.waitForUploadResult();
+ up.completionFuture().join();
LOG.debug("synchronous upload to identifier [{}] completed.", identifier);
} catch (Exception e2 ) {
throw new DataStoreException("Could not upload " + key, e2);
@@ -377,8 +382,7 @@ public class S3Backend extends AbstractSharedBackend {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
- LOG.debug("write of [{}], length=[{}], in [{}]ms",
- identifier, file.length(), (System.currentTimeMillis() - start));
+ LOG.debug("write of [{}], length=[{}], in [{}]ms", identifier, file.length(), System.currentTimeMillis() - start);
}
/**
@@ -391,21 +395,10 @@ public class S3Backend extends AbstractSharedBackend {
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- ObjectMetadata objectMetaData = s3service.getObjectMetadata(s3ReqDecorator.decorate(new GetObjectMetadataRequest(bucket, key)));
- if (objectMetaData != null) {
- LOG.trace("exists [{}]: [true] took [{}] ms.",
- identifier, (System.currentTimeMillis() - start) );
- return true;
- }
- return false;
- } catch (AmazonServiceException e) {
- if (e.getStatusCode() == 404 || e.getStatusCode() == 403) {
- LOG.debug("exists [{}]: [false] took [{}] ms.",
- identifier, (System.currentTimeMillis() - start) );
- return false;
- }
- throw new DataStoreException(
- "Error occured to getObjectMetadata for key [" + identifier.toString() + "]", e);
+ return S3BackendHelper.objectExists(s3Client, bucket, key, s3ReqDecorator);
+ } catch (S3Exception e) {
+ LOG.debug("exists [{}]: [false] took [{}] ms.", identifier, System.currentTimeMillis() - start);
+ throw new DataStoreException("Error occurred to getObjectMetadata for key [" + identifier + "]", e);
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
@@ -421,15 +414,20 @@ public class S3Backend extends AbstractSharedBackend {
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- S3Object object = s3service.getObject(bucket, key);
- InputStream in = object.getObjectContent();
- LOG.debug("[{}] read took [{}]ms", identifier, (System.currentTimeMillis() - start));
+ InputStream in = s3Client.getObject(s3ReqDecorator.decorate(
+ GetObjectRequest.builder()
+ .bucket(bucket)
+ .key(key)
+ .build()),
+ ResponseTransformer.toInputStream());
+
+ LOG.debug("[{}] read took [{}]ms", identifier, System.currentTimeMillis() - start);
if (LOG_STREAMS_DOWNLOAD.isDebugEnabled()) {
// Log message, with exception so we can get a trace to see where the call came from
LOG_STREAMS_DOWNLOAD.debug("Binary downloaded from S3 - identifier={}", key, new Exception());
}
return in;
- } catch (AmazonServiceException e) {
+ } catch (S3Exception e) {
throw new DataStoreException("Object not found: " + key, e);
} finally {
if (contextClassLoader != null) {
@@ -439,9 +437,8 @@ public class S3Backend extends AbstractSharedBackend {
}
@Override
- public Iterator<DataIdentifier> getAllIdentifiers()
- throws DataStoreException {
- return new RecordsIterator<DataIdentifier>(input -> new DataIdentifier(getIdentifierName(input.getKey())));
+ public Iterator<DataIdentifier> getAllIdentifiers() {
+ return getAllRecords(s3Object -> new DataIdentifier(getIdentifierName(s3Object.key())));
}
@Override
@@ -451,14 +448,11 @@ public class S3Backend extends AbstractSharedBackend {
String key = getKeyName(identifier);
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
- Thread.currentThread().setContextClassLoader(
- getClass().getClassLoader());
- s3service.deleteObject(bucket, key);
- LOG.debug("Identifier [{}] deleted. It took [{}]ms.", new Object[] {
- identifier, (System.currentTimeMillis() - start) });
- } catch (AmazonServiceException e) {
- throw new DataStoreException(
- "Could not delete dataIdentifier " + identifier, e);
+ Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
+ s3Client.deleteObject(delReq -> delReq.bucket(bucket).key(key).build());
+ LOG.debug("Identifier [{}] deleted. It took [{}]ms.", identifier, (System.currentTimeMillis() - start));
+ } catch (S3Exception e) {
+ throw new DataStoreException("Could not delete dataIdentifier " + identifier, e);
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
@@ -469,11 +463,23 @@ public class S3Backend extends AbstractSharedBackend {
@Override
public void close() {
// backend is closing. abort all mulitpart uploads from start.
- if(s3service.doesBucketExist(bucket)) {
- tmx.abortMultipartUploads(bucket, startTime);
+ try {
+ if(S3BackendHelper.bucketExists(s3Client, bucket)) {
+ // List and abort multipart uploads initiated before startTime
+ S3BackendHelper.abortMultipartUpload(bucket, startTime, s3Client);
+ }
+ } finally {
+ if (tmx != null) {
+ tmx.close();
+ }
+ if (s3AsyncClient != null) {
+ s3AsyncClient.close();
+ }
+ if (s3PresignService != null) {
+ s3PresignService.close();
+ }
+ s3Client.close();
}
- tmx.shutdownNow();
- s3service.shutdown();
LOG.info("S3Backend closed.");
}
@@ -493,20 +499,7 @@ public class S3Backend extends AbstractSharedBackend {
*/
public void setProperties(Properties properties) {
this.properties = properties;
- setRemoteStorageMode();
- }
-
- private void setRemoteStorageMode() {
- String s3EndPoint = properties.getProperty(S3Constants.S3_END_POINT, "");
- if (s3EndPoint.contains("googleapis")) {
- if (properties.get(S3Constants.MODE) == RemoteStorageMode.S3) {
- LOG.warn("Mismatch between remote storage mode and s3EndPoint, overriding mode to GCP");
- }
- properties.put(S3Constants.MODE, RemoteStorageMode.GCP);
- return;
- }
- // default mode is S3
- properties.put(S3Constants.MODE, RemoteStorageMode.S3);
+ Utils.setRemoteStorageMode(this.properties);
}
@Override
@@ -516,19 +509,34 @@ public class S3Backend extends AbstractSharedBackend {
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
+ // Executor required to handle reading from the InputStream on a separate thread so the main upload is not blocked.
+ final ExecutorService executor = Executors.newSingleThreadExecutor();
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
-
- Upload upload = tmx.upload(s3ReqDecorator
- .decorate(new PutObjectRequest(bucket, addMetaKeyPrefix(name), input, new ObjectMetadata())));
- upload.waitForUploadResult();
- } catch (InterruptedException e) {
- LOG.error("Error in uploading", e);
- throw new DataStoreException("Error in uploading", e);
+ // Specify `null` for the content length when you don't know the content length.
+ final AsyncRequestBody body = AsyncRequestBody.fromInputStream(input, null, executor);
+ final Upload upload = tmx.upload(uploadReq ->
+ uploadReq.requestBody(body).
+ putObjectRequest(
+ s3ReqDecorator.decorate(PutObjectRequest.builder().bucket(bucket).key(addMetaKeyPrefix(name)).build()))
+ .build());
+ upload.completionFuture().join();
+ } catch (Exception e) {
+ LOG.error("Exception in uploading {}", e.getMessage());
+ throw new DataStoreException("Error in uploading metadata file", e);
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
+ executor.shutdown();
+ try {
+ if (!executor.awaitTermination(60, TimeUnit.SECONDS)) {
+ executor.shutdownNow();
+ }
+ } catch (InterruptedException e) {
+ executor.shutdownNow();
+ Thread.currentThread().interrupt();
+ }
}
}
@@ -541,11 +549,16 @@ public class S3Backend extends AbstractSharedBackend {
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- Upload upload = tmx.upload(s3ReqDecorator
- .decorate(new PutObjectRequest(bucket, addMetaKeyPrefix(name), input)));
- upload.waitForUploadResult();
- } catch (InterruptedException e) {
- LOG.error("Exception in uploading metadata file {}", new Object[] {input, e});
+ FileUpload upload = tmx.uploadFile(uploadReq ->
+ uploadReq.source(input).
+ putObjectRequest(
+ s3ReqDecorator.decorate(
+ PutObjectRequest.builder().bucket(bucket).key(addMetaKeyPrefix(name)).build()))
+ .build());
+
+ upload.completionFuture().join();
+ } catch (Exception e) {
+ LOG.error("Exception in uploading metadata file {}", input);
throw new DataStoreException("Error in uploading metadata file", e);
} finally {
if (contextClassLoader != null) {
@@ -562,9 +575,9 @@ public class S3Backend extends AbstractSharedBackend {
try {
Thread.currentThread().setContextClassLoader(
getClass().getClassLoader());
- ObjectMetadata meta = s3service.getObjectMetadata(bucket, addMetaKeyPrefix(name));
- return new S3DataRecord(this, s3service, bucket, new DataIdentifier(name),
- meta.getLastModified().getTime(), meta.getContentLength(), true, s3ReqDecorator);
+ HeadObjectResponse meta = s3Client.headObject(s3ReqDecorator.decorate(HeadObjectRequest.builder().bucket(bucket).key(addMetaKeyPrefix(name)).build()));
+ return new S3DataRecord(this, s3Client, bucket, new DataIdentifier(name),
+ meta.lastModified().toEpochMilli(), meta.contentLength(), true, s3ReqDecorator);
} catch(Exception e) {
LOG.error("Error getting metadata record for {}", name, e);
}
@@ -580,18 +593,17 @@ public class S3Backend extends AbstractSharedBackend {
public List<DataRecord> getAllMetadataRecords(String prefix) {
checkArgument(null != prefix, "prefix should not be null");
- List<DataRecord> metadataList = new ArrayList<DataRecord>();
+ List<DataRecord> metadataList = new ArrayList<>();
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
- Thread.currentThread().setContextClassLoader(
- getClass().getClassLoader());
- ListObjectsRequest listObjectsRequest =
- new ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix));
- ObjectListing prevObjectListing = s3service.listObjects(listObjectsRequest);
- for (final S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
- metadataList.add(new S3DataRecord(this, s3service, bucket,
- new DataIdentifier(stripMetaKeyPrefix(s3ObjSumm.getKey())),
- s3ObjSumm.getLastModified().getTime(), s3ObjSumm.getSize(), true, s3ReqDecorator));
+ Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
+ final ListObjectsV2Iterable prevObjectListing = s3Client.listObjectsV2Paginator(listReq ->
+ listReq.bucket(bucket).prefix(addMetaKeyPrefix(prefix)).build());
+
+ for (final S3Object s3ObjSumm : prevObjectListing.contents()) {
+ metadataList.add(new S3DataRecord(this, s3Client, bucket,
+ new DataIdentifier(stripMetaKeyPrefix(s3ObjSumm.key())),
+ s3ObjSumm.lastModified().toEpochMilli(), s3ObjSumm.size(), true, s3ReqDecorator));
}
} finally {
if (contextClassLoader != null) {
@@ -609,7 +621,7 @@ public class S3Backend extends AbstractSharedBackend {
try {
Thread.currentThread().setContextClassLoader(
getClass().getClassLoader());
- s3service.deleteObject(bucket, addMetaKeyPrefix(name));
+ s3Client.deleteObject(delReq -> delReq.bucket(bucket).key(addMetaKeyPrefix(name)).build());
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
@@ -624,29 +636,13 @@ public class S3Backend extends AbstractSharedBackend {
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
- Thread.currentThread().setContextClassLoader(
- getClass().getClassLoader());
+ Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- ListObjectsRequest listObjectsRequest =
- new ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix));
- ObjectListing metaList = s3service.listObjects(listObjectsRequest);
- List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
- List<String> keysToDelete = new ArrayList<>();
- for (S3ObjectSummary s3ObjSumm : metaList.getObjectSummaries()) {
- deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
- keysToDelete.add(s3ObjSumm.getKey());
- }
- if (!deleteList.isEmpty()) {
- RemoteStorageMode mode = (RemoteStorageMode) properties.getOrDefault(S3Constants.MODE, RemoteStorageMode.S3);
- if (mode == RemoteStorageMode.S3) {
- DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
- delObjsReq.setKeys(deleteList);
- s3service.deleteObjects(delObjsReq);
- } else {
- // GCP does not support bulk delete operations, hence we need to delete each object individually
- keysToDelete.forEach(key -> s3service.deleteObject(bucket, key));
- }
- }
+ ListObjectsV2Iterable metaList = s3Client.listObjectsV2Paginator(builder -> builder
+ .bucket(bucket)
+ .prefix(addMetaKeyPrefix(prefix))
+ .build());
+ S3BackendHelper.deleteBucketObjects(bucket, properties, s3Client, metaList);
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
@@ -657,9 +653,9 @@ public class S3Backend extends AbstractSharedBackend {
@Override
public Iterator<DataRecord> getAllRecords() {
final AbstractSharedBackend backend = this;
- return new RecordsIterator<DataRecord>(
- input -> new S3DataRecord(backend, s3service, bucket, new DataIdentifier(getIdentifierName(input.getKey())),
- input.getLastModified().getTime(), input.getSize(), s3ReqDecorator));
+ return getAllRecords(
+ s3Object -> new S3DataRecord(backend, s3Client, bucket, new DataIdentifier(getIdentifierName(s3Object.key())),
+ s3Object.lastModified().toEpochMilli(), s3Object.size(), s3ReqDecorator));
}
@Override
@@ -670,15 +666,20 @@ public class S3Backend extends AbstractSharedBackend {
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- ObjectMetadata object = s3service.getObjectMetadata(s3ReqDecorator.decorate(new GetObjectMetadataRequest(bucket, key)));
- S3DataRecord record = new S3DataRecord(this, s3service, bucket, identifier,
- object.getLastModified().getTime(), object.getContentLength(), s3ReqDecorator);
+ HeadObjectResponse object = s3Client.headObject(s3ReqDecorator.decorate(
+ HeadObjectRequest.builder()
+ .bucket(bucket)
+ .key(key)
+ .build()));
+ S3DataRecord s3DataRecord = new S3DataRecord(this, s3Client, bucket, identifier,
+ object.lastModified().toEpochMilli(), object.contentLength(), s3ReqDecorator);
LOG.debug("Identifier [{}]'s getRecord = [{}] took [{}]ms.",
- identifier, record, (System.currentTimeMillis() - start));
+ identifier, s3DataRecord, (System.currentTimeMillis() - start));
- return record;
- } catch (AmazonServiceException e) {
- if (e.getStatusCode() == 404 || e.getStatusCode() == 403) {
+ return s3DataRecord;
+ } catch (AwsServiceException e) {
+ int statusCode = e.awsErrorDetails().sdkHttpResponse().statusCode();
+ if (statusCode == 404 || statusCode == 403) {
LOG.debug(
"getRecord:Identifier [{}] not found. Took [{}] ms.",
identifier, (System.currentTimeMillis() - start));
@@ -732,7 +733,7 @@ public class S3Backend extends AbstractSharedBackend {
try {
Thread.currentThread().setContextClassLoader(
getClass().getClassLoader());
- return s3service.doesObjectExist(bucket, addMetaKeyPrefix(name));
+ return S3BackendHelper.objectExists(s3Client, bucket, addMetaKeyPrefix(name), s3ReqDecorator);
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
@@ -759,7 +760,7 @@ public class S3Backend extends AbstractSharedBackend {
return null;
}
- return createPresignedURI(identifier, HttpMethod.PUT, httpUploadURIExpirySeconds);
+ return createPresignedURI(identifier, httpUploadURIExpirySeconds);
}
void setHttpDownloadURIExpirySeconds(int seconds) {
@@ -829,7 +830,7 @@ public class S3Backend extends AbstractSharedBackend {
}
uri = createPresignedURI(identifier,
- HttpMethod.GET,
+ SdkHttpMethod.GET,
httpDownloadURIExpirySeconds,
requestParams);
if (uri != null && httpDownloadURICache != null) {
@@ -881,9 +882,13 @@ public class S3Backend extends AbstractSharedBackend {
}
else {
// multi-part
- InitiateMultipartUploadRequest req = new InitiateMultipartUploadRequest(bucket, blobId);
- InitiateMultipartUploadResult res = s3service.initiateMultipartUpload(s3ReqDecorator.decorate(req));
- uploadId = res.getUploadId();
+ final CreateMultipartUploadResponse res = s3Client.createMultipartUpload(
+ s3ReqDecorator.decorate(
+ CreateMultipartUploadRequest.builder()
+ .bucket(bucket)
+ .key(blobId)
+ .build()));
+ uploadId = res.uploadId();
long numParts;
if (maxNumberOfURIs > 1) {
@@ -912,7 +917,7 @@ public class S3Backend extends AbstractSharedBackend {
presignedURIRequestParams.put("partNumber", String.valueOf(blockId));
presignedURIRequestParams.put("uploadId", uploadId);
uploadPartURIs.add(createPresignedURI(newIdentifier,
- HttpMethod.PUT,
+ SdkHttpMethod.PUT,
httpUploadURIExpirySeconds,
presignedURIRequestParams));
}
@@ -976,35 +981,34 @@ public class S3Backend extends AbstractSharedBackend {
if (uploadToken.getUploadId().isPresent()) {
// An existing upload ID means this is a multi-part upload
String uploadId = uploadToken.getUploadId().get();
- ListPartsRequest listPartsRequest = new ListPartsRequest(bucket, key, uploadId);
- PartListing listing = s3service.listParts(listPartsRequest);
- List<PartETag> eTags = new ArrayList<>();
+ ListPartsResponse listing = s3Client.listParts(builder -> builder.bucket(bucket).key(key).uploadId(uploadId).build());
+ final List<CompletedPart> completedParts = new ArrayList<>();
long size = 0L;
- Date lastModified = null;
- for (PartSummary partSummary : listing.getParts()) {
- PartETag eTag = new PartETag(partSummary.getPartNumber(), partSummary.getETag());
- eTags.add(eTag);
- size += partSummary.getSize();
- if (null == lastModified || partSummary.getLastModified().after(lastModified)) {
- lastModified = partSummary.getLastModified();
+ Instant lastModified = null;
+ for (Part part : listing.parts()) {
+ completedParts.add(CompletedPart.builder().partNumber(part.partNumber()).eTag(part.eTag()).build());
+ size += part.size();
+ if (null == lastModified || part.lastModified().isAfter(lastModified)) {
+ lastModified = part.lastModified();
}
}
- CompleteMultipartUploadRequest completeReq = new CompleteMultipartUploadRequest(
- bucket,
- key,
- uploadId,
- eTags
- );
-
- s3service.completeMultipartUpload(completeReq);
+ s3Client.completeMultipartUpload(s3ReqDecorator.decorate(
+ CompleteMultipartUploadRequest.builder()
+ .bucket(bucket)
+ .key(key)
+ .uploadId(uploadId)
+ .multipartUpload(CompletedMultipartUpload.builder()
+ .parts(completedParts)
+ .build())
+ .build()));
record = new S3DataRecord(
this,
- s3service,
+ s3Client,
bucket,
blobId,
- lastModified.getTime(),
+ lastModified.toEpochMilli(),
size,
s3ReqDecorator
);
@@ -1022,134 +1026,102 @@ public class S3Backend extends AbstractSharedBackend {
return record;
}
- private URI createPresignedURI(DataIdentifier identifier,
- HttpMethod method,
- int expirySeconds) {
- return createPresignedURI(identifier, method, expirySeconds, new HashMap<>());
+ private URI createPresignedURI(DataIdentifier identifier, int expirySeconds) {
+ return createPresignedURI(identifier, SdkHttpMethod.PUT, expirySeconds, new HashMap<>());
}
private URI createPresignedURI(DataIdentifier identifier,
- HttpMethod method,
+ SdkHttpMethod method,
int expirySeconds,
Map<String, String> reqParams) {
final String key = getKeyName(identifier);
+ // SSE-C is supported with presigned URLs when encryption headers are included in both
+ // the signature calculation (handled by S3RequestDecorator) and the actual HTTP request
+
try {
final Date expiration = new Date();
- expiration.setTime(expiration.getTime() + expirySeconds * 1000);
+ URL presignedURL = null;
+ expiration.setTime(expiration.getTime() + expirySeconds * 1000L);
- GeneratePresignedUrlRequest request = s3ReqDecorator.decorate(new GeneratePresignedUrlRequest(bucket, key)
- .withMethod(method)
- .withExpiration(expiration));
+ // Add raw query parameters via override configuration
+ AwsRequestOverrideConfiguration.Builder overrideBuilder = AwsRequestOverrideConfiguration.builder();
for (Map.Entry<String, String> e : reqParams.entrySet()) {
- request.addRequestParameter(e.getKey(), e.getValue());
+ overrideBuilder.putRawQueryParameter(e.getKey(), e.getValue());
+ }
+
+ switch (method) {
+ case GET:
+ presignedURL = s3PresignService.presignGetObject(builder -> builder
+ .signatureDuration(Duration.between(
+ Instant.now(), // current time
+ expiration.toInstant() // expiration time from your Date object
+ ))
+ .getObjectRequest(s3ReqDecorator.decorate(
+ GetObjectRequest.builder()
+ .bucket(bucket)
+ .key(key)
+ .overrideConfiguration(overrideBuilder.build())
+ .build()))
+ .build()).url();
+ break;
+ case PUT:
+
+ presignedURL = s3PresignService.presignPutObject(builder -> builder
+ .signatureDuration(Duration.between(
+ Instant.now(), // current time
+ expiration.toInstant() // expiration time from your Date object
+ ))
+ .putObjectRequest(s3ReqDecorator.decorate(
+ PutObjectRequest.builder()
+ .bucket(bucket)
+ .key(key)
+ .overrideConfiguration(overrideBuilder.build())
+ .build()))
+ .build()).url();
+ break;
+ default:
+ throw new IllegalArgumentException("Unsupported method: " + method);
}
+
URI uri = null;
- URL presignedURL = null;
try {
- presignedURL = s3PresignService.generatePresignedUrl(request);
uri = presignedURL.toURI();
-
- LOG.debug("Presigned {} URI for key {}: {}", method.name(), key, uri.toString());
+ LOG.debug("Presigned {} URI for key {}: {}", method.name(), key, uri);
}
catch (URISyntaxException e) {
- LOG.error("AWS request to create presigned S3 URI failed - could not convert '{}' to URI",
- (null != presignedURL ? presignedURL.toString() : "")
- );
+ LOG.error("AWS request to create presigned S3 URI failed - could not convert '{}' to URI", presignedURL);
}
-
return uri;
- } catch (AmazonServiceException e) {
+ } catch (S3Exception e) {
LOG.error("AWS request to create presigned S3 {} URI failed. " +
- "Key: {}, Error: {}, HTTP Code: {}, AWS Error Code: {}, Error Type: {}, Request ID: {}",
- method.name(), key, e.getMessage(), e.getStatusCode(), e.getErrorCode(), e.getErrorType(), e.getRequestId());
+ "Key: {}, Error: {}, HTTP Code: {}, AWS Error Code: {}, Request ID: {}",
+ method.name(), key,
+ e.getMessage(),
+ e.statusCode(),
+ e.awsErrorDetails() != null ? e.awsErrorDetails().errorCode() : "N/A",
+ e.requestId());
return null;
}
}
- /**
- * Returns an iterator over the S3 objects
- * @param <T>
- */
- class RecordsIterator<T> extends AbstractIterator<T> {
- ObjectListing prevObjectListing;
- Queue<S3ObjectSummary> queue;
- long size;
- Function<S3ObjectSummary, T> transformer;
-
- public RecordsIterator (Function<S3ObjectSummary, T> transformer) {
- queue = new LinkedList<>();
- this.transformer = transformer;
- }
-
- @Override
- protected T computeNext() {
- if (queue.isEmpty()) {
- loadBatch();
- }
-
- while (queue.isEmpty() && prevObjectListing.getNextMarker() != null) {
- LOG.debug("Queue is empty, but there is more data in the S3 bucket");
- loadBatch();
- }
+ private <T> Iterator<T> getAllRecords(Function<S3Object, T> transformer) {
- if (!queue.isEmpty()) {
- return transformer.apply(queue.remove());
+ final ListObjectsV2Iterable prevObjectListing = s3Client.listObjectsV2Paginator(builder -> {
+ builder.bucket(bucket);
+ if (properties.containsKey(S3Constants.MAX_KEYS)) {
+ builder.maxKeys(Integer.valueOf(properties.getProperty(S3Constants.MAX_KEYS)));
}
+ builder.build();
+ });
- return endOfData();
- }
-
- private boolean loadBatch() {
- ClassLoader contextClassLoader = currentThread().getContextClassLoader();
- long start = System.currentTimeMillis();
- try {
- currentThread().setContextClassLoader(getClass().getClassLoader());
-
- // initialize the listing the first time
- if (prevObjectListing == null) {
- ListObjectsRequest listReq = new ListObjectsRequest();
- listReq.withBucketName(bucket);
- if (properties.containsKey(S3Constants.MAX_KEYS)) {
- listReq.setMaxKeys(Integer.valueOf(properties.getProperty(S3Constants.MAX_KEYS)));
- }
-
- prevObjectListing = s3service.listObjects(listReq);
- } else if (prevObjectListing.isTruncated()) { //already initialized more objects available
- prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
- } else { // no more available
- return false;
- }
-
- List<S3ObjectSummary> listing = ListUtils.toList(
- IterableUtils.filter(prevObjectListing.getObjectSummaries(),
- input -> !input.getKey().startsWith(META_KEY_PREFIX)));
-
- // After filtering no elements
- if (listing.isEmpty()) {
- return false;
- }
-
- size += listing.size();
- queue.addAll(listing);
-
- LOG.info("Loaded batch of size [{}] in [{}] ms.",
- listing.size(), (System.currentTimeMillis() - start));
-
- return true;
- } catch (AmazonServiceException e) {
- LOG.warn("Could not list objects", e);
- } finally {
- if (contextClassLoader != null) {
- currentThread().setContextClassLoader(contextClassLoader);
- }
- }
- return false;
- }
+ return IteratorUtils.transform(IteratorUtils.filter(
+ prevObjectListing.contents().iterator(),
+ s3Object -> !s3Object.key().startsWith(META_KEY_PREFIX)), transformer);
}
private static String addMetaKeyPrefix(String key) {
@@ -1167,22 +1139,22 @@ public class S3Backend extends AbstractSharedBackend {
* S3DataRecord which lazily retrieves the input stream of the record.
*/
static class S3DataRecord extends AbstractDataRecord {
- private AmazonS3Client s3service;
- private long length;
- private long lastModified;
- private String bucket;
- private boolean isMeta;
+ private final S3Client s3Client;
+ private final long length;
+ private final long lastModified;
+ private final String bucket;
+ private final boolean isMeta;
private final S3RequestDecorator s3ReqDecorator;
- public S3DataRecord(AbstractSharedBackend backend, AmazonS3Client s3service, String bucket,
- DataIdentifier key, long lastModified, long length, final S3RequestDecorator s3ReqDecorator) {
- this(backend, s3service, bucket, key, lastModified, length, false, s3ReqDecorator);
+ public S3DataRecord(AbstractSharedBackend backend, S3Client s3Client, String bucket,
+ DataIdentifier key, long lastModified, long length, final S3RequestDecorator s3ReqDecorator) {
+ this(backend, s3Client, bucket, key, lastModified, length, false, s3ReqDecorator);
}
- public S3DataRecord(AbstractSharedBackend backend, AmazonS3Client s3service, String bucket,
- DataIdentifier key, long lastModified, long length, boolean isMeta, final S3RequestDecorator s3ReqDecorator) {
+ public S3DataRecord(AbstractSharedBackend backend, S3Client s3Client, String bucket,
+ DataIdentifier key, long lastModified, long length, boolean isMeta, final S3RequestDecorator s3ReqDecorator) {
super(backend, key);
- this.s3service = s3service;
+ this.s3Client = s3Client;
this.lastModified = lastModified;
this.length = length;
this.bucket = bucket;
@@ -1200,7 +1172,11 @@ public class S3Backend extends AbstractSharedBackend {
String id = getKeyName(getIdentifier());
if (isMeta) {
id = addMetaKeyPrefix(getIdentifier().toString());
- return s3service.getObject(bucket, id).getObjectContent();
+ return s3Client.getObject(s3ReqDecorator.decorate(GetObjectRequest.builder()
+ .bucket(bucket)
+ .key(id)
+ .build()),
+ ResponseTransformer.toInputStream());
}
else {
// Don't worry about stream logging for metadata records
@@ -1209,7 +1185,12 @@ public class S3Backend extends AbstractSharedBackend {
LOG_STREAMS_DOWNLOAD.debug("Binary downloaded from S3 - identifier={}", id, new Exception());
}
}
- return s3service.getObject(s3ReqDecorator.decorate(new GetObjectRequest(bucket, id))).getObjectContent();
+ return s3Client.getObject(s3ReqDecorator.decorate(
+ GetObjectRequest.builder()
+ .bucket(bucket)
+ .key(id)
+ .build()),
+ ResponseTransformer.toInputStream());
}
@Override
@@ -1234,34 +1215,30 @@ public class S3Backend extends AbstractSharedBackend {
* aws.properties. As S3 doesn't have "move" command, this method simulate
* move as copy object object to new key and then delete older key.
*/
- private void renameKeys() throws DataStoreException {
+ private void renameKeys() {
long startTime = System.currentTimeMillis();
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
long count = 0;
try {
- Thread.currentThread().setContextClassLoader(
- getClass().getClassLoader());
- ObjectListing prevObjectListing = s3service.listObjects(bucket);
- List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
+ Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
+
+ List<ObjectIdentifier> deleteList = new ArrayList<>();
List<String> keysToDelete = new ArrayList<>();
+
+ ListObjectsV2Iterable prevObjectListing = s3Client.listObjectsV2Paginator(getObj -> getObj.bucket(bucket).build());
int nThreads = Integer.parseInt(properties.getProperty("maxConnections"));
ExecutorService executor = Executors.newFixedThreadPool(nThreads,
BasicThreadFactory.builder().namingPattern("s3-object-rename-worker-%d").build());
boolean taskAdded = false;
- while (true) {
- for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
- executor.execute(new KeyRenameThread(s3ObjSumm.getKey()));
- taskAdded = true;
- count++;
- // delete the object if it follows old key name format
- if( s3ObjSumm.getKey().startsWith(KEY_PREFIX)) {
- deleteList.add(new DeleteObjectsRequest.KeyVersion(
- s3ObjSumm.getKey()));
- keysToDelete.add(s3ObjSumm.getKey());
- }
+ for (S3Object s3Obj : prevObjectListing.contents()) {
+ executor.execute(new KeyRenameThread(s3Obj.key()));
+ taskAdded = true;
+ count++;
+ // delete the object if it follows old key name format
+ if( s3Obj.key().startsWith(KEY_PREFIX)) {
+ deleteList.add(ObjectIdentifier.builder().key(s3Obj.key()).build());
+ keysToDelete.add(s3Obj.key());
}
- if (!prevObjectListing.isTruncated()) break;
- prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
}
// This will make the executor accept no new threads
// and finish all existing threads in the queue
@@ -1276,22 +1253,22 @@ public class S3Backend extends AbstractSharedBackend {
} catch (InterruptedException ie) {
}
- LOG.info("Renamed [{}] keys, time taken [{}]sec", count,
- ((System.currentTimeMillis() - startTime) / 1000));
+ LOG.info("Renamed [{}] keys, time taken [{}]sec", count, ((System.currentTimeMillis() - startTime) / 1000));
// Delete older keys.
if (!deleteList.isEmpty()) {
RemoteStorageMode mode = (RemoteStorageMode) properties.getOrDefault(S3Constants.MODE, RemoteStorageMode.S3);
if (mode == RemoteStorageMode.S3) {
- DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(
- bucket);
int batchSize = 500, startIndex = 0, size = deleteList.size();
int endIndex = Math.min(batchSize, size);
while (endIndex <= size) {
- delObjsReq.setKeys(Collections.unmodifiableList(deleteList.subList(
- startIndex, endIndex)));
- DeleteObjectsResult dobjs = s3service.deleteObjects(delObjsReq);
+ DeleteObjectsResponse dobjs = s3Client.deleteObjects(DeleteObjectsRequest.builder()
+ .bucket(bucket)
+ .delete(Delete.builder()
+ .objects(Collections.unmodifiableList(
+ deleteList.subList(startIndex, startIndex))).build())
+ .build());
LOG.info("Records[{}] deleted in datastore from index [{}] to [{}]",
- dobjs.getDeletedObjects().size(), startIndex, (endIndex - 1));
+ dobjs.deleted().size(), startIndex, (endIndex - 1));
if (endIndex == size) {
break;
} else {
@@ -1301,7 +1278,7 @@ public class S3Backend extends AbstractSharedBackend {
}
} else {
long keysDeleteStartTime = System.currentTimeMillis();
- keysToDelete.forEach(key -> s3service.deleteObject(bucket, key));
+ keysToDelete.forEach(key -> s3Client.deleteObject(delObj -> delObj.bucket(bucket).key(key).build()));
LOG.debug("Delete operation for rename keys from gcp took: {} seconds", ((System.currentTimeMillis() - keysDeleteStartTime) / 1000));
}
}
@@ -1346,13 +1323,12 @@ public class S3Backend extends AbstractSharedBackend {
return key.substring(0, 4) + key.substring(5);
}
-
/**
* The class renames object key in S3 in a thread.
*/
private class KeyRenameThread implements Runnable {
- private String oldKey;
+ private final String oldKey;
public void run() {
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
@@ -1360,20 +1336,25 @@ public class S3Backend extends AbstractSharedBackend {
Thread.currentThread().setContextClassLoader(
getClass().getClassLoader());
String newS3Key = convertKey(oldKey);
- CopyObjectRequest copReq = new CopyObjectRequest(bucket,
- oldKey, bucket, newS3Key);
- Copy copy = tmx.copy(s3ReqDecorator.decorate(copReq));
+ Copy copy = tmx.copy(copyReq ->
+ copyReq.copyObjectRequest(s3ReqDecorator.decorate(
+ CopyObjectRequest.builder()
+ .sourceBucket(bucket)
+ .sourceKey(oldKey)
+ .destinationBucket(bucket)
+ .destinationKey(newS3Key)
+ .build()))
+ .build());
try {
- copy.waitForCopyResult();
+ copy.completionFuture().join();
LOG.debug("[{}] renamed to [{}] ", oldKey, newS3Key);
- } catch (InterruptedException ie) {
- LOG.error(" Exception in renaming [{}] to [{}] ", ie, oldKey, newS3Key);
+ } catch (Exception ie) {
+ LOG.error(" Exception {} in renaming [{}] to [{}] ", ie, oldKey, newS3Key);
}
} finally {
if (contextClassLoader != null) {
- Thread.currentThread().setContextClassLoader(
- contextClassLoader);
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
}
diff --git a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3BackendHelper.java b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3BackendHelper.java
new file mode 100644
index 0000000000..9114c9bc3a
--- /dev/null
+++ b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3BackendHelper.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.blob.cloud.s3;
+
+import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
+import software.amazon.awssdk.services.s3.model.MultipartUpload;
+import software.amazon.awssdk.services.s3.model.NoSuchBucketException;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.services.s3.paginators.ListMultipartUploadsIterable;
+import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * A Utils class to handle S3 related crud operations
+ */
+public class S3BackendHelper {
+
+ private S3BackendHelper() {
+ // no instances for you
+ }
+
+ private static final Logger LOG = LoggerFactory.getLogger(S3BackendHelper.class);
+
+ /**
+ * Delete S3 bucket. This method first deletes all objects from bucket and
+ * then delete empty bucket.
+ *
+ * @param bucketName the bucket name.
+ */
+ public static void deleteBucket(final String bucketName) throws IOException {
+ Properties prop = Utils.readConfig(Utils.DEFAULT_CONFIG_FILE);
+ S3Client s3Client = Utils.openService(prop, false);
+
+ deleteBucketObjects(bucketName, prop, s3Client);
+
+ // Delete the actual bucket
+ s3Client.deleteBucket(delReq -> delReq.bucket(bucketName).build());
+ }
+
+ public static void deleteBucketAndAbortMultipartUploads(final String bucket, final Date date, final Properties props) {
+ try (S3Client s3Client = Utils.openService(props, false)) {
+ if (!bucketExists(s3Client, bucket)) {
+ LOG.info("bucket [{}] doesn't exists", bucket);
+ return;
+ }
+
+ for (int i = 0; i < 4; i++) {
+ abortMultipartUpload(bucket, date, s3Client);
+ }
+ // Delete objects in the bucket with pagination
+ deleteBucketObjects(bucket, props, s3Client);
+ // Delete the bucket
+ s3Client.deleteBucket(delBucket -> delBucket.bucket(bucket).build());
+ LOG.info("bucket [ {} ] cleaned", bucket);
+ }
+ }
+
+ public static void deleteBucketObjects(final String bucket, final Properties props, final S3Client s3Client) {
+ deleteBucketObjects(bucket, props, s3Client, s3Client.listObjectsV2Paginator(
+ ListObjectsV2Request.builder().bucket(bucket).build()));
+ }
+
+ static void deleteBucketObjects(final String bucket, final Properties props, final S3Client s3Client,
+ final ListObjectsV2Iterable listResponses) {
+ for (ListObjectsV2Response listRes : listResponses) {
+ List<ObjectIdentifier> deleteList = new ArrayList<>();
+ for (S3Object s3Obj : listRes.contents()) {
+ deleteList.add(ObjectIdentifier.builder().key(s3Obj.key()).build());
+ }
+
+ if (!deleteList.isEmpty()) {
+ S3Backend.RemoteStorageMode mode = Utils.getMode(props);
+ if (mode == S3Backend.RemoteStorageMode.S3) {
+ s3Client.deleteObjects(delReq ->
+ delReq.bucket(bucket)
+ .delete(delObj ->
+ delObj.objects(deleteList)
+ .build())
+ .build());
+ } else {
+ // Delete objects one by one
+ deleteList.forEach(obj -> s3Client.deleteObject(delObj -> delObj
+ .bucket(bucket)
+ .key(obj.key())
+ .build()));
+
+ }
+ }
+ }
+ }
+
+ static void abortMultipartUpload(String bucket, Date date, S3Client s3Client) {
+ ListMultipartUploadsIterable multiUploadsResponse = s3Client.listMultipartUploadsPaginator(listReq ->
+ listReq.bucket(bucket).build());
+ for (MultipartUpload multipartUpload : multiUploadsResponse.uploads()) {
+ Instant initiated = multipartUpload.initiated();
+ if (initiated.isBefore(date.toInstant())) {
+ s3Client.abortMultipartUpload(abortReq -> abortReq
+ .bucket(bucket)
+ .key(multipartUpload.key())
+ .uploadId(multipartUpload.uploadId())
+ .build());
+ }
+ }
+ }
+
+ /**
+ * Checks if the specified Amazon S3 bucket exists and is accessible with the provided S3 client.
+ * <p>
+ * This method attempts to perform a {@code headBucket} request. If the bucket exists and is accessible,
+ * it returns {@code true}. If the bucket does not exist, it returns {@code false}.
+ * Other unexpected exceptions (such as permission errors or network issues) will propagate.
+ * </p>
+ *
+ * @param s3Client the {@link S3Client} to use for the request
+ * @param bucketName the name of the S3 bucket to check
+ * @return {@code true} if the bucket exists and is accessible; {@code false} if the bucket does not exist
+ * @throws NullPointerException if {@code s3Client} or {@code bucketName} is null
+ * @throws S3Exception if an AWS error other than {@link NoSuchBucketException} occurs
+ */
+ static boolean bucketExists(final S3Client s3Client, final String bucketName) {
+ try {
+ s3Client.headBucket(request -> request.bucket(bucketName));
+ return true;
+ }
+ catch (NoSuchBucketException exception) {
+ return false;
+ }
+ }
+
+ /**
+ * Checks if a specific object exists in the given S3 bucket.
+ * <p>
+ * Performs a {@code headObject} request using the provided {@link S3RequestDecorator}.
+ * Returns {@code true} if the object exists, {@code false} if it does not (404 or 403).
+ * Other {@link S3Exception}s are propagated.
+ * </p>
+ *
+ * @param s3Client the {@link S3Client} to use for the request
+ * @param bucket the S3 bucket name
+ * @param key the object key
+ * @param s3ReqDecorator decorator for the {@link HeadObjectRequest}
+ * @return {@code true} if the object exists, {@code false} if not found or forbidden
+ * @throws S3Exception for AWS errors other than 404 or 403
+ */
+ static boolean objectExists(final S3Client s3Client, final String bucket, final String key, S3RequestDecorator s3ReqDecorator) {
+ try {
+ s3Client.headObject(s3ReqDecorator.decorate(HeadObjectRequest.builder().bucket(bucket).key(key).build()));
+
+ LOG.debug("Object {} exists in bucket {}", key, bucket);
+ return true;
+ } catch (S3Exception e) {
+ if (e.statusCode() == 404 || e.statusCode() == 403) {
+ LOG.info("Object {} doesn't exists in bucket {}", key, bucket);
+ return false;
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ /**
+ * Waits for an S3 bucket, one we expect to exist, to report that it exists.
+ * A check for the bucket is called with a limited number of repeats with
+ * an increasing backoff.
+ *
+ * Usually you would call this after creating a bucket to block until the
+ * bucket is actually available before moving forward with other tasks that
+ * expect the bucket to be available.
+ *
+ * @param s3Client The AmazonS3 client connection to the storage service.
+ * @param bucketName The name of the bucket to check.
+ * @return True if the bucket exists; false otherwise.
+ */
+ public static boolean waitForBucket(@NotNull final S3Client s3Client, @NotNull final String bucketName,
+ final int maxAttempts, final long delayMillis) {
+ for (int attempt = 1; attempt <= maxAttempts; attempt++) {
+ try {
+ s3Client.headBucket(headReq -> headReq.bucket(bucketName).build());
+ return true;
+ } catch (NoSuchBucketException e) {
+ // Bucket doesn't exist yet; backoff and retry
+ try {
+ Thread.sleep(delayMillis * attempt);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt(); // Preserve interrupt status!
+ break;
+ }
+ } catch (Exception e) {
+ // Log or handle unexpected errors
+ LOG.error("Error during waitForBucket:", e);
+ break;
+ }
+ }
+ return false;
+ }
+}
diff --git a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java
index e77b4bd7f2..7e70b6f13d 100644
--- a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java
+++ b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java
@@ -52,6 +52,11 @@ public final class S3Constants {
*/
public static final String S3_MAX_ERR_RETRY = "maxErrorRetry";
+ /**
+ * Amazon S3 cross region bucket access
+ */
+ public static final String S3_CROSS_REGION_ACCESS = "crossRegionAccess";
+
/**
* Amazon aws S3 bucket.
*/
@@ -124,11 +129,6 @@ public final class S3Constants {
// generate base64 encoded 32 bytes string customer key for SSE_C
public static final String S3_SSE_C_KEY = "sseCustomerKey";
- /**
- * Constant to set S3 signature for SSE_KMS encryption.
- */
- public static final String S3_SIG_V4 = "AWSS3V4SignerType";
-
/**
* Constant to set proxy host.
*/
diff --git a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3RequestDecorator.java b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3RequestDecorator.java
index a99e8b67ef..9fc5f9bf47 100644
--- a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3RequestDecorator.java
+++ b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3RequestDecorator.java
@@ -17,22 +17,17 @@
package org.apache.jackrabbit.oak.blob.cloud.s3;
-import com.amazonaws.services.s3.model.CopyObjectRequest;
-import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
-import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.SSEAlgorithm;
-import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams;
-import com.amazonaws.services.s3.model.SSECustomerKey;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.ServerSideEncryption;
+import software.amazon.awssdk.utils.StringUtils;
import java.util.Properties;
-import static com.amazonaws.HttpMethod.GET;
-import static com.amazonaws.services.s3.model.SSEAlgorithm.AES256;
-import static com.amazonaws.util.StringUtils.hasValue;
import static java.util.Objects.requireNonNull;
import static org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants.S3_ENCRYPTION;
import static org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants.S3_ENCRYPTION_SSE_C;
@@ -41,14 +36,13 @@ import static org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants.S3_SSE_C_KEY;
import static org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants.S3_SSE_KMS_KEYID;
/**
- * This class to sets encrption mode in S3 request.
+ * This class to sets encryption mode related properties for S3 request.
*
*/
public class S3RequestDecorator {
DataEncryption dataEncryption = DataEncryption.NONE;
- SSEAwsKeyManagementParams sseParams;
-
- SSECustomerKey sseCustomerKey;
+ String sseKmsKey;
+ String sseCustomerKey;
public S3RequestDecorator(Properties props) {
final String encryptionType = props.getProperty(S3_ENCRYPTION);
@@ -58,16 +52,17 @@ public class S3RequestDecorator {
switch (encryptionType) {
case S3_ENCRYPTION_SSE_KMS: {
final String keyId = props.getProperty(S3_SSE_KMS_KEYID);
- sseParams = new SSEAwsKeyManagementParams();
- if (hasValue(keyId)) {
- sseParams.withAwsKmsKeyId(keyId);
+ if (StringUtils.isNotBlank(keyId)) {
+ sseKmsKey = keyId;
}
break;
}
case S3_ENCRYPTION_SSE_C: {
final String keyId = props.getProperty(S3_SSE_C_KEY);
- if (hasValue(keyId)) {
- sseCustomerKey = new SSECustomerKey(keyId);
+ if (StringUtils.isNotBlank(keyId)) {
+ sseCustomerKey = keyId;
+ } else {
+ throw new IllegalArgumentException("sseCustomerKey is empty for SSE_C encryption mode");
}
break;
}
@@ -78,11 +73,16 @@ public class S3RequestDecorator {
}
/**
- * Set encryption in {@link GetObjectMetadataRequest}
+ * Set encryption in {@link HeadObjectRequest}
*/
- public GetObjectMetadataRequest decorate(final GetObjectMetadataRequest request) {
+ public HeadObjectRequest decorate(final HeadObjectRequest request) {
if (requireNonNull(getDataEncryption()) == DataEncryption.SSE_C) {
- request.withSSECustomerKey(sseCustomerKey);
+ // Assume sseCustomerKey is already of type software.amazon.awssdk.services.s3.model.SseCustomerKey
+ return request.toBuilder()
+ .sseCustomerKey(sseCustomerKey)
+ .sseCustomerAlgorithm(ServerSideEncryption.AES256.toString())
+ .sseCustomerKeyMD5(Utils.calculateMD5(sseCustomerKey))
+ .build();
}
return request;
}
@@ -92,7 +92,25 @@ public class S3RequestDecorator {
*/
public GetObjectRequest decorate(final GetObjectRequest request) {
if (requireNonNull(getDataEncryption()) == DataEncryption.SSE_C) {
- request.withSSECustomerKey(sseCustomerKey);
+ return request.toBuilder()
+ .sseCustomerAlgorithm(ServerSideEncryption.AES256.toString())
+ .sseCustomerKey(sseCustomerKey)
+ .sseCustomerKeyMD5(Utils.calculateMD5(sseCustomerKey))
+ .build();
+ }
+ return request;
+ }
+
+ /**
+ * Set encryption in {@link CompleteMultipartUploadRequest}
+ */
+ public CompleteMultipartUploadRequest decorate(final CompleteMultipartUploadRequest request) {
+ if (requireNonNull(getDataEncryption()) == DataEncryption.SSE_C) {
+ return request.toBuilder()
+ .sseCustomerAlgorithm(ServerSideEncryption.AES256.toString())
+ .sseCustomerKey(sseCustomerKey)
+ .sseCustomerKeyMD5(Utils.calculateMD5(sseCustomerKey))
+ .build();
}
return request;
}
@@ -101,109 +119,93 @@ public class S3RequestDecorator {
* Set encryption in {@link PutObjectRequest}
*/
public PutObjectRequest decorate(PutObjectRequest request) {
- ObjectMetadata metadata = request.getMetadata() == null
- ? new ObjectMetadata()
- : request.getMetadata();
- switch (getDataEncryption()) {
+ PutObjectRequest.Builder builder = request.toBuilder();
+
+ DataEncryption encryption = getDataEncryption();
+
+ switch (encryption) {
case SSE_S3:
- metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
+ builder.serverSideEncryption(ServerSideEncryption.AES256);
break;
case SSE_KMS:
- metadata.setSSEAlgorithm(SSEAlgorithm.KMS.getAlgorithm());
- /*Set*/
- request.withSSEAwsKeyManagementParams(sseParams);
+ builder.serverSideEncryption(ServerSideEncryption.AWS_KMS);
+ if (sseKmsKey != null) {
+ builder.ssekmsKeyId(sseKmsKey);
+ }
break;
case SSE_C:
- request.withSSECustomerKey(sseCustomerKey);
+ builder.sseCustomerAlgorithm(ServerSideEncryption.AES256.toString())
+ .sseCustomerKey(sseCustomerKey)
+ .sseCustomerKeyMD5(Utils.calculateMD5(sseCustomerKey));
break;
case NONE:
break;
}
- request.setMetadata(metadata);
- return request;
+ return builder.build();
}
/**
* Set encryption in {@link CopyObjectRequest}
*/
public CopyObjectRequest decorate(CopyObjectRequest request) {
- ObjectMetadata metadata = request.getNewObjectMetadata() == null
- ? new ObjectMetadata()
- : request.getNewObjectMetadata();;
+
+ CopyObjectRequest.Builder builder = request.toBuilder();
+
switch (getDataEncryption()) {
case SSE_S3:
- metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
+ builder.serverSideEncryption(ServerSideEncryption.AES256);
break;
case SSE_KMS:
- metadata.setSSEAlgorithm(SSEAlgorithm.KMS.getAlgorithm());
- request.withSSEAwsKeyManagementParams(sseParams);
+ builder.serverSideEncryption(ServerSideEncryption.AWS_KMS);
+ if (sseKmsKey != null) {
+ // sseParams is typically a KMS Key ID string in SDK 2.x.
+ builder.ssekmsKeyId(sseKmsKey);
+ }
break;
case SSE_C:
- metadata.setSSEAlgorithm(AES256.getAlgorithm());
- request.withSourceSSECustomerKey(sseCustomerKey).withDestinationSSECustomerKey(sseCustomerKey);
+ // destination headers
+ builder.sseCustomerAlgorithm(ServerSideEncryption.AES256.toString())
+ .sseCustomerKey(sseCustomerKey)
+ .sseCustomerKeyMD5(Utils.calculateMD5(sseCustomerKey));
+
+ // source headers
+ builder.copySourceSSECustomerAlgorithm(ServerSideEncryption.AES256.toString())
+ .copySourceSSECustomerKey(sseCustomerKey)
+ .copySourceSSECustomerKeyMD5(Utils.calculateMD5(sseCustomerKey));
break;
case NONE:
break;
}
- request.setNewObjectMetadata(metadata);
- return request;
+ return builder.build();
}
- public InitiateMultipartUploadRequest decorate(InitiateMultipartUploadRequest request) {
- ObjectMetadata metadata = request.getObjectMetadata() == null
- ? new ObjectMetadata()
- : request.getObjectMetadata();;
+ public CreateMultipartUploadRequest decorate(CreateMultipartUploadRequest request) {
+
+ CreateMultipartUploadRequest.Builder builder = request.toBuilder();
+
switch (getDataEncryption()) {
case SSE_S3:
- metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
+ builder.serverSideEncryption(ServerSideEncryption.AES256);
break;
case SSE_KMS:
- metadata.setSSEAlgorithm(SSEAlgorithm.KMS.getAlgorithm());
- request.withSSEAwsKeyManagementParams(sseParams);
+ builder.serverSideEncryption(ServerSideEncryption.AWS_KMS);
+ if (sseKmsKey != null) {
+ builder.ssekmsKeyId(sseKmsKey);
+ }
break;
case SSE_C:
- request.withSSECustomerKey(sseCustomerKey);
+ builder.sseCustomerAlgorithm(ServerSideEncryption.AES256.toString())
+ .sseCustomerKey(sseCustomerKey)
+ .sseCustomerKeyMD5(Utils.calculateMD5(sseCustomerKey));
break;
case NONE:
break;
}
- request.setObjectMetadata(metadata);
- return request;
- }
-
- public GeneratePresignedUrlRequest decorate(GeneratePresignedUrlRequest request) {
- switch (getDataEncryption()) {
- case SSE_KMS:
- if (request.getMethod() == GET) break; // KMS is not valid for GET Requests
- String keyId = getSSEParams().getAwsKmsKeyId();
- request = request.withSSEAlgorithm(SSEAlgorithm.KMS.getAlgorithm());
- if (keyId != null) {
- request = request.withKmsCmkId(keyId);
- }
- break;
- case SSE_C:
- request = request.withSSECustomerKey(sseCustomerKey);
- break;
- default:
- break;
- }
- return request;
- }
-
- private SSEAwsKeyManagementParams getSSEParams() {
- return this.sseParams;
+ return builder.build();
}
private DataEncryption getDataEncryption() {
return this.dataEncryption;
}
- /**
- * Enum to indicate S3 encryption mode
- *
- */
- private enum DataEncryption {
- SSE_S3, SSE_KMS, SSE_C, NONE;
- }
-
}
diff --git a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java
index d32b5159f3..d193689684 100644
--- a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java
+++ b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java
@@ -20,25 +20,46 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.net.URI;
+import java.time.Duration;
import java.util.HashMap;
import java.util.Map;
+import java.util.Objects;
import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
-import org.apache.commons.lang3.StringUtils;
+import org.apache.jackrabbit.oak.blob.cloud.s3.S3Backend.RemoteStorageMode;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.Protocol;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.regions.Regions;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.auth.signer.AwsS3V4Signer;
+import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
+import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption;
+import software.amazon.awssdk.core.exception.SdkClientException;
+import software.amazon.awssdk.http.SdkHttpClient;
+import software.amazon.awssdk.http.apache.ApacheHttpClient;
+import software.amazon.awssdk.http.apache.ProxyConfiguration;
+import software.amazon.awssdk.http.async.SdkAsyncHttpClient;
+import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.S3AsyncClientBuilder;
+import software.amazon.awssdk.services.s3.S3BaseClientBuilder;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
+import software.amazon.awssdk.services.s3.presigner.S3Presigner;
+import software.amazon.awssdk.utils.BinaryUtils;
+import software.amazon.awssdk.utils.Md5Utils;
+import software.amazon.awssdk.utils.StringUtils;
+
+import static org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants.S3_ENCRYPTION;
/**
* Amazon S3 utilities.
@@ -51,6 +72,8 @@ public final class Utils {
private static final String DELETE_CONFIG_SUFFIX = ";burn";
+ private static final String HTTPS = "https";
+
/**
* The default value AWS bucket region.
*/
@@ -67,125 +90,76 @@ public final class Utils {
public static final String AWSDOTCOM = "amazonaws.com";
public static final String S3 = "s3";
+ public static final String S3_ACCELERATION = "s3-accelerate";
public static final String DOT = ".";
public static final String DASH = "-";
/**
- * private constructor so that class cannot initialized from outside.
+ * private constructor so that class cannot initialize from outside.
*/
private Utils() {
}
/**
- * Create AmazonS3Client from properties.
- *
- * @param prop properties to configure @link {@link AmazonS3Client}
- * @return {@link AmazonS3Client}
+ * Create S3Client from properties.
+ *
+ * @param prop properties to configure @link {@link S3Client}
+ * @param accReq boolean indicating whether to accelerate requests
+ * @return {@link S3Client}
*/
- public static AmazonS3Client openService(final Properties prop) {
- String accessKey = prop.getProperty(S3Constants.ACCESS_KEY);
- String secretKey = prop.getProperty(S3Constants.SECRET_KEY);
- AmazonS3Client s3service = null;
- if (StringUtils.isEmpty(accessKey)
- || StringUtils.isEmpty(secretKey)) {
- LOG.info("Configuring Amazon Client from environment");
- s3service = new AmazonS3Client(getClientConfiguration(prop));
- } else {
- LOG.info("Configuring Amazon Client from property file.");
- AWSCredentials credentials = new BasicAWSCredentials(accessKey,
- secretKey);
- s3service = new AmazonS3Client(credentials,
- getClientConfiguration(prop));
- }
- String region = prop.getProperty(S3Constants.S3_REGION);
- String endpoint = null;
- String propEndPoint = prop.getProperty(S3Constants.S3_END_POINT);
- if ((propEndPoint != null) & !"".equals(propEndPoint)) {
- endpoint = propEndPoint;
- } else {
- if (StringUtils.isEmpty(region)) {
- com.amazonaws.regions.Region s3Region = Regions.getCurrentRegion();
- if (s3Region != null) {
- region = s3Region.getName();
- } else {
- throw new AmazonClientException(
- "parameter ["
- + S3Constants.S3_REGION
- + "] not configured and cannot be derived from environment");
- }
- }
- if (DEFAULT_AWS_BUCKET_REGION.equals(region)) {
- endpoint = S3 + DOT + AWSDOTCOM;
- } else {
- endpoint = S3 + DOT + region + DOT + AWSDOTCOM;
- }
- }
- /*
- * setting endpoint to remove latency of redirection. If endpoint is
- * not set, invocation first goes us standard region, which
- * redirects it to correct location.
- */
- s3service.setEndpoint(endpoint);
- LOG.info("S3 service endpoint [{}] ", endpoint);
- return s3service;
+ public static S3Client openService(final Properties prop, boolean accReq) {
+
+ S3ClientBuilder builder = S3Client.builder();
+
+ configureBuilder(builder, prop, accReq);
+ // sync http client
+ builder.httpClient(getSdkHttpClient(prop));
+
+ return builder.build();
}
/**
- * Waits for an S3 bucket, one we expect to exist, to report that it exists.
- * A check for the bucket is called with a limited number of repeats with
- * an increasing backoff.
- *
- * Usually you would call this after creating a bucket to block until the
- * bucket is actually available before moving forward with other tasks that
- * expect the bucket to be available.
+ * Create S3Client from properties.
*
- * @param s3Client The AmazonS3 client connection to the storage service.
- * @param bucketName The name of the bucket to check.
- * @return True if the bucket exists; false otherwise.
+ * @param prop properties to configure @link {@link S3Client}
+ * @return {@link S3Client}
*/
- public static boolean waitForBucket(@NotNull final AmazonS3 s3Client, @NotNull final String bucketName) {
- int tries = 0;
- boolean bucketExists = false;
- while (20 > tries++) {
- bucketExists = s3Client.doesBucketExistV2(bucketName);
- if (bucketExists) break;
- try {
- Thread.sleep(100 * tries);
- }
- catch (InterruptedException e) { }
- }
- return bucketExists;
+ public static S3AsyncClient openAsyncService(final Properties prop) {
+ S3AsyncClientBuilder builder = S3AsyncClient.builder();
+
+ configureBuilder(builder, prop, false);
+ // async http client
+ builder.httpClient(getSdkAsyncHttpClient(prop));
+ builder.multipartEnabled(true);
+
+ return builder.build();
}
/**
- * Delete S3 bucket. This method first deletes all objects from bucket and
- * then delete empty bucket.
- *
- * @param bucketName the bucket name.
+ * Creates an {@link S3Presigner} instance using the provided {@link S3Client} and S3 configuration properties.
+ * <p>
+ * The presigner is used to generate pre-signed URLs for S3 operations, allowing secure, time-limited access
+ * to S3 resources without exposing credentials.
+ * </p>
+ *
+ * @param s3Client the {@link S3Client} to use for presigning requests
+ * @param props the properties containing S3 configuration (credentials, region, etc.)
+ * @return a configured {@link S3Presigner} instance
*/
- public static void deleteBucket(final String bucketName) throws IOException {
- Properties prop = readConfig(DEFAULT_CONFIG_FILE);
- AmazonS3 s3service = openService(prop);
- ObjectListing prevObjectListing = s3service.listObjects(bucketName);
- while (true) {
- for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
- s3service.deleteObject(bucketName, s3ObjSumm.getKey());
- }
- if (!prevObjectListing.isTruncated()) {
- break;
- }
- prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
- }
- s3service.deleteBucket(bucketName);
+ public static S3Presigner createPresigner(final S3Client s3Client, final Properties props) {
+ return S3Presigner.builder().s3Client(s3Client).
+ credentialsProvider(Utils.getAwsCredentials(props))
+ .region(Region.of(Utils.getRegion(props)))
+ .build();
}
/**
* Read a configuration properties file. If the file name ends with ";burn",
* the file is deleted after reading.
- *
+ *
* @param fileName the properties file name
* @return the properties
* @throws java.io.IOException if the file doesn't exist
@@ -195,7 +169,7 @@ public final class Utils {
if (fileName.endsWith(DELETE_CONFIG_SUFFIX)) {
delete = true;
fileName = fileName.substring(0, fileName.length()
- - DELETE_CONFIG_SUFFIX.length());
+ - DELETE_CONFIG_SUFFIX.length());
}
if (!new File(fileName).exists()) {
throw new IOException("Config file not found: " + fileName);
@@ -216,54 +190,316 @@ public final class Utils {
return prop;
}
- private static void deleteIfPossible(final File file) {
- boolean deleted = file.delete();
- if (!deleted) {
- LOG.warn("Could not delete " + file.getAbsolutePath());
+ public static Map<String, Object> asMap(Properties props) {
+ Map<String, Object> map = new HashMap<>();
+ for (Object key : props.keySet()) {
+ map.put((String)key, props.get(key));
}
+ return map;
}
- private static ClientConfiguration getClientConfiguration(Properties prop) {
- int connectionTimeOut = Integer.parseInt(prop.getProperty(S3Constants.S3_CONN_TIMEOUT));
- int socketTimeOut = Integer.parseInt(prop.getProperty(S3Constants.S3_SOCK_TIMEOUT));
- int maxConnections = Integer.parseInt(prop.getProperty(S3Constants.S3_MAX_CONNS));
+ /**
+ * Determines the data encryption type to use for S3 operations based on the provided properties.
+ * <p>
+ * If the property {@code S3_ENCRYPTION} is set, returns the corresponding {@link DataEncryption} enum value.
+ * If the property is not set or is invalid, returns {@link DataEncryption#NONE}.
+ *
+ * @param props the properties containing S3 configuration
+ * @return the {@link DataEncryption} type to use
+ */
+ public static DataEncryption getDataEncryption(final Properties props) {
+ final String encryptionType = props.getProperty(S3_ENCRYPTION);
+ DataEncryption encryption = DataEncryption.NONE;
+ if (encryptionType != null) {
+ encryption = DataEncryption.valueOf(encryptionType);
+ }
+ return encryption;
+ }
+
+ /**
+ * Sets the remote storage mode in the provided properties based on the S3 endpoint.
+ * <p>
+ * If the endpoint contains "googleapis", the mode is set to GCP. Otherwise, it is set to S3.
+ * If the mode was previously set to S3 but the endpoint is for GCP, a warning is logged and the mode is overridden.
+ *
+ * @param properties the properties to update with the remote storage mode
+ */
+ public static void setRemoteStorageMode(final Properties properties) {
+ String s3EndPoint = properties.getProperty(S3Constants.S3_END_POINT, "");
+ if (s3EndPoint.contains("googleapis")) {
+ if (properties.get(S3Constants.MODE) == RemoteStorageMode.S3) {
+ LOG.warn("Mismatch between remote storage mode and s3EndPoint, overriding mode to GCP");
+ }
+ properties.put(S3Constants.MODE, RemoteStorageMode.GCP);
+ return;
+ }
+ // default mode is S3
+ properties.put(S3Constants.MODE, RemoteStorageMode.S3);
+ }
+
+ /**
+ * Determines the AWS region to use based on the provided properties.
+ * <p>
+ * The method attempts to extract the region in the following order:
+ * <ol>
+ * <li>If the S3 endpoint property is set, tries to parse the region from the endpoint URL.</li>
+ * <li>If not found, uses the S3 region property.</li>
+ * <li>If the region property is empty, falls back to the default region from the environment.</li>
+ * <li>If the region is "us-standard", returns "us-east-1".</li>
+ * </ol>
+ *
+ * @param prop the properties containing S3 configuration
+ * @return the AWS region as a string
+ */
+ static String getRegion(final Properties prop) {
+
+ String region = null;
+
+ if (Objects.nonNull(prop.getProperty(S3Constants.S3_END_POINT))) {
+ region = getRegionFromEndpoint(prop.getProperty(S3Constants.S3_END_POINT));
+ }
+
+ if (Objects.nonNull(region)) {
+ return region;
+ }
+
+ region = prop.getProperty(S3Constants.S3_REGION);
+
+ if (region.isEmpty()) {
+ region = Utils.getDefaultRegion();
+ }
+
+ if (Objects.equals(DEFAULT_AWS_BUCKET_REGION, region)) {
+ return US_EAST_1_AWS_BUCKET_REGION;
+ }
+ return region;
+ }
+
+ /**
+ * Extracts the AWS region from a given S3 endpoint URL.
+ * <p>
+ * Supports both path-style and virtual-hosted-style S3 endpoints, e.g.:
+ * <ul>
+ * <li>https://s3.eu-west-1.amazonaws.com</li>
+ * <li>https://bucket.s3.eu-west-1.amazonaws.com</li>
+ * <li>https://s3.amazonaws.com (returns us-east-1)</li>
+ * </ul>
+ * If the region cannot be determined, returns null.
+ *
+ * @param endpoint the S3 endpoint URL as a string
+ * @return the AWS region string, or null if not found
+ */
+ static String getRegionFromEndpoint(final String endpoint) {
+ try {
+ URI uri = URI.create(endpoint);
+ String host = uri.getHost();
+
+ // Pattern for standard S3 endpoints: s3.region.amazonaws.com or bucket.s3.region.amazonaws.com
+ Pattern pattern = Pattern.compile("s3[.-]([a-z0-9-]+)\\.amazonaws\\.com");
+ Matcher matcher = pattern.matcher(host);
+
+ if (matcher.find()) {
+ return matcher.group(1);
+ }
+
+ // Handle us-east-1 special case (s3.amazonaws.com)
+ // Handle virtual-hosted-style URLs: bucket.s3.amazonaws.com
+ if (host.equals("s3.amazonaws.com") || host.endsWith(".s3.amazonaws.com")) {
+ return Region.US_EAST_1.id();
+ }
+
+ LOG.warn("Cannot parse region from endpoint: {}", endpoint);
+
+ return null;
+
+ } catch (Exception e) {
+ LOG.error("Invalid endpoint format: {}", endpoint);
+ return null;
+ }
+ }
+
+ /**
+ * Constructs the S3 endpoint URI based on the provided properties, acceleration flag, and region.
+ * <p>
+ * The method determines the endpoint as follows:
+ * <ol>
+ * <li>If the S3 endpoint property is set, uses it directly.</li>
+ * <li>Otherwise, constructs the endpoint using the region and acceleration flag.</li>
+ * <li>The protocol is taken from the properties or defaults to "https".</li>
+ * </ol>
+ *
+ * @param prop the properties containing S3 configuration
+ * @param accReq whether to use S3 acceleration endpoint
+ * @param region the AWS region
+ * @return the constructed S3 endpoint URI
+ */
+ @NotNull
+ static URI getEndPointUri(Properties prop, boolean accReq, String region) {
+ String endPoint;
+ String propEndPoint = prop.getProperty(S3Constants.S3_END_POINT);
+ if (propEndPoint != null && !propEndPoint.isEmpty()) {
+ endPoint = propEndPoint;
+ } else {
+ // in aws sdk 2.x, there is no us-standard region. us-east-1 should be used in its place.
+ endPoint = (accReq ? S3_ACCELERATION : S3) + DOT + region + DOT + AWSDOTCOM;
+
+ }
+
+ /*
+ * setting endpoint to remove latency of redirection. If endpoint is
+ * not set, invocation first goes us standard region, which
+ * redirects it to correct location.
+ */
+
+ // Check if endpoint already contains protocol
+ if (endPoint.startsWith("http://") || endPoint.startsWith("https://")) {
+ LOG.info("S3 service endpoint [{}] ", endPoint);
+ return URI.create(endPoint);
+ }
+
+ String protocol = prop.getProperty(S3Constants.S3_CONN_PROTOCOL);
+ if (protocol == null || protocol.isEmpty()) {
+ protocol = HTTPS; // default protocol
+ }
+
+ LOG.info("S3 service endpoint [{}] ", endPoint);
+ return URI.create(protocol + "://" + endPoint);
+ }
+
+ // Add this helper method for SSE-C:
+ static String calculateMD5(final String customerKey) {
+ return Md5Utils.md5AsBase64(BinaryUtils.fromBase64(customerKey));
+ }
+
+ private static ClientOverrideConfiguration getClientConfiguration(Properties prop) {
int maxErrorRetry = Integer.parseInt(prop.getProperty(S3Constants.S3_MAX_ERR_RETRY));
+ int connectionTimeOut = Integer.parseInt(prop.getProperty(S3Constants.S3_CONN_TIMEOUT));
String encryptionType = prop.getProperty(S3Constants.S3_ENCRYPTION);
- String protocol = prop.getProperty(S3Constants.S3_CONN_PROTOCOL);
- String proxyHost = prop.getProperty(S3Constants.PROXY_HOST);
- String proxyPort = prop.getProperty(S3Constants.PROXY_PORT);
+ ClientOverrideConfiguration.Builder builder = ClientOverrideConfiguration.builder();
- ClientConfiguration cc = new ClientConfiguration();
+ builder.retryStrategy(b -> b.maxAttempts(maxErrorRetry));
+ builder.apiCallTimeout(Duration.ofMillis(connectionTimeOut));
- if (protocol != null && protocol.equalsIgnoreCase("http")) {
- cc.setProtocol(Protocol.HTTP);
+ if (S3Constants.S3_ENCRYPTION_SSE_KMS.equals(encryptionType)) {
+ builder.putAdvancedOption(SdkAdvancedClientOption.SIGNER, AwsS3V4Signer.create());
}
+ return builder.build();
+ }
+
+ private static SdkHttpClient getSdkHttpClient(Properties prop) {
+ HttpClientConfig config = new HttpClientConfig(prop);
+ final ApacheHttpClient.Builder builder = ApacheHttpClient.builder();
- if (proxyHost != null && !proxyHost.isEmpty()) {
- cc.setProxyHost(proxyHost);
+ builder.connectionTimeout(Duration.ofMillis(config.connectionTimeout))
+ .socketTimeout(Duration.ofMillis(config.socketTimeout))
+ .maxConnections(config.maxConnections);
+
+ if (config.proxyHost != null && !config.proxyHost.isEmpty() && config.proxyPort != null && !config.proxyPort.isEmpty()) {
+ String protocol = "http".equalsIgnoreCase(config.protocol) ? "http" : config.protocol;
+ builder.proxyConfiguration(
+ ProxyConfiguration.builder()
+ .scheme(protocol)
+ .endpoint(URI.create(protocol + "://" + config.proxyHost + ":" + config.proxyPort))
+ .build()
+ );
}
- if (proxyPort != null && !proxyPort.isEmpty()) {
- cc.setProxyPort(Integer.parseInt(proxyPort));
+ return builder.build();
+ }
+
+ private static SdkAsyncHttpClient getSdkAsyncHttpClient(Properties prop) {
+ HttpClientConfig config = new HttpClientConfig(prop);
+ final NettyNioAsyncHttpClient.Builder builder = NettyNioAsyncHttpClient.builder();
+
+ builder.connectionTimeout(Duration.ofMillis(config.connectionTimeout))
+ .readTimeout(Duration.ofMillis(config.socketTimeout))
+ .writeTimeout(Duration.ofMillis(config.socketTimeout))
+ .maxConcurrency(config.maxConnections);
+
+ if (config.proxyHost != null && !config.proxyHost.isEmpty() && config.proxyPort != null && !config.proxyPort.isEmpty()) {
+ String protocol = HTTPS.equalsIgnoreCase(config.protocol) ? HTTPS : config.protocol;
+ builder.proxyConfiguration(
+ software.amazon.awssdk.http.nio.netty.ProxyConfiguration.builder()
+ .host(config.proxyHost)
+ .port(Integer.parseInt(config.proxyPort))
+ .scheme(protocol)
+ .build()
+ );
}
+ return builder.build();
+ }
- cc.setConnectionTimeout(connectionTimeOut);
- cc.setSocketTimeout(socketTimeOut);
- cc.setMaxConnections(maxConnections);
- cc.setMaxErrorRetry(maxErrorRetry);
- if (encryptionType != null
- && encryptionType.equals(S3Constants.S3_ENCRYPTION_SSE_KMS)) {
- cc.withSignerOverride("AWSS3V4SignerType");
+ private static void deleteIfPossible(final File file) {
+ boolean deleted = file.delete();
+ if (!deleted) {
+ LOG.warn("Could not delete {}", file.getAbsolutePath());
}
- return cc;
}
- public static Map<String, Object> asMap(Properties props) {
- Map<String, Object> map = new HashMap<>();
- for (Object key : props.keySet()) {
- map.put((String)key, props.get(key));
+ @NotNull
+ static RemoteStorageMode getMode(@NotNull Properties props) {
+ return props.getProperty(S3Constants.S3_END_POINT, "").contains("googleapis") ?
+ RemoteStorageMode.GCP : RemoteStorageMode.S3;
+ }
+
+ private static String getDefaultRegion() {
+ String region;
+ final DefaultAwsRegionProviderChain regionProvider = new DefaultAwsRegionProviderChain();
+ try {
+ Region currentRegion = regionProvider.getRegion();
+ region = currentRegion.id();
+ } catch (Exception e) {
+ throw SdkClientException.builder().message(
+ "parameter ["
+ + S3Constants.S3_REGION
+ + "] not configured and cannot be derived from environment").build();
+ }
+ return region;
+ }
+
+ @NotNull
+ private static AwsCredentialsProvider getAwsCredentials(Properties prop) {
+ String accessKey = prop.getProperty(S3Constants.ACCESS_KEY);
+ String secretKey = prop.getProperty(S3Constants.SECRET_KEY);
+ if (StringUtils.isEmpty(accessKey) || StringUtils.isEmpty(secretKey)) {
+ LOG.info("Configuring Amazon Client from environment");
+ return DefaultCredentialsProvider.builder().build();
+ } else {
+ LOG.info("Configuring Amazon Client from property file.");
+ final AwsBasicCredentials credentials = AwsBasicCredentials.create(accessKey, secretKey);
+ return StaticCredentialsProvider.create(credentials);
+ }
+ }
+
+ private static void configureBuilder(final S3BaseClientBuilder builder, final Properties prop, final boolean accReq) {
+ builder.credentialsProvider(getAwsCredentials(prop));
+ builder.overrideConfiguration(getClientConfiguration(prop));
+
+ // region is mandatory even with endpointOverride
+ String region = getRegion(prop);
+ builder.region(Region.of(region));
+
+ builder.endpointOverride(getEndPointUri(prop, accReq, region));
+ builder.crossRegionAccessEnabled(Boolean.parseBoolean(prop.getProperty(S3Constants.S3_CROSS_REGION_ACCESS)));
+ }
+
+ // Helper class to hold common Http config
+ private static class HttpClientConfig {
+ final int connectionTimeout;
+ final int socketTimeout;
+ final int maxConnections;
+ final String protocol;
+ final String proxyHost;
+ final String proxyPort;
+
+ HttpClientConfig(Properties prop) {
+ this.connectionTimeout = Integer.parseInt(prop.getProperty(S3Constants.S3_CONN_TIMEOUT));
+ this.socketTimeout = Integer.parseInt(prop.getProperty(S3Constants.S3_SOCK_TIMEOUT));
+ this.maxConnections = Integer.parseInt(prop.getProperty(S3Constants.S3_MAX_CONNS));
+ this.protocol = prop.getProperty(S3Constants.S3_CONN_PROTOCOL);
+ this.proxyHost = prop.getProperty(S3Constants.PROXY_HOST);
+ this.proxyPort = prop.getProperty(S3Constants.PROXY_PORT);
}
- return map;
}
}
diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3BackendHelperTest.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3BackendHelperTest.java
new file mode 100644
index 0000000000..ad78d90164
--- /dev/null
+++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3BackendHelperTest.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.blob.cloud.s3;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.HeadBucketResponse;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.NoSuchBucketException;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+
+import java.util.function.Consumer;
+
+public class S3BackendHelperTest {
+
+ @Test
+ public void testBucketExistsTrue() {
+ S3Client mockClient = Mockito.mock(S3Client.class);
+ // No exception means bucket exists
+ Assert.assertTrue(S3BackendHelper.bucketExists(mockClient, Mockito.anyString()));
+ Mockito.verify(mockClient).headBucket(Mockito.any(Consumer.class));
+ }
+
+ @Test
+ public void testBucketExistsFalse() {
+ S3Client mockClient = Mockito.mock(S3Client.class);
+ Mockito.doThrow(NoSuchBucketException.builder().build())
+ .when(mockClient).headBucket(Mockito.any(Consumer.class));
+ Assert.assertFalse(S3BackendHelper.bucketExists(mockClient, Mockito.anyString()));
+ Mockito.verify(mockClient).headBucket(Mockito.any(Consumer.class));
+ }
+
+ @Test
+ public void testObjectExistsTrue() {
+ S3Client mockClient = Mockito.mock(S3Client.class);
+ S3RequestDecorator decorator = Mockito.mock(S3RequestDecorator.class);
+
+ // identity decorator
+ Mockito.when(decorator.decorate(Mockito.any(HeadObjectRequest.class))).thenReturn(Mockito.mock(HeadObjectRequest.class));
+ Assert.assertTrue(S3BackendHelper.objectExists(mockClient, "bucket", "key", decorator));
+ Mockito.verify(mockClient).headObject(Mockito.any(HeadObjectRequest.class));
+ }
+
+ @Test
+ public void testObjectExistsNotFound() {
+ S3Client mockClient = Mockito.mock(S3Client.class);
+ S3RequestDecorator decorator = Mockito.mock(S3RequestDecorator.class);
+
+ // identity decorator
+ Mockito.when(decorator.decorate(Mockito.any(HeadObjectRequest.class))).thenReturn(Mockito.mock(HeadObjectRequest.class));
+ AwsServiceException notFound = S3Exception.builder().statusCode(404).build();
+ Mockito.doThrow(notFound).when(mockClient).headObject(Mockito.any(HeadObjectRequest.class));
+ Assert.assertFalse(S3BackendHelper.objectExists(mockClient, "bucket", "key", decorator));
+ }
+
+ @Test
+ public void testObjectExistsForbidden() {
+ S3Client mockClient = Mockito.mock(S3Client.class);
+ S3RequestDecorator decorator = Mockito.mock(S3RequestDecorator.class);
+
+ // identity decorator
+ Mockito.when(decorator.decorate(Mockito.any(HeadObjectRequest.class))).thenReturn(Mockito.mock(HeadObjectRequest.class));
+ AwsServiceException forbidden = S3Exception.builder().statusCode(403).build();
+ Mockito.doThrow(forbidden).when(mockClient).headObject(Mockito.any(HeadObjectRequest.class));
+ Assert.assertFalse(S3BackendHelper.objectExists(mockClient, "bucket", "key", decorator));
+ }
+
+ @Test(expected = S3Exception.class)
+ public void testObjectExistsOtherException() {
+ S3Client mockClient = Mockito.mock(S3Client.class);
+ S3RequestDecorator decorator = Mockito.mock(S3RequestDecorator.class);
+
+ // identity decorator
+ Mockito.when(decorator.decorate(Mockito.any(HeadObjectRequest.class))).thenReturn(Mockito.mock(HeadObjectRequest.class));
+ AwsServiceException other = S3Exception.builder().statusCode(500).build();
+ Mockito.doThrow(other).when(mockClient).headObject(Mockito.any(HeadObjectRequest.class));
+ S3BackendHelper.objectExists(mockClient, "bucket", "key", decorator);
+ }
+
+ @Test
+ public void testWaitForBucketExistsImmediately() {
+ S3Client mockClient = Mockito.mock(S3Client.class);
+ boolean result = S3BackendHelper.waitForBucket(mockClient, "bucket", 5, 1L);
+ Assert.assertTrue(result);
+ Mockito.verify(mockClient, Mockito.times(1)).headBucket(Mockito.any(Consumer.class));
+ }
+
+ @Test
+ public void testWaitForBucketExistsAfterRetries() {
+ S3Client mockClient = Mockito.mock(S3Client.class);
+ Mockito.doThrow(NoSuchBucketException.builder().build())
+ .doThrow(NoSuchBucketException.builder().build())
+ .doReturn(HeadBucketResponse.builder().build())
+ .when(mockClient).headBucket(Mockito.any(Consumer.class));
+ boolean result = S3BackendHelper.waitForBucket(mockClient, "bucket", 5, 1L);
+ Assert.assertTrue(result);
+ Mockito.verify(mockClient, Mockito.times(3)).headBucket(Mockito.any(Consumer.class));
+ }
+
+ @Test
+ public void testWaitForBucketNeverExists() {
+ S3Client mockClient = Mockito.mock(S3Client.class);
+ Mockito.doThrow(NoSuchBucketException.builder().build())
+ .when(mockClient).headBucket(Mockito.any(Consumer.class));
+ boolean result = S3BackendHelper.waitForBucket(mockClient, "bucket", 5, 1L);
+ Assert.assertFalse(result);
+ Mockito.verify(mockClient, Mockito.times(5)).headBucket(Mockito.any(Consumer.class));
+ }
+
+ @Test
+ public void testWaitForBucketOtherException() {
+ S3Client mockClient = Mockito.mock(S3Client.class);
+ Mockito.doThrow(new RuntimeException("fail"))
+ .when(mockClient).headBucket(Mockito.any(Consumer.class));
+ boolean result = S3BackendHelper.waitForBucket(mockClient, "bucket", 5, 1L);
+ Assert.assertFalse(result);
+ Mockito.verify(mockClient, Mockito.times(1)).headBucket(Mockito.any(Consumer.class));
+ }
+
+}
\ No newline at end of file
diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderTest.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderTest.java
index 551b279f3b..eba404c3a0 100644
--- a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderTest.java
+++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderTest.java
@@ -31,6 +31,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.util.Map;
+import java.util.Objects;
import java.util.Properties;
import javax.net.ssl.HttpsURLConnection;
@@ -100,6 +101,12 @@ public class S3DataRecordAccessProviderTest extends AbstractDataRecordAccessProv
((S3DataStore)ds).deleteRecord(identifier);
}
+ @Override
+ protected boolean isSSECustomerKeyEncryption() {
+ String encryption = getS3Config().getProperty(S3Constants.S3_ENCRYPTION, DataEncryption.NONE.toString());
+ return Objects.equals(DataEncryption.valueOf(encryption), DataEncryption.SSE_C);
+ }
+
@Override
protected long getProviderMinPartSize() {
return Math.max(0L, S3Backend.MIN_MULTIPART_UPLOAD_PART_SIZE);
diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStoreUtils.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStoreUtils.java
index 86694c1cfc..1d6ce896ec 100644
--- a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStoreUtils.java
+++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStoreUtils.java
@@ -23,34 +23,22 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
-import java.time.Instant;
-import java.time.ZoneOffset;
-import java.time.format.DateTimeFormatter;
-import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Properties;
import javax.net.ssl.HttpsURLConnection;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.amazonaws.services.s3.transfer.TransferManager;
-
-import org.apache.commons.lang3.StringUtils;
import org.apache.commons.io.IOUtils;
import org.apache.jackrabbit.core.data.DataStore;
import org.apache.jackrabbit.oak.commons.PropertiesUtil;
import org.apache.jackrabbit.oak.commons.collections.MapUtils;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils;
-import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
-import static org.apache.jackrabbit.oak.blob.cloud.s3.S3Backend.RemoteStorageMode;
+import software.amazon.awssdk.utils.StringUtils;
/**
* Extension to {@link DataStoreUtils} to enable S3 extensions for cleaning and initialization.
@@ -92,6 +80,16 @@ public class S3DataStoreUtils extends DataStoreUtils {
return true;
}
+ /**
+ *
+ * @return true if SSE_C encryption is configured
+ */
+ public static boolean isSseCustomerKeyEncrypted() {
+ final Properties s3Config = getS3Config();
+ return Objects.equals(Utils.getDataEncryption(s3Config), DataEncryption.SSE_C)
+ || Objects.nonNull(s3Config.getProperty(S3Constants.S3_SSE_C_KEY));
+ }
+
/**
* Read any config property configured.
* Also, read any props available as system properties.
@@ -146,50 +144,10 @@ public class S3DataStoreUtils extends DataStoreUtils {
return getS3DataStore(className, getS3Config(), homeDir);
}
- public static void deleteBucket(String bucket, Date date) throws Exception {
- log.info("cleaning bucket [" + bucket + "]");
+ public static void deleteBucket(String bucket, Date date) {
+ log.info("cleaning bucket [ {} ]", bucket);
Properties props = getS3Config();
- AmazonS3Client s3service = Utils.openService(props);
- TransferManager tmx = new TransferManager(s3service);
- if (s3service.doesBucketExist(bucket)) {
- for (int i = 0; i < 4; i++) {
- tmx.abortMultipartUploads(bucket, date);
- ObjectListing prevObjectListing = s3service.listObjects(bucket);
- while (prevObjectListing != null) {
- List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
- List<String> keysToDelete = new ArrayList<>();
- for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
- deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
- keysToDelete.add(s3ObjSumm.getKey());
- }
- if (!deleteList.isEmpty()) {
- RemoteStorageMode mode = getMode(props);
- if (mode == RemoteStorageMode.S3) {
- DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
- delObjsReq.setKeys(deleteList);
- s3service.deleteObjects(delObjsReq);
- } else {
- keysToDelete.forEach(key -> s3service.deleteObject(bucket, key));
- }
- }
- if (!prevObjectListing.isTruncated())
- break;
- prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
- }
- }
- s3service.deleteBucket(bucket);
- log.info("bucket [ " + bucket + "] cleaned");
- } else {
- log.info("bucket [" + bucket + "] doesn't exists");
- }
- tmx.shutdownNow();
- s3service.shutdown();
- }
-
- @NotNull
- private static RemoteStorageMode getMode(@NotNull Properties props) {
- return props.getProperty(S3Constants.S3_END_POINT, "").contains("googleapis") ?
- RemoteStorageMode.GCP : RemoteStorageMode.S3;
+ S3BackendHelper.deleteBucketAndAbortMultipartUploads(bucket, date, props);
}
protected static HttpsURLConnection getHttpsConnection(long length, URI uri) throws IOException {
@@ -197,9 +155,33 @@ public class S3DataStoreUtils extends DataStoreUtils {
conn.setDoOutput(true);
conn.setRequestMethod("PUT");
conn.setRequestProperty("Content-Length", String.valueOf(length));
- conn.setRequestProperty("Date", DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssX").withZone(ZoneOffset.UTC).format(Instant.now()));
- conn.setRequestProperty("Host", uri.getHost());
+ final Properties props = getS3Config();
+ DataEncryption encryption = Utils.getDataEncryption(props);
+
+ switch (encryption) {
+ case SSE_S3:
+ conn.setRequestProperty("x-amz-server-side-encryption", "AES256");
+ System.out.println("Added SSE-S3 header: AES256");
+ break;
+ case SSE_KMS:
+ conn.setRequestProperty("x-amz-server-side-encryption", "aws:kms");
+ System.out.println("Added SSE-KMS header: aws:kms");
+ if (Objects.nonNull(props.getProperty(S3Constants.S3_SSE_KMS_KEYID))) {
+ conn.setRequestProperty("x-amz-server-side-encryption-aws-kms-key-id", props.getProperty(S3Constants.S3_SSE_KMS_KEYID));
+ System.out.println("Added KMS Key ID: " + props.getProperty(S3Constants.S3_SSE_KMS_KEYID));
+ }
+ break;
+ case SSE_C:
+ String customerKey = props.getProperty(S3Constants.S3_SSE_C_KEY);
+ String customerKeyMD5 = Utils.calculateMD5(customerKey);
+ conn.setRequestProperty("x-amz-server-side-encryption-customer-algorithm", "AES256");
+ conn.setRequestProperty("x-amz-server-side-encryption-customer-key", customerKey);
+ conn.setRequestProperty("x-amz-server-side-encryption-customer-key-MD5", customerKeyMD5);
+ break;
+ case NONE:
+ break;
+ }
return conn;
}
}
diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3RequestDecoratorTest.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3RequestDecoratorTest.java
new file mode 100644
index 0000000000..6f9c181d1d
--- /dev/null
+++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3RequestDecoratorTest.java
@@ -0,0 +1,509 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.blob.cloud.s3;
+
+import org.junit.Test;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.ServerSideEncryption;
+import software.amazon.awssdk.utils.BinaryUtils;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+/**
+ * Unit cases for S3RequestDecorator
+ */
+public class S3RequestDecoratorTest {
+
+ private final String sseCustomerKey = BinaryUtils.toBase64("customer-key".getBytes(StandardCharsets.UTF_8));
+ private final String kmsKeyId = "kms-key-id";
+
+ @Test
+ public void testConstructorWithNoneEncryption() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "NONE");
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+ assertEquals(DataEncryption.NONE, decorator.dataEncryption);
+ assertNull(decorator.sseKmsKey);
+ assertNull(decorator.sseCustomerKey);
+ }
+
+ @Test
+ public void testConstructorWithSSES3Encryption() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "SSE_S3");
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+ assertEquals(DataEncryption.SSE_S3, decorator.dataEncryption);
+ assertNull(decorator.sseKmsKey);
+ assertNull(decorator.sseCustomerKey);
+ }
+
+ @Test
+ public void testConstructorWithSSEKMSEncryptionWithKey() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "SSE_KMS");
+ props.setProperty(S3Constants.S3_SSE_KMS_KEYID, "kms-key-id");
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+ assertEquals(DataEncryption.SSE_KMS, decorator.dataEncryption);
+ assertEquals("kms-key-id", decorator.sseKmsKey);
+ assertNull(decorator.sseCustomerKey);
+ }
+
+ @Test
+ public void testConstructorWithSSEKMSEncryptionWithoutKey() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "SSE_KMS");
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+ assertEquals(DataEncryption.SSE_KMS, decorator.dataEncryption);
+ assertNull(decorator.sseKmsKey);
+ assertNull(decorator.sseCustomerKey);
+ }
+
+ @Test
+ public void testConstructorWithSSECEncryptionWithKey() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "SSE_C");
+ props.setProperty(S3Constants.S3_SSE_C_KEY, "customer-key");
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+ assertEquals(DataEncryption.SSE_C, decorator.dataEncryption);
+ assertNull(decorator.sseKmsKey);
+ assertEquals("customer-key", decorator.sseCustomerKey);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testConstructorWithSSECEncryptionWithoutKey() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "SSE_C");
+ new S3RequestDecorator(props);
+ }
+
+ @Test
+ public void testConstructorWithNullEncryptionType() {
+ Properties props = new Properties();
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+ assertEquals(DataEncryption.NONE, decorator.dataEncryption);
+ assertNull(decorator.sseKmsKey);
+ assertNull(decorator.sseCustomerKey);
+ }
+
+ @Test
+ public void testDecorateHeadObjectWithSSEC() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "SSE_C");
+ props.setProperty(S3Constants.S3_SSE_C_KEY, sseCustomerKey);
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+
+ HeadObjectRequest req = HeadObjectRequest.builder()
+ .bucket("bucket")
+ .key("key")
+ .build();
+
+ HeadObjectRequest decorated = decorator.decorate(req);
+
+ assertEquals(sseCustomerKey, decorated.sseCustomerKey());
+ assertEquals("AES256", decorated.sseCustomerAlgorithm());
+ assertEquals(Utils.calculateMD5(sseCustomerKey), decorated.sseCustomerKeyMD5());
+ assertEquals("bucket", decorated.bucket());
+ assertEquals("key", decorated.key());
+ }
+
+ @Test
+ public void testDecorateHeadObjectWithNone() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "NONE");
+ props.setProperty(S3Constants.S3_SSE_C_KEY, sseCustomerKey);
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+
+ HeadObjectRequest req = HeadObjectRequest.builder()
+ .bucket("bucket")
+ .key("key")
+ .build();
+
+ HeadObjectRequest decorated = decorator.decorate(req);
+
+ assertNull(decorated.sseCustomerKey());
+ assertNull(decorated.sseCustomerAlgorithm());
+ assertNull(decorated.sseCustomerKeyMD5());
+ assertEquals("bucket", decorated.bucket());
+ assertEquals("key", decorated.key());
+ }
+
+ @Test
+ public void testDecorateGetObjectWithSSEC() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "SSE_C");
+ props.setProperty(S3Constants.S3_SSE_C_KEY, sseCustomerKey);
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+
+ GetObjectRequest req = GetObjectRequest.builder()
+ .bucket("bucket")
+ .key("key")
+ .build();
+
+ GetObjectRequest decorated = decorator.decorate(req);
+
+ assertEquals(sseCustomerKey, decorated.sseCustomerKey());
+ assertEquals("AES256", decorated.sseCustomerAlgorithm());
+ assertEquals(Utils.calculateMD5(sseCustomerKey), decorated.sseCustomerKeyMD5());
+ assertEquals("bucket", decorated.bucket());
+ assertEquals("key", decorated.key());
+ }
+
+ @Test
+ public void testDecorateGetObjectWithNone() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "NONE");
+ props.setProperty(S3Constants.S3_SSE_C_KEY, sseCustomerKey);
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+
+ GetObjectRequest req = GetObjectRequest.builder()
+ .bucket("bucket")
+ .key("key")
+ .build();
+
+ GetObjectRequest decorated = decorator.decorate(req);
+
+ assertNull(decorated.sseCustomerKey());
+ assertNull(decorated.sseCustomerAlgorithm());
+ assertNull(decorated.sseCustomerKeyMD5());
+ assertEquals("bucket", decorated.bucket());
+ assertEquals("key", decorated.key());
+ }
+
+ @Test
+ public void testDecorateCompleteMultipartUploadWithSSEC() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "SSE_C");
+ props.setProperty(S3Constants.S3_SSE_C_KEY, sseCustomerKey);
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+
+ CompleteMultipartUploadRequest req = CompleteMultipartUploadRequest.builder()
+ .bucket("bucket")
+ .key("key")
+ .uploadId("uploadId")
+ .build();
+
+ CompleteMultipartUploadRequest decorated = decorator.decorate(req);
+
+ assertEquals(sseCustomerKey, decorated.sseCustomerKey());
+ assertEquals("AES256", decorated.sseCustomerAlgorithm());
+ assertEquals(Utils.calculateMD5(sseCustomerKey), decorated.sseCustomerKeyMD5());
+ assertEquals("bucket", decorated.bucket());
+ assertEquals("key", decorated.key());
+ assertEquals("uploadId", decorated.uploadId());
+ }
+
+ @Test
+ public void testDecorateCompleteMultipartUploadRequestWithNone() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "NONE");
+ props.setProperty(S3Constants.S3_SSE_C_KEY, sseCustomerKey);
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+
+ CompleteMultipartUploadRequest req = CompleteMultipartUploadRequest.builder()
+ .bucket("bucket")
+ .key("key")
+ .uploadId("uploadId")
+ .build();
+
+ CompleteMultipartUploadRequest decorated = decorator.decorate(req);
+
+ assertNull(decorated.sseCustomerKey());
+ assertNull(decorated.sseCustomerAlgorithm());
+ assertNull(decorated.sseCustomerKeyMD5());
+ assertEquals("bucket", decorated.bucket());
+ assertEquals("key", decorated.key());
+ assertEquals("uploadId", decorated.uploadId());
+ }
+
+ @Test
+ public void testDecoratePutObjectWithSSES3() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "SSE_S3");
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+
+ PutObjectRequest req = PutObjectRequest.builder()
+ .bucket("bucket")
+ .key("key")
+ .build();
+
+ PutObjectRequest decorated = decorator.decorate(req);
+
+ assertEquals(ServerSideEncryption.AES256, decorated.serverSideEncryption());
+ assertEquals("bucket", decorated.bucket());
+ assertEquals("key", decorated.key());
+ }
+
+ @Test
+ public void testDecoratePutObjectWithSSEKMS() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "SSE_KMS");
+ props.setProperty(S3Constants.S3_SSE_KMS_KEYID, kmsKeyId);
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+
+ PutObjectRequest req = PutObjectRequest.builder()
+ .bucket("bucket")
+ .key("key")
+ .build();
+
+ PutObjectRequest decorated = decorator.decorate(req);
+
+ assertEquals(ServerSideEncryption.AWS_KMS, decorated.serverSideEncryption());
+ assertEquals(kmsKeyId, decorated.ssekmsKeyId());
+ assertEquals("bucket", decorated.bucket());
+ assertEquals("key", decorated.key());
+ }
+
+ @Test
+ public void testDecoratePutObjectWithSSEC() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "SSE_C");
+ props.setProperty(S3Constants.S3_SSE_C_KEY, sseCustomerKey);
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+
+ PutObjectRequest req = PutObjectRequest.builder()
+ .bucket("bucket")
+ .key("key")
+ .build();
+
+ PutObjectRequest decorated = decorator.decorate(req);
+
+ assertEquals("AES256", decorated.sseCustomerAlgorithm());
+ assertEquals(sseCustomerKey, decorated.sseCustomerKey());
+ assertEquals(Utils.calculateMD5(sseCustomerKey), decorated.sseCustomerKeyMD5());
+ assertEquals("bucket", decorated.bucket());
+ assertEquals("key", decorated.key());
+ }
+
+ @Test
+ public void testDecoratePutObjectWithNone() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "NONE");
+ props.setProperty(S3Constants.S3_SSE_C_KEY, sseCustomerKey);
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+
+ PutObjectRequest req = PutObjectRequest.builder()
+ .bucket("bucket")
+ .key("key")
+ .build();
+
+ PutObjectRequest decorated = decorator.decorate(req);
+
+ assertNull(decorated.serverSideEncryption());
+ assertNull(decorated.ssekmsKeyId());
+ assertNull(decorated.sseCustomerAlgorithm());
+ assertNull(decorated.sseCustomerKey());
+ assertNull(decorated.sseCustomerKeyMD5());
+ assertEquals("bucket", decorated.bucket());
+ assertEquals("key", decorated.key());
+ }
+
+ @Test
+ public void testDecorateCopyObjectWithSSES3() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "SSE_S3");
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+
+ CopyObjectRequest req = CopyObjectRequest.builder()
+ .destinationBucket("bucket")
+ .destinationKey("key")
+ .sourceBucket("src-bucket")
+ .sourceKey("src-key")
+ .build();
+
+ CopyObjectRequest decorated = decorator.decorate(req);
+
+ assertEquals(ServerSideEncryption.AES256, decorated.serverSideEncryption());
+ assertEquals("bucket", decorated.destinationBucket());
+ assertEquals("key", decorated.destinationKey());
+ assertEquals("src-bucket", decorated.sourceBucket());
+ assertEquals("src-key", decorated.sourceKey());
+ }
+
+ @Test
+ public void testDecorateCopyObjectWithSSEKMS() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "SSE_KMS");
+ props.setProperty(S3Constants.S3_SSE_KMS_KEYID, kmsKeyId);
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+
+ CopyObjectRequest req = CopyObjectRequest.builder()
+ .destinationBucket("bucket")
+ .destinationKey("key")
+ .sourceBucket("src-bucket")
+ .sourceKey("src-key")
+ .build();
+
+ CopyObjectRequest decorated = decorator.decorate(req);
+
+ assertEquals(ServerSideEncryption.AWS_KMS, decorated.serverSideEncryption());
+ assertEquals(kmsKeyId, decorated.ssekmsKeyId());
+ assertEquals("bucket", decorated.destinationBucket());
+ assertEquals("key", decorated.destinationKey());
+ assertEquals("src-bucket", decorated.sourceBucket());
+ assertEquals("src-key", decorated.sourceKey());
+ }
+
+ @Test
+ public void testDecorateCopyObjectWithSSEC() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "SSE_C");
+ props.setProperty(S3Constants.S3_SSE_C_KEY, sseCustomerKey);
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+
+ CopyObjectRequest req = CopyObjectRequest.builder()
+ .destinationBucket("bucket")
+ .destinationKey("key")
+ .sourceBucket("src-bucket")
+ .sourceKey("src-key")
+ .build();
+
+ CopyObjectRequest decorated = decorator.decorate(req);
+
+ assertEquals("AES256", decorated.sseCustomerAlgorithm());
+ assertEquals("AES256", decorated.copySourceSSECustomerAlgorithm());
+ assertEquals(sseCustomerKey, decorated.sseCustomerKey());
+ assertEquals(sseCustomerKey, decorated.copySourceSSECustomerKey());
+ assertEquals(Utils.calculateMD5(sseCustomerKey), decorated.sseCustomerKeyMD5());
+ assertEquals(Utils.calculateMD5(sseCustomerKey), decorated.copySourceSSECustomerKeyMD5());
+ assertEquals("bucket", decorated.destinationBucket());
+ assertEquals("key", decorated.destinationKey());
+ assertEquals("src-bucket", decorated.sourceBucket());
+ assertEquals("src-key", decorated.sourceKey());
+ }
+
+ @Test
+ public void testDecorateCopyObjectWithNone() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "NONE");
+ props.setProperty(S3Constants.S3_SSE_C_KEY, sseCustomerKey);
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+
+ CopyObjectRequest req = CopyObjectRequest.builder()
+ .destinationBucket("bucket")
+ .destinationKey("key")
+ .sourceBucket("src-bucket")
+ .sourceKey("src-key")
+ .build();
+
+ CopyObjectRequest decorated = decorator.decorate(req);
+
+ assertNull(decorated.serverSideEncryption());
+ assertNull(decorated.ssekmsKeyId());
+ assertNull(decorated.sseCustomerAlgorithm());
+ assertNull(decorated.copySourceSSECustomerAlgorithm());
+ assertNull(decorated.sseCustomerKey());
+ assertNull(decorated.copySourceSSECustomerKey());
+ assertNull(decorated.sseCustomerKeyMD5());
+ assertNull(decorated.copySourceSSECustomerKeyMD5());
+ assertEquals("bucket", decorated.destinationBucket());
+ assertEquals("key", decorated.destinationKey());
+ assertEquals("src-bucket", decorated.sourceBucket());
+ assertEquals("src-key", decorated.sourceKey());
+ }
+
+ @Test
+ public void testDecorateCreateMultipartUploadWithSSES3() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "SSE_S3");
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+
+ CreateMultipartUploadRequest req = CreateMultipartUploadRequest.builder()
+ .bucket("bucket")
+ .key("key")
+ .build();
+
+ CreateMultipartUploadRequest decorated = decorator.decorate(req);
+
+ assertEquals(ServerSideEncryption.AES256, decorated.serverSideEncryption());
+ assertEquals("bucket", decorated.bucket());
+ assertEquals("key", decorated.key());
+ }
+
+ @Test
+ public void testDecorateCreateMultipartUploadWithSSEKMS() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "SSE_KMS");
+ props.setProperty(S3Constants.S3_SSE_KMS_KEYID, kmsKeyId);
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+
+ CreateMultipartUploadRequest req = CreateMultipartUploadRequest.builder()
+ .bucket("bucket")
+ .key("key")
+ .build();
+
+ CreateMultipartUploadRequest decorated = decorator.decorate(req);
+
+ assertEquals(ServerSideEncryption.AWS_KMS, decorated.serverSideEncryption());
+ assertEquals(kmsKeyId, decorated.ssekmsKeyId());
+ assertEquals("bucket", decorated.bucket());
+ assertEquals("key", decorated.key());
+ }
+
+ @Test
+ public void testDecorateCreateMultipartUploadWithSSEC() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "SSE_C");
+ props.setProperty(S3Constants.S3_SSE_C_KEY, sseCustomerKey);
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+
+ CreateMultipartUploadRequest req = CreateMultipartUploadRequest.builder()
+ .bucket("bucket")
+ .key("key")
+ .build();
+
+ CreateMultipartUploadRequest decorated = decorator.decorate(req);
+
+ assertEquals("AES256", decorated.sseCustomerAlgorithm());
+ assertEquals(sseCustomerKey, decorated.sseCustomerKey());
+ assertEquals(Utils.calculateMD5(sseCustomerKey), decorated.sseCustomerKeyMD5());
+ assertEquals("bucket", decorated.bucket());
+ assertEquals("key", decorated.key());
+ }
+
+ @Test
+ public void testDecorateCreateMultipartUploadWithNone() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "NONE");
+ props.setProperty(S3Constants.S3_SSE_C_KEY, sseCustomerKey);
+ S3RequestDecorator decorator = new S3RequestDecorator(props);
+
+ CreateMultipartUploadRequest req = CreateMultipartUploadRequest.builder()
+ .bucket("bucket")
+ .key("key")
+ .build();
+
+ CreateMultipartUploadRequest decorated = decorator.decorate(req);
+
+ assertNull(decorated.serverSideEncryption());
+ assertNull(decorated.ssekmsKeyId());
+ assertNull(decorated.sseCustomerAlgorithm());
+ assertNull(decorated.sseCustomerKey());
+ assertNull(decorated.sseCustomerKeyMD5());
+ assertEquals("bucket", decorated.bucket());
+ assertEquals("key", decorated.key());
+ }
+
+}
\ No newline at end of file
diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSECustomerKey.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSECustomerKey.java
index c8d1b20e64..6863802f29 100644
--- a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSECustomerKey.java
+++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSECustomerKey.java
@@ -19,8 +19,8 @@ package org.apache.jackrabbit.oak.blob.cloud.s3;
import org.junit.AssumptionViolatedException;
import org.junit.Before;
import org.slf4j.Logger;
+import software.amazon.awssdk.utils.StringUtils;
-import static com.amazonaws.util.StringUtils.hasValue;
import static org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants.S3_ENCRYPTION;
import static org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants.S3_ENCRYPTION_SSE_C;
import static org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants.S3_SSE_C_KEY;
@@ -36,19 +36,23 @@ import static org.slf4j.LoggerFactory.getLogger;
*/
public class TestS3DSWithSSECustomerKey extends TestS3Ds {
- protected static final Logger LOG = getLogger(TestS3DSWithSSECustomerKey.class);
+ protected static final Logger LOG = getLogger(TestS3DSWithSSECustomerKey.class);
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- String keyId = props.getProperty(S3_SSE_C_KEY);
- if (hasValue(keyId)) {
- props.setProperty(S3_ENCRYPTION, S3_ENCRYPTION_SSE_C);
- props.setProperty(S3_SSE_C_KEY, keyId);
- } else {
- LOG.info("SSE Customer Key ID not configured so ignoring test");
- throw new AssumptionViolatedException("SSE Customer key Id not configured");
- }
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ }
+
+ @Override
+ protected void setEncryptionData() {
+ String keyId = props.getProperty(S3_SSE_C_KEY);
+ if (StringUtils.isNotBlank(keyId)) {
+ props.setProperty(S3_ENCRYPTION, S3_ENCRYPTION_SSE_C);
+ props.setProperty(S3_SSE_C_KEY, keyId);
+ } else {
+ LOG.info("SSE Customer Key ID not configured so ignoring test");
+ throw new AssumptionViolatedException("SSE Customer key Id not configured");
}
+ }
}
diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSEKMS.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSEKMS.java
index 3a545ad534..b1bd8550fb 100644
--- a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSEKMS.java
+++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSEKMS.java
@@ -20,8 +20,6 @@ import org.junit.Before;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getS3Config;
-
/**
* Test S3DataStore operation with SSE_KMS encryption.
* It requires to pass aws config file via system property or system properties by prefixing with 'ds.'.
@@ -32,16 +30,20 @@ import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getS3Conf
*/
public class TestS3DSWithSSEKMS extends TestS3Ds {
- protected static final Logger LOG = LoggerFactory.getLogger(TestS3DSWithSSES3.class);
+ protected static final Logger LOG = LoggerFactory.getLogger(TestS3DSWithSSES3.class);
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ }
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- props.setProperty(S3Constants.S3_ENCRYPTION, S3Constants.S3_ENCRYPTION_SSE_KMS);
- String kmsKey = props.getProperty(S3Constants.S3_SSE_KMS_KEYID);
- if (kmsKey != null) {
- props.remove(S3Constants.S3_SSE_KMS_KEYID);
- }
+ @Override
+ protected void setEncryptionData() {
+ props.setProperty(S3Constants.S3_ENCRYPTION, S3Constants.S3_ENCRYPTION_SSE_KMS);
+ String kmsKey = props.getProperty(S3Constants.S3_SSE_KMS_KEYID);
+ if (kmsKey != null) {
+ props.remove(S3Constants.S3_SSE_KMS_KEYID);
}
+ }
}
diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSEKMSwithKey.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSEKMSwithKey.java
index 6b5e9dd066..2a5745dbf0 100644
--- a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSEKMSwithKey.java
+++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSEKMSwithKey.java
@@ -32,19 +32,23 @@ import org.slf4j.LoggerFactory;
*/
public class TestS3DSWithSSEKMSwithKey extends TestS3Ds {
- protected static final Logger LOG = LoggerFactory.getLogger(TestS3DSWithSSES3.class);
+ protected static final Logger LOG = LoggerFactory.getLogger(TestS3DSWithSSES3.class);
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- String keyid = props.getProperty(S3Constants.S3_SSE_KMS_KEYID);
- if (!StringUtils.isEmpty(keyid)) {
- props.setProperty(S3Constants.S3_ENCRYPTION, S3Constants.S3_ENCRYPTION_SSE_KMS);
- props.setProperty(S3Constants.S3_SSE_KMS_KEYID, keyid);
- } else {
- LOG.info("Key ID not configured so ignoring test");
- throw new AssumptionViolatedException("KMS key Id not configured");
- }
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ }
+
+ @Override
+ protected void setEncryptionData() {
+ String keyid = props.getProperty(S3Constants.S3_SSE_KMS_KEYID);
+ if (!StringUtils.isEmpty(keyid)) {
+ props.setProperty(S3Constants.S3_ENCRYPTION, S3Constants.S3_ENCRYPTION_SSE_KMS);
+ props.setProperty(S3Constants.S3_SSE_KMS_KEYID, keyid);
+ } else {
+ LOG.info("Key ID not configured so ignoring test");
+ throw new AssumptionViolatedException("KMS key Id not configured");
}
+ }
}
diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSES3.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSES3.java
index 7fd2dd4f69..eeea71b02a 100644
--- a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSES3.java
+++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSES3.java
@@ -36,6 +36,10 @@ public class TestS3DSWithSSES3 extends TestS3Ds {
@Before
public void setUp() throws Exception {
super.setUp();
+ }
+
+ @Override
+ protected void setEncryptionData() {
props.setProperty(S3Constants.S3_ENCRYPTION, S3Constants.S3_ENCRYPTION_SSE_S3);
}
}
diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java
index 23ddea3972..b64dfd7bf3 100644
--- a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java
+++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java
@@ -29,14 +29,6 @@ import java.util.Properties;
import javax.jcr.RepositoryException;
import org.apache.commons.lang3.time.DateUtils;
-import org.apache.http.HttpEntity;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.entity.InputStreamEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.message.BasicHeader;
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStore;
import org.apache.jackrabbit.core.data.DataStoreException;
@@ -51,6 +43,7 @@ import org.jetbrains.annotations.Nullable;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -58,16 +51,16 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.core.sync.ResponseTransformer;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3Uri;
+import software.amazon.awssdk.services.s3.S3Utilities;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
+import software.amazon.awssdk.services.s3.model.ServerSideEncryption;
-import static com.amazonaws.services.s3.Headers.SERVER_SIDE_ENCRYPTION;
-import static com.amazonaws.services.s3.Headers.SERVER_SIDE_ENCRYPTION_AWS_KMS_KEYID;
-import static com.amazonaws.services.s3.Headers.SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM;
-import static com.amazonaws.services.s3.Headers.SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY;
-import static com.amazonaws.services.s3.Headers.SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5;
-import static com.amazonaws.services.s3.model.SSEAlgorithm.AES256;
-import static com.amazonaws.services.s3.model.SSEAlgorithm.KMS;
-import static com.amazonaws.util.Base64.decode;
-import static com.amazonaws.util.Md5Utils.md5AsBase64;
import static org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants.S3_ENCRYPTION;
import static org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants.S3_ENCRYPTION_SSE_C;
import static org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants.S3_ENCRYPTION_SSE_KMS;
@@ -77,9 +70,10 @@ import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getFixtur
import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getS3Config;
import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getS3DataStore;
import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.isS3Configured;
+import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.isSseCustomerKeyEncrypted;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
-import static org.junit.Assume.assumeTrue;
+import static software.amazon.awssdk.services.s3.model.ServerSideEncryption.AES256;
/**
* Test {@link S3DataStore} with S3Backend and local cache on.
@@ -117,9 +111,11 @@ public class TestS3Ds extends AbstractDataStoreTest {
return DateUtils.addMinutes(new Date(), -1);
}
+ protected void setEncryptionData() {}
+
@BeforeClass
public static void assumptions() {
- assumeTrue(isS3Configured());
+ Assume.assumeTrue(isS3Configured());
}
private static List<String> createdBucketNames = new ArrayList<>();
@@ -139,6 +135,7 @@ public class TestS3Ds extends AbstractDataStoreTest {
props.setProperty(S3Constants.PRESIGNED_URI_ENABLE_ACCELERATION, "60");
props.setProperty(S3Constants.PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE, "60");
props.setProperty(S3_ENCRYPTION, S3Constants.S3_ENCRYPTION_NONE);
+ setEncryptionData();
super.setUp();
}
@@ -164,6 +161,7 @@ public class TestS3Ds extends AbstractDataStoreTest {
@Test
public void testGetDownloadURI() throws IOException, RepositoryException {
+ Assume.assumeTrue("SSE-C doesn't support presigned GET URLs", !isSseCustomerKeyEncrypted());
DataStore ds = createDataStore();
byte[] data = new byte[dataLength];
@@ -172,21 +170,22 @@ public class TestS3Ds extends AbstractDataStoreTest {
DataRecord record = doSynchronousAddRecord(ds, new ByteArrayInputStream(data));
URI uri = ((DataRecordAccessProvider) ds).getDownloadURI(record.getIdentifier(),
DataRecordDownloadOptions.DEFAULT);
- Assert.assertNotNull("uri is null", uri);
+ assertNotNull("uri is null", uri);
// Download content from the URI directly and check
- HttpEntity entity = httpGet(uri);
- assertStream(new ByteArrayInputStream(data), entity.getContent());
+ InputStream entity = httpGet(uri);
+ assertStream(new ByteArrayInputStream(data), entity);
// Download with DataStore API and check
DataRecord getrec = ds.getRecord(record.getIdentifier());
- Assert.assertNotNull(getrec);
+ assertNotNull(getrec);
Assert.assertEquals(data.length, getrec.getLength());
assertRecord(data, getrec);
}
@Test
public void testDataMigration() {
+ Assume.assumeTrue("For SSE-C we can't change encryption without manual intervention", !isSseCustomerKeyEncrypted());
try {
String encryption = props.getProperty(S3_ENCRYPTION);
@@ -243,8 +242,8 @@ public class TestS3Ds extends AbstractDataStoreTest {
randomGen.nextBytes(data);
// Upload directly using the URI and check
- CloseableHttpResponse response = httpPut(uploadContext, new ByteArrayInputStream(data), data.length);
- Assert.assertEquals(200, response.getStatusLine().getStatusCode());
+ PutObjectResponse response = httpPut(uploadContext, new ByteArrayInputStream(data), data.length);
+ Assert.assertEquals(200, response.sdkHttpResponse().statusCode());
DataRecord uploadedRecord = ds.completeDataRecordUpload(uploadToken);
assertNotNull(uploadedRecord);
Assert.assertEquals(data.length, uploadedRecord.getLength());
@@ -252,64 +251,11 @@ public class TestS3Ds extends AbstractDataStoreTest {
// Retieve through DataStore API and check
DataRecord getrec = ds.getRecord(uploadedRecord.getIdentifier());
- Assert.assertNotNull(getrec);
+ assertNotNull(getrec);
Assert.assertEquals(data.length, getrec.getLength());
assertRecord(data, getrec);
}
- public CloseableHttpResponse httpPut(@Nullable DataRecordUpload uploadContext, InputStream inputstream, long length) throws IOException {
- // this weird combination of @Nullable and assertNotNull() is for IDEs not warning in test methods
- assertNotNull(uploadContext);
-
- URI puturl = uploadContext.getUploadURIs().iterator().next();
- HttpPut putreq = new HttpPut(puturl);
-
- String keyId = null;
- String encryptionType = props.getProperty(S3_ENCRYPTION);
-
- switch (encryptionType) {
- case S3_ENCRYPTION_SSE_KMS:
- keyId = props.getProperty(S3_SSE_KMS_KEYID);
- putreq.addHeader(new BasicHeader(SERVER_SIDE_ENCRYPTION, KMS.getAlgorithm()));
- if (keyId != null) {
- putreq.addHeader(new BasicHeader(SERVER_SIDE_ENCRYPTION_AWS_KMS_KEYID, keyId));
- }
- break;
- case S3_ENCRYPTION_SSE_C:
- keyId = props.getProperty(S3_SSE_C_KEY);
- putreq.addHeader(new BasicHeader(SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, AES256.getAlgorithm()));
- putreq.addHeader(new BasicHeader(SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, keyId));
- putreq.addHeader(new BasicHeader(SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, md5AsBase64(decode(keyId))));
- break;
- default:
- break;
- }
-
- putreq.setEntity(new InputStreamEntity(inputstream , length));
- CloseableHttpClient httpclient = HttpClients.createDefault();
- CloseableHttpResponse response = httpclient.execute(putreq);
- return response;
- }
-
-
- private HttpEntity httpGet(URI uri) throws IOException {
- HttpGet getreq = new HttpGet(uri);
-
- final String encryptionType = props.getProperty(S3_ENCRYPTION);
-
- if (Objects.equals(S3_ENCRYPTION_SSE_C, encryptionType)) {
- String keyId = props.getProperty(S3_SSE_C_KEY);
- getreq.addHeader(new BasicHeader(SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, AES256.getAlgorithm()));
- getreq.addHeader(new BasicHeader(SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, keyId));
- getreq.addHeader(new BasicHeader(SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, md5AsBase64(decode(keyId))));
- }
-
- CloseableHttpClient httpclient = HttpClients.createDefault();
- CloseableHttpResponse res = httpclient.execute(getreq);
- Assert.assertEquals(200, res.getStatusLine().getStatusCode());
- return res.getEntity();
- }
-
protected DataRecord doSynchronousAddRecord(DataStore ds, InputStream in) throws DataStoreException {
return ((S3DataStore)ds).addRecord(in, new BlobOptions().setUpload(BlobOptions.UploadType.SYNCHRONOUS));
}
@@ -368,4 +314,86 @@ public class TestS3Ds extends AbstractDataStoreTest {
@Override
public void testDeleteAllOlderThan() {
}
+
+ // helper methods
+
+ private PutObjectResponse httpPut(@Nullable DataRecordUpload uploadContext, InputStream inputstream, long length) {
+ // this weird combination of @Nullable and assertNotNull() is for IDEs not warning in test methods
+ assertNotNull(uploadContext);
+
+ URI putUri = uploadContext.getUploadURIs().iterator().next();
+ try (S3Client s3Client = Utils.openService(props, false)) {
+ String bucketName = extractBucketFromUri(s3Client, putUri);
+ String key = extractKeyFromUri(s3Client, putUri);
+
+ String encryptionType = props.getProperty(S3_ENCRYPTION);
+
+ PutObjectRequest.Builder putReqBuilder = PutObjectRequest.builder()
+ .bucket(bucketName)
+ .key(key);
+
+ switch (encryptionType) {
+ case S3_ENCRYPTION_SSE_KMS:
+ String keyId = props.getProperty(S3_SSE_KMS_KEYID);
+ putReqBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS);
+ if (keyId != null) {
+ putReqBuilder.ssekmsKeyId(keyId);
+ }
+ break;
+ case S3_ENCRYPTION_SSE_C:
+ String sseCustomerKey = props.getProperty(S3_SSE_C_KEY);
+ putReqBuilder
+ .sseCustomerAlgorithm(AES256.toString())
+ .sseCustomerKey(sseCustomerKey)
+ .sseCustomerKeyMD5(Utils.calculateMD5(sseCustomerKey)); // implement helpers
+ break;
+ default:
+ // No encryption
+ break;
+ }
+ return s3Client.putObject(putReqBuilder.build(), RequestBody.fromInputStream(inputstream, length));
+ }
+ }
+
+ private InputStream httpGet(URI uri) {
+ String encryptionType = props.getProperty(S3_ENCRYPTION);
+
+ try (S3Client s3Client = Utils.openService(props, false)) {
+ String bucketName = extractBucketFromUri(s3Client, uri);
+ String key = extractKeyFromUri(s3Client, uri);
+
+ GetObjectRequest.Builder req = GetObjectRequest.builder()
+ .bucket(bucketName)
+ .key(key);
+
+ if (Objects.equals(S3_ENCRYPTION_SSE_C, encryptionType)) {
+ String keyId = props.getProperty(S3_SSE_C_KEY);
+ if (keyId != null) {
+ req.sseCustomerAlgorithm(AES256.toString())
+ .sseCustomerKey(keyId)
+ .sseCustomerKeyMD5(Utils.calculateMD5(keyId));
+ }
+ }
+
+ return s3Client.getObject(req.build(), ResponseTransformer.toInputStream());
+ }
+ }
+
+ private static String extractBucketFromUri(S3Client s3Client, URI uri) {
+ LOG.info("Extracting bucket from URI {}", uri);
+ S3Utilities s3Utilities = s3Client.utilities();
+
+ S3Uri s3Uri = s3Utilities.parseUri(uri);
+
+ return s3Uri.bucket().orElse(null);
+ }
+
+ private static String extractKeyFromUri(S3Client s3Client, URI uri) {
+ LOG.info("Extracting key from URI {}", uri);
+ S3Utilities s3Utilities = s3Client.utilities();
+
+ S3Uri s3Uri = s3Utilities.parseUri(uri);
+
+ return s3Uri.key().orElse(null);
+ }
}
diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/UtilsTest.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/UtilsTest.java
new file mode 100644
index 0000000000..e47fe2af52
--- /dev/null
+++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/UtilsTest.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.blob.cloud.s3;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.net.URI;
+import java.util.Properties;
+
+/**
+ * Unit cases for Utils class
+ */
+public class UtilsTest {
+
+ @Test
+ public void testGetRegionFromEndpoint() {
+ Properties props = new Properties();
+ props.setProperty("s3EndPoint", "https://s3.eu-west-1.amazonaws.com");
+ props.setProperty("protocol", "https");
+ Assert.assertEquals("eu-west-1", Utils.getRegion(props));
+ }
+
+ @Test
+ public void testGetRegionFromEndpointUsEast1() {
+ Properties props = new Properties();
+ props.setProperty("s3EndPoint", "https://s3.amazonaws.com");
+ Assert.assertEquals("us-east-1", Utils.getRegion(props));
+ }
+
+ @Test
+ public void testGetRegionFromProperty() {
+ Properties props = new Properties();
+ props.setProperty("s3Region", "ap-south-1");
+ Assert.assertEquals("ap-south-1", Utils.getRegion(props));
+ }
+
+ @Test
+ public void testGetRegionFromPropertyUsStandard() {
+ Properties props = new Properties();
+ props.setProperty("s3Region", "us-standard");
+ Assert.assertEquals("us-east-1", Utils.getRegion(props));
+ }
+
+ @Test
+ public void testGetRegionFallbackToDefault() {
+ String previous = System.getProperty("aws.region");
+ System.setProperty("aws.region", "us-west-2");
+ try {
+ Properties props = new Properties();
+ props.setProperty("s3Region", "");
+ String region = Utils.getRegion(props);
+ Assert.assertNotNull(region);
+ Assert.assertFalse(region.isEmpty());
+ } finally {
+ if (previous != null) {
+ System.setProperty("aws.region", previous);
+ } else {
+ System.clearProperty("aws.region");
+ }
+ }
+ }
+
+ @Test
+ public void testGetEndPointUriWithCustomEndpoint() {
+ Properties props = new Properties();
+ props.setProperty("s3EndPoint", "https://custom.endpoint.com");
+ props.setProperty("s3ConnProtocol", "http");
+ URI uri = Utils.getEndPointUri(props, false, "eu-west-1");
+ Assert.assertEquals("https://custom.endpoint.com", uri.toString());
+ }
+
+ @Test
+ public void testGetEndPointUriWithAcceleration() {
+ Properties props = new Properties();
+ props.setProperty("s3ConnProtocol", "https");
+ URI uri = Utils.getEndPointUri(props, true, "ap-south-1");
+ Assert.assertEquals("https://s3-accelerate.ap-south-1.amazonaws.com", uri.toString());
+ }
+
+ @Test
+ public void testGetEndPointUriWithRegion() {
+ Properties props = new Properties();
+ props.setProperty("s3ConnProtocol", "https");
+ URI uri = Utils.getEndPointUri(props, false, "us-east-2");
+ Assert.assertEquals("https://s3.us-east-2.amazonaws.com", uri.toString());
+ }
+
+ @Test
+ public void testGetEndPointUriDefaultProtocol() {
+ Properties props = new Properties();
+ // No protocol set, should default to https
+ URI uri = Utils.getEndPointUri(props, false, "us-west-1");
+ Assert.assertEquals("https://s3.us-west-1.amazonaws.com", uri.toString());
+ }
+
+ @Test
+ public void testGetRegionFromStandardEndpoint() {
+ Assert.assertEquals("eu-west-1", Utils.getRegionFromEndpoint("https://s3.eu-west-1.amazonaws.com"));
+ }
+
+ @Test
+ public void testGetRegionFromVirtualHostedEndpoint() {
+ Assert.assertEquals("ap-south-1", Utils.getRegionFromEndpoint("https://bucket.s3.ap-south-1.amazonaws.com"));
+ }
+
+ @Test
+ public void testGetRegionFromUsEast1Endpoint() {
+ Assert.assertEquals("us-east-1", Utils.getRegionFromEndpoint("https://s3.amazonaws.com"));
+ }
+
+ @Test
+ public void testGetRegionFromVirtualHostedUsEast1() {
+ Assert.assertEquals("us-east-1", Utils.getRegionFromEndpoint("https://bucket.s3.amazonaws.com"));
+ }
+
+ @Test
+ public void testGetRegionFromInvalidEndpoint() {
+ Assert.assertNull(Utils.getRegionFromEndpoint("https://example.com"));
+ }
+
+ @Test
+ public void testGetRegionFromMalformedEndpoint() {
+ Assert.assertNull(Utils.getRegionFromEndpoint("not-a-valid-uri"));
+ }
+
+ @Test
+ public void testGetDataEncryptionNoneWhenUnset() {
+ Properties props = new Properties();
+ Assert.assertEquals(DataEncryption.NONE, Utils.getDataEncryption(props));
+ }
+
+ @Test
+ public void testGetDataEncryptionSSEKMS() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "SSE_KMS");
+ Assert.assertEquals(DataEncryption.SSE_KMS, Utils.getDataEncryption(props));
+ }
+
+ @Test
+ public void testGetDataEncryptionSSES3() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "SSE_S3");
+ Assert.assertEquals(DataEncryption.SSE_S3, Utils.getDataEncryption(props));
+ }
+
+ @Test
+ public void testGetDataEncryptionInvalidValue() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.S3_ENCRYPTION, "INVALID");
+ try {
+ Utils.getDataEncryption(props);
+ Assert.fail("Expected IllegalArgumentException for invalid encryption type");
+ } catch (IllegalArgumentException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testSetRemoteStorageModeS3() {
+ Properties props = new Properties();
+ props.setProperty("s3EndPoint", "https://s3.amazonaws.com");
+ Utils.setRemoteStorageMode(props);
+ Assert.assertEquals(S3Backend.RemoteStorageMode.S3, props.get(S3Constants.MODE));
+ }
+
+ @Test
+ public void testSetRemoteStorageModeGCP() {
+ Properties props = new Properties();
+ props.setProperty("s3EndPoint", "https://storage.googleapis.com");
+ Utils.setRemoteStorageMode(props);
+ Assert.assertEquals(S3Backend.RemoteStorageMode.GCP, props.get(S3Constants.MODE));
+ }
+
+ @Test
+ public void testSetRemoteStorageModeOverrideWarning() {
+ Properties props = new Properties();
+ props.setProperty("s3EndPoint", "https://storage.googleapis.com");
+ props.put(S3Constants.MODE, S3Backend.RemoteStorageMode.S3);
+ Utils.setRemoteStorageMode(props);
+ Assert.assertEquals(S3Backend.RemoteStorageMode.GCP, props.get(S3Constants.MODE));
+ }
+
+}
\ No newline at end of file
diff --git a/oak-blob-cloud/src/test/resources/aws.properties b/oak-blob-cloud/src/test/resources/aws.properties
index fb5623d4bb..0817a4a775 100644
--- a/oak-blob-cloud/src/test/resources/aws.properties
+++ b/oak-blob-cloud/src/test/resources/aws.properties
@@ -22,7 +22,7 @@ secretKey=
s3Bucket=
# AWS bucket region
# Mapping of S3 regions to their constants
-# US Standard us-standard
+# US Standard us-east-1
# US West us-west-2
# US West (Northern California) us-west-1
# EU (Ireland) EU
@@ -44,8 +44,15 @@ socketTimeout=120000
maxConnections=20
maxErrorRetry=10
endpoint-url=
+# S3 connection protocol
+s3ConnProtocol=https
# maximum concurrent threads to write to S3.
writeThreads=10
# proxy configurations (optional)
#proxyHost=
#proxyPort=
+s3Encryption=
+# Mode of this connector, set value to GCP to store objects on GCP, default value is S3
+mode=
+# cross region s3 bucket access
+crossRegionAccess=false
diff --git a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/directaccess/AbstractDataRecordAccessProviderTest.java b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/directaccess/AbstractDataRecordAccessProviderTest.java
index 5972c391e0..7f5b37e753 100644
--- a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/directaccess/AbstractDataRecordAccessProviderTest.java
+++ b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/directaccess/AbstractDataRecordAccessProviderTest.java
@@ -43,6 +43,7 @@ import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.api.blob.BlobDownloadOptions;
import org.apache.jackrabbit.util.Base64;
import org.jetbrains.annotations.NotNull;
+import org.junit.Assume;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -69,6 +70,9 @@ public abstract class AbstractDataRecordAccessProviderTest {
protected abstract DataRecord doGetRecord(DataStore ds, DataIdentifier identifier) throws DataStoreException;
protected abstract DataRecord doSynchronousAddRecord(DataStore ds, InputStream in) throws DataStoreException;
protected abstract void doDeleteRecord(DataStore ds, DataIdentifier identifier) throws DataStoreException;
+ protected boolean isSSECustomerKeyEncryption() {
+ return false;
+ }
protected static int expirySeconds = 60*15;
@@ -87,6 +91,7 @@ public abstract class AbstractDataRecordAccessProviderTest {
//
@Test
public void testGetDownloadURIProvidesValidURIIT() throws DataStoreException {
+ Assume.assumeTrue("SSE-C doesn't support presigned GET URLs", !isSSECustomerKeyEncryption());
DataRecord record = null;
ConfigurableDataRecordAccessProvider dataStore = getDataStore();
try {
@@ -152,6 +157,7 @@ public abstract class AbstractDataRecordAccessProviderTest {
@Test
public void testGetDownloadURIIT() throws DataStoreException, IOException {
+ Assume.assumeTrue("SSE-C doesn't support presigned GET URLs", !isSSECustomerKeyEncryption());
DataRecord record = null;
DataRecordAccessProvider dataStore = getDataStore();
try {
@@ -174,6 +180,7 @@ public abstract class AbstractDataRecordAccessProviderTest {
@Test
public void testGetDownloadURIWithCustomHeadersIT() throws DataStoreException, IOException {
+ Assume.assumeTrue("SSE-C doesn't support presigned GET URLs", !isSSECustomerKeyEncryption());
String umlautFilename = "Uml\u00e4utfile.png";
String umlautFilename_ISO_8859_1 = new String(
StandardCharsets.ISO_8859_1.encode(umlautFilename).array(),
@@ -250,6 +257,7 @@ public abstract class AbstractDataRecordAccessProviderTest {
@Test
public void testGetExpiredReadURIFailsIT() throws DataStoreException, IOException {
+ Assume.assumeTrue("SSE-C doesn't support presigned GET URLs", !isSSECustomerKeyEncryption());
DataRecord record = null;
ConfigurableDataRecordAccessProvider dataStore = getDataStore();
try {
diff --git a/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/datastore/S3DataStoreFixture.java b/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/datastore/S3DataStoreFixture.java
index 0b602ae005..8d688e6a16 100644
--- a/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/datastore/S3DataStoreFixture.java
+++ b/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/datastore/S3DataStoreFixture.java
@@ -24,17 +24,11 @@ import static org.junit.Assert.assertTrue;
import java.util.Properties;
import java.util.UUID;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.model.BucketAccelerateConfiguration;
-import com.amazonaws.services.s3.model.BucketAccelerateStatus;
-import com.amazonaws.services.s3.model.CreateBucketRequest;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.amazonaws.services.s3.model.SetBucketAccelerateConfigurationRequest;
import org.apache.jackrabbit.core.data.DataStore;
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.blob.cloud.s3.S3Backend;
import org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants;
+import org.apache.jackrabbit.oak.blob.cloud.s3.S3BackendHelper;
import org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStore;
import org.apache.jackrabbit.oak.blob.cloud.s3.Utils;
import org.apache.jackrabbit.oak.fixture.NodeStoreFixture;
@@ -43,6 +37,12 @@ import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.AccelerateConfiguration;
+import software.amazon.awssdk.services.s3.model.BucketAccelerateStatus;
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
+import software.amazon.awssdk.services.s3.model.PutBucketAccelerateConfigurationRequest;
/**
* Fixture for S3DataStore based on an aws.properties config file. It creates
@@ -93,44 +93,46 @@ public class S3DataStoreFixture implements DataStoreFixture {
throw new AssertionError("createDataStore() called but this fixture is not available");
}
- AmazonS3 s3Client = Utils.openService(s3Props);
-
- // Create a temporary bucket that will be removed at test completion
- String bucketName = "direct-binary-test-" + UUID.randomUUID().toString();
-
- log.info("Creating S3 test bucket {}", bucketName);
- CreateBucketRequest createBucket = new CreateBucketRequest(bucketName);
- s3Client.createBucket(createBucket);
- assertTrue("Failed to create test bucket [" + bucketName + "]", Utils.waitForBucket(s3Client, bucketName));
-
- log.info("Enabling S3 acceleration for bucket {}", bucketName);
- s3Client.setBucketAccelerateConfiguration(
- new SetBucketAccelerateConfigurationRequest(bucketName,
- new BucketAccelerateConfiguration(
- BucketAccelerateStatus.Enabled
- )
- )
- );
-
- s3Client.shutdown();
+ String bucketName = null;
+
+ try (S3Client s3Client = Utils.openService(s3Props, false)) {
+ // Create a temporary bucket that will be removed at test completion
+ bucketName = "direct-binary-test-" + UUID.randomUUID();
+
+ log.info("Creating S3 test bucket {}", bucketName);
+ CreateBucketRequest createBucketRequest = CreateBucketRequest.builder().bucket(bucketName).build();
+ s3Client.createBucket(createBucketRequest);
+ assertTrue("Failed to create test bucket [" + bucketName + "]", S3BackendHelper.waitForBucket(s3Client, bucketName, 20, 100L));
+
+ log.info("Enabling S3 acceleration for bucket {}", bucketName);
+ s3Client.putBucketAccelerateConfiguration(
+ PutBucketAccelerateConfigurationRequest.builder()
+ .bucket(bucketName)
+ .accelerateConfiguration(
+ AccelerateConfiguration.builder()
+ .status(BucketAccelerateStatus.ENABLED)
+ .build())
+ .build()
+ );
+ }
// create new properties since azProps is shared for all created DataStores
Properties clonedS3Props = new Properties(s3Props);
- clonedS3Props.setProperty(S3Constants.S3_BUCKET, createBucket.getBucketName());
+ clonedS3Props.setProperty(S3Constants.S3_BUCKET, bucketName);
// setup Oak DS
S3DataStore dataStore = new S3DataStore();
dataStore.setProperties(clonedS3Props);
dataStore.setStagingSplitPercentage(0);
- log.info("s3props: " + s3Props.toString());
+ log.info("s3props: {}", s3Props);
return dataStore;
}
@Override
public void dispose(DataStore dataStore) {
- if (dataStore != null && dataStore instanceof S3DataStore) {
+ if (dataStore instanceof S3DataStore) {
try {
dataStore.close();
} catch (DataStoreException e) {
@@ -148,26 +150,10 @@ public class S3DataStoreFixture implements DataStoreFixture {
return;
}
- AmazonS3 s3Client = Utils.openService(s3Props);
-
- // For S3, you have to empty the bucket before removing the bucket itself
- log.info("Emptying S3 test bucket {}", bucketName);
-
- ObjectListing listing = s3Client.listObjects(bucketName);
- while (true) {
- for (S3ObjectSummary summary : listing.getObjectSummaries()) {
- s3Client.deleteObject(bucketName, summary.getKey());
- }
- if (! listing.isTruncated()) {
- break;
- }
- listing = s3Client.listNextBatchOfObjects(listing);
+ try (S3Client s3Client = Utils.openService(s3Props, false)) {
+ S3BackendHelper.deleteBucketObjects(bucketName, s3Props, s3Client);
+ s3Client.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build());
}
-
- log.info("Removing S3 test bucket {}", bucketName);
- s3Client.deleteBucket(bucketName);
-
- s3Client.shutdown();
}
}
}
diff --git a/oak-parent/pom.xml b/oak-parent/pom.xml
index 2c9c14726c..1b2279932b 100644
--- a/oak-parent/pom.xml
+++ b/oak-parent/pom.xml
@@ -785,6 +785,13 @@
<artifactId>accessors-smart</artifactId>
<version>2.4.9</version>
</dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>bom</artifactId>
+ <version>2.34.1</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
<!-- Pax Exam Integration Test Dependencies -->
<dependency>
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java
index 56502ea90f..1a77a7f441 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java
@@ -18,11 +18,6 @@
*/
package org.apache.jackrabbit.oak.fixture;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.amazonaws.services.s3.transfer.TransferManager;
import com.microsoft.azure.storage.blob.CloudBlobContainer;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
@@ -32,8 +27,8 @@ import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureBlobContainer
import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants;
import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureDataStore;
import org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants;
+import org.apache.jackrabbit.oak.blob.cloud.s3.S3BackendHelper;
import org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStore;
-import org.apache.jackrabbit.oak.blob.cloud.s3.Utils;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -41,9 +36,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.util.ArrayList;
import java.util.Date;
-import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -51,6 +44,9 @@ import java.util.Properties;
* Extension to {@link DataStoreUtils} to enable S3 / AzureBlob extensions for cleaning and initialization.
*/
public class DataStoreUtils {
+
+ private DataStoreUtils() {}
+
private static final Logger log = LoggerFactory.getLogger(DataStoreUtils.class);
private static Class S3 = S3DataStore.class;
@@ -67,7 +63,7 @@ public class DataStoreUtils {
public static DataStore configureIfCloudDataStore(String className, DataStore ds,
Map<String, ?> config, String bucket,
- StatisticsProvider statisticsProvider) throws Exception {
+ StatisticsProvider statisticsProvider) {
// Add bucket info
Properties props = new Properties();
props.putAll(config);
@@ -112,40 +108,11 @@ public class DataStoreUtils {
}
}
- public static void deleteBucket(String bucket, Map<String, ?> map, Date date) throws Exception {
- log.info("cleaning bucket [" + bucket + "]");
+ public static void deleteBucket(String bucket, Map<String, ?> map, Date date) {
+ log.info("cleaning bucket [ {} ]", bucket);
Properties props = new Properties();
props.putAll(map);
- AmazonS3Client s3service = Utils.openService(props);
- TransferManager tmx = new TransferManager(s3service);
- if (s3service.doesBucketExist(bucket)) {
- for (int i = 0; i < 4; i++) {
- tmx.abortMultipartUploads(bucket, date);
- ObjectListing prevObjectListing = s3service.listObjects(bucket);
- while (prevObjectListing != null) {
- List<DeleteObjectsRequest.KeyVersion>
- deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
- for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
- deleteList.add(new DeleteObjectsRequest.KeyVersion(
- s3ObjSumm.getKey()));
- }
- if (deleteList.size() > 0) {
- DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(
- bucket);
- delObjsReq.setKeys(deleteList);
- s3service.deleteObjects(delObjsReq);
- }
- if (!prevObjectListing.isTruncated()) break;
- prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
- }
- }
- s3service.deleteBucket(bucket);
- log.info("bucket [ " + bucket + "] cleaned");
- } else {
- log.info("bucket [" + bucket + "] doesn't exists");
- }
- tmx.shutdownNow();
- s3service.shutdown();
+ S3BackendHelper.deleteBucketAndAbortMultipartUploads(bucket, date, props);
}
public static void deleteAzureContainer(Map<String, ?> config, String containerName) throws Exception {
diff --git a/oak-run-elastic/pom.xml b/oak-run-elastic/pom.xml
index 48a585fc8d..27145562c2 100644
--- a/oak-run-elastic/pom.xml
+++ b/oak-run-elastic/pom.xml
@@ -42,8 +42,9 @@
105 MB: Azure updates
107 MB: RDB/Tomcat (OAK-10752)
120 MB: added commons-csv for csv parsing (OAK-10790)
+ 124 MB: Aws Sdk 2.x (OAK-11935)
-->
- <max.jar.size>120000000</max.jar.size>
+ <max.jar.size>130000000</max.jar.size>
</properties>
diff --git a/oak-run/pom.xml b/oak-run/pom.xml
index 14c0f6e55f..cb72d4dc28 100644
--- a/oak-run/pom.xml
+++ b/oak-run/pom.xml
@@ -34,6 +34,7 @@
<jetty.version>9.4.53.v20231009</jetty.version>
<!--
Size History:
+ + 87 MB (Aws Sdk 2.x, OAK-11935)
+ 84 MB (RDB/Tomcat, OAK-10752)
+ 80 MB (Java 17, OAK-10638)
+ 78.7 MB Apache Commons Lang ยป 3.0 (OAK-10615)
@@ -51,7 +52,7 @@
+ 41 MB build failing on the release profile (OAK-6250)
+ 38 MB. Initial value. Current 35MB plus a 10%
-->
- <max.jar.size>88080384</max.jar.size>
+ <max.jar.size>91226112</max.jar.size>
</properties>
<build>
diff --git a/oak-segment-aws/pom.xml b/oak-segment-aws/pom.xml
index c26f5321dd..5ecf1e673a 100644
--- a/oak-segment-aws/pom.xml
+++ b/oak-segment-aws/pom.xml
@@ -235,6 +235,11 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>s3</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>