You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2020/03/09 06:54:28 UTC
svn commit: r1874995 - in /jackrabbit/oak/branches/1.22: ./
oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/
oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/
oak-blob-cloud/src/test/resources/ oak-doc/src/site/...
Author: amitj
Date: Mon Mar 9 06:54:27 2020
New Revision: 1874995
URL: http://svn.apache.org/viewvc?rev=1874995&view=rev
Log:
OAK-8494: Support AWS Key Managed Service (SSE-KMS)
Merge r1874153 from trunk
Added:
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSEKMS.java
- copied unchanged from r1874153, jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSEKMS.java
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSEKMSwithKey.java
- copied unchanged from r1874153, jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSEKMSwithKey.java
jackrabbit/oak/branches/1.22/oak-doc/src/site/markdown/features/direct-binary-access-upload-file.md
- copied unchanged from r1874153, jackrabbit/oak/trunk/oak-doc/src/site/markdown/features/direct-binary-access-upload-file.md
Modified:
jackrabbit/oak/branches/1.22/ (props changed)
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3RequestDecorator.java
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSES3.java
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/test/resources/aws.properties
jackrabbit/oak/branches/1.22/oak-doc/src/site/site.xml
Propchange: jackrabbit/oak/branches/1.22/
------------------------------------------------------------------------------
Merged /jackrabbit/oak/trunk:r1874153
Modified: jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java?rev=1874995&r1=1874994&r2=1874995&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java (original)
+++ jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java Mon Mar 9 06:54:27 2020
@@ -84,6 +84,7 @@ import com.google.common.collect.Abstrac
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.commons.io.IOUtils;
+import org.apache.http.protocol.HTTP;
import org.apache.jackrabbit.core.data.DataIdentifier;
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
@@ -865,7 +866,7 @@ public class S3Backend extends AbstractS
else {
// multi-part
InitiateMultipartUploadRequest req = new InitiateMultipartUploadRequest(bucket, blobId);
- InitiateMultipartUploadResult res = s3service.initiateMultipartUpload(req);
+ InitiateMultipartUploadResult res = s3service.initiateMultipartUpload(s3ReqDecorator.decorate(req));
uploadId = res.getUploadId();
long numParts;
@@ -1024,6 +1025,10 @@ public class S3Backend extends AbstractS
.withMethod(method)
.withExpiration(expiration);
+ if (method != HttpMethod.GET) {
+ request = s3ReqDecorator.decorate(request);
+ }
+
for (Map.Entry<String, String> e : reqParams.entrySet()) {
request.addRequestParameter(e.getKey(), e.getValue());
}
Modified: jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java?rev=1874995&r1=1874994&r2=1874995&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java (original)
+++ jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java Mon Mar 9 06:54:27 2020
@@ -103,6 +103,21 @@ public final class S3Constants {
public static final String S3_ENCRYPTION_SSE_S3 = "SSE_S3";
/**
+ * Constant to set SSE_KMS encryption.
+ */
+ public static final String S3_ENCRYPTION_SSE_KMS = "SSE_KMS";
+
+ /**
+ * Constant to set keyID for SSE_KMS encryption.
+ */
+ public static final String S3_SSE_KMS_KEYID = "kmsKeyId";
+
+ /**
+ * Constant to set S3 signature for SSE_KMS encryption.
+ */
+ public static final String S3_SIG_V4 = "AWSS3V4SignerType";
+
+ /**
* Constant to set proxy host.
*/
public static final String PROXY_HOST = "proxyHost";
Modified: jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3RequestDecorator.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3RequestDecorator.java?rev=1874995&r1=1874994&r2=1874995&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3RequestDecorator.java (original)
+++ jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3RequestDecorator.java Mon Mar 9 06:54:27 2020
@@ -20,8 +20,13 @@ package org.apache.jackrabbit.oak.blob.c
import java.util.Properties;
import com.amazonaws.services.s3.model.CopyObjectRequest;
+import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
+import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.services.s3.model.SSEAlgorithm;
+import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams;
+import com.amazonaws.util.StringUtils;
/**
* This class to sets encrption mode in S3 request.
@@ -29,10 +34,21 @@ import com.amazonaws.services.s3.model.P
*/
public class S3RequestDecorator {
DataEncryption dataEncryption = DataEncryption.NONE;
+ Properties props;
+ SSEAwsKeyManagementParams sseParams;
public S3RequestDecorator(Properties props) {
- if (props.getProperty(S3Constants.S3_ENCRYPTION) != null) {
- this.dataEncryption = dataEncryption.valueOf(props.getProperty(S3Constants.S3_ENCRYPTION));
+ String encryptionType = props.getProperty(S3Constants.S3_ENCRYPTION);
+ if (encryptionType != null) {
+ this.dataEncryption = dataEncryption.valueOf(encryptionType);
+
+ if (encryptionType.equals(S3Constants.S3_ENCRYPTION_SSE_KMS)) {
+ String keyId = props.getProperty(S3Constants.S3_SSE_KMS_KEYID);
+ sseParams = new SSEAwsKeyManagementParams();
+ if (!StringUtils.isNullOrEmpty(keyId)) {
+ sseParams.withAwsKmsKeyId(keyId);
+ }
+ }
}
}
@@ -40,17 +56,22 @@ public class S3RequestDecorator {
* Set encryption in {@link PutObjectRequest}
*/
public PutObjectRequest decorate(PutObjectRequest request) {
+ ObjectMetadata metadata = request.getMetadata() == null
+ ? new ObjectMetadata()
+ : request.getMetadata();
switch (getDataEncryption()) {
case SSE_S3:
- ObjectMetadata metadata = request.getMetadata() == null
- ? new ObjectMetadata()
- : request.getMetadata();
metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
- request.setMetadata(metadata);
+ break;
+ case SSE_KMS:
+ metadata.setSSEAlgorithm(SSEAlgorithm.KMS.getAlgorithm());
+ /*Set*/
+ request.withSSEAwsKeyManagementParams(sseParams);
break;
case NONE:
break;
}
+ request.setMetadata(metadata);
return request;
}
@@ -58,20 +79,59 @@ public class S3RequestDecorator {
* Set encryption in {@link CopyObjectRequest}
*/
public CopyObjectRequest decorate(CopyObjectRequest request) {
+ ObjectMetadata metadata = request.getNewObjectMetadata() == null
+ ? new ObjectMetadata()
+ : request.getNewObjectMetadata();;
+ switch (getDataEncryption()) {
+ case SSE_S3:
+ metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
+ break;
+ case SSE_KMS:
+ metadata.setSSEAlgorithm(SSEAlgorithm.KMS.getAlgorithm());
+ request.withSSEAwsKeyManagementParams(sseParams);
+ break;
+ case NONE:
+ break;
+ }
+ request.setNewObjectMetadata(metadata);
+ return request;
+ }
+
+ public InitiateMultipartUploadRequest decorate(InitiateMultipartUploadRequest request) {
+ ObjectMetadata metadata = request.getObjectMetadata() == null
+ ? new ObjectMetadata()
+ : request.getObjectMetadata();;
switch (getDataEncryption()) {
case SSE_S3:
- ObjectMetadata metadata = request.getNewObjectMetadata() == null
- ? new ObjectMetadata()
- : request.getNewObjectMetadata();
metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
- request.setNewObjectMetadata(metadata);
+ break;
+ case SSE_KMS:
+ metadata.setSSEAlgorithm(SSEAlgorithm.KMS.getAlgorithm());
+ request.withSSEAwsKeyManagementParams(sseParams);
break;
case NONE:
break;
}
+ request.setObjectMetadata(metadata);
+ return request;
+ }
+
+ public GeneratePresignedUrlRequest decorate(GeneratePresignedUrlRequest request) {
+ switch (getDataEncryption()) {
+ case SSE_KMS:
+ String keyId = getSSEParams().getAwsKmsKeyId();
+ request = request.withSSEAlgorithm(SSEAlgorithm.KMS.getAlgorithm());
+ if (keyId != null) {
+ request = request.withKmsCmkId(keyId);
+ }
+ }
return request;
}
+ private SSEAwsKeyManagementParams getSSEParams() {
+ return this.sseParams;
+ }
+
private DataEncryption getDataEncryption() {
return this.dataEncryption;
}
@@ -81,7 +141,7 @@ public class S3RequestDecorator {
*
*/
private enum DataEncryption {
- SSE_S3, NONE;
+ SSE_S3, SSE_KMS, NONE;
}
}
Modified: jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java?rev=1874995&r1=1874994&r2=1874995&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java (original)
+++ jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java Mon Mar 9 06:54:27 2020
@@ -223,6 +223,7 @@ public final class Utils {
int socketTimeOut = Integer.parseInt(prop.getProperty(S3Constants.S3_SOCK_TIMEOUT));
int maxConnections = Integer.parseInt(prop.getProperty(S3Constants.S3_MAX_CONNS));
int maxErrorRetry = Integer.parseInt(prop.getProperty(S3Constants.S3_MAX_ERR_RETRY));
+ String encryptionType = prop.getProperty(S3Constants.S3_ENCRYPTION);
String protocol = prop.getProperty(S3Constants.S3_CONN_PROTOCOL);
String proxyHost = prop.getProperty(S3Constants.PROXY_HOST);
@@ -246,6 +247,10 @@ public final class Utils {
cc.setSocketTimeout(socketTimeOut);
cc.setMaxConnections(maxConnections);
cc.setMaxErrorRetry(maxErrorRetry);
+ if (encryptionType != null
+ && encryptionType.equals(S3Constants.S3_ENCRYPTION_SSE_KMS)) {
+ cc.withSignerOverride("AWSS3V4SignerType");
+ }
return cc;
}
Modified: jackrabbit/oak/branches/1.22/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSES3.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.22/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSES3.java?rev=1874995&r1=1874994&r2=1874995&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.22/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSES3.java (original)
+++ jackrabbit/oak/branches/1.22/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSES3.java Mon Mar 9 06:54:27 2020
@@ -17,24 +17,16 @@
package org.apache.jackrabbit.oak.blob.cloud.s3;
-import java.io.ByteArrayInputStream;
-
-import org.apache.jackrabbit.core.data.DataRecord;
-import org.junit.Assert;
import org.junit.Before;
-import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.junit.Assert.fail;
-
/**
* Test S3DataStore operation with SSE_S3 encryption.
* It requires to pass aws config file via system property or system properties by prefixing with 'ds.'.
* See details @ {@link S3DataStoreUtils}.
* For e.g. -Dconfig=/opt/cq/aws.properties. Sample aws properties located at
* src/test/resources/aws.properties
-
*/
public class TestS3DSWithSSES3 extends TestS3Ds {
@@ -45,42 +37,5 @@ public class TestS3DSWithSSES3 extends T
public void setUp() throws Exception {
super.setUp();
props.setProperty(S3Constants.S3_ENCRYPTION, S3Constants.S3_ENCRYPTION_SSE_S3);
- props.setProperty("cacheSize", "0");
- }
-
- /**
- * Test data migration enabling SSE_S3 encryption.
- */
- @Test
- public void testDataMigration() {
- try {
- //manually close the setup ds and remove encryption
- ds.close();
- props.remove(S3Constants.S3_ENCRYPTION);
- ds = createDataStore();
-
- byte[] data = new byte[dataLength];
- randomGen.nextBytes(data);
- DataRecord rec = ds.addRecord(new ByteArrayInputStream(data));
- Assert.assertEquals(data.length, rec.getLength());
- assertRecord(data, rec);
- ds.close();
-
- // turn encryption now anc recreate datastore instance
- props.setProperty(S3Constants.S3_ENCRYPTION, S3Constants.S3_ENCRYPTION_SSE_S3);
- props.setProperty(S3Constants.S3_RENAME_KEYS, "true");
- ds = createDataStore();
-
- rec = ds.getRecord(rec.getIdentifier());
- Assert.assertEquals(data.length, rec.getLength());
- assertRecord(data, rec);
-
- randomGen.nextBytes(data);
- ds.addRecord(new ByteArrayInputStream(data));
- ds.close();
- } catch (Exception e) {
- LOG.error("error:", e);
- fail(e.getMessage());
- }
}
}
Modified: jackrabbit/oak/branches/1.22/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.22/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java?rev=1874995&r1=1874994&r2=1874995&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.22/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java (original)
+++ jackrabbit/oak/branches/1.22/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java Mon Mar 9 06:54:27 2020
@@ -16,31 +16,58 @@
*/
package org.apache.jackrabbit.oak.blob.cloud.s3;
-import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getFixtures;
-import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getS3Config;
-import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getS3DataStore;
-import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.isS3Configured;
-import static org.junit.Assume.assumeTrue;
-
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
import java.util.Date;
import java.util.List;
import java.util.Properties;
import javax.jcr.RepositoryException;
+import com.amazonaws.services.s3.Headers;
+import com.amazonaws.services.s3.model.SSEAlgorithm;
import com.google.common.collect.Lists;
import org.apache.commons.lang3.time.DateUtils;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.entity.InputStreamEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicHeader;
+import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStore;
+import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.plugins.blob.datastore.AbstractDataStoreTest;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.ConfigurableDataRecordAccessProvider;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordAccessProvider;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUpload;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadException;
+import org.apache.jackrabbit.oak.spi.blob.BlobOptions;
+import org.jetbrains.annotations.Nullable;
import org.junit.After;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getFixtures;
+import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getS3Config;
+import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getS3DataStore;
+import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.isS3Configured;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+import static org.junit.Assume.assumeTrue;
+
/**
* Test {@link S3DataStore} with S3Backend and local cache on.
* It requires to pass aws config file via system property or system properties by prefixing with 'ds.'.
@@ -52,6 +79,10 @@ import org.slf4j.LoggerFactory;
public class TestS3Ds extends AbstractDataStoreTest {
protected static final Logger LOG = LoggerFactory.getLogger(TestS3Ds.class);
+ protected static long ONE_KB = 1024;
+ protected static long ONE_MB = ONE_KB * ONE_KB;
+ protected static long ONE_HUNDRED_MB = ONE_MB * 100;
+ protected static long ONE_GB = ONE_HUNDRED_MB * 10;
private static Date overallStartTime = getBackdatedDate();
private Date thisTestStartTime = null;
@@ -85,15 +116,184 @@ public class TestS3Ds extends AbstractDa
public void setUp() throws Exception {
props = getS3Config();
thisTestStartTime = getBackdatedDate();
- bucket =
- String.valueOf(randomGen.nextInt(9999)) + "-" + String.valueOf(randomGen.nextInt(9999))
- + "-s3ds-unittest-autogenerated";
+ bucket = randomGen.nextInt(9999) + "-" +
+ randomGen.nextInt(9999) + "-s3ds-unittest-autogenerated";
createdBucketNames.add(bucket);
props.setProperty(S3Constants.S3_BUCKET, bucket);
props.setProperty("secret", "123456");
+ props.setProperty(S3Constants.PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS,"60");
+ props.setProperty(S3Constants.PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS, "60");
+ props.setProperty(S3Constants.PRESIGNED_URI_ENABLE_ACCELERATION, "60");
+ props.setProperty(S3Constants.PRESIGNED_HTTP_DOWNLOAD_URI_CACHE_MAX_SIZE, "60");
+ props.setProperty(S3Constants.S3_ENCRYPTION, S3Constants.S3_ENCRYPTION_NONE);
super.setUp();
}
+ @Test
+ public void testInitiateDirectUploadUnlimitedURIs() throws DataRecordUploadException,
+ RepositoryException {
+ ConfigurableDataRecordAccessProvider ds
+ = (ConfigurableDataRecordAccessProvider) createDataStore();
+ long uploadSize = ONE_GB * 50;
+ int expectedNumURIs = 5000;
+ DataRecordUpload upload = ds.initiateDataRecordUpload(uploadSize, -1);
+ Assert.assertEquals(expectedNumURIs, upload.getUploadURIs().size());
+
+ uploadSize = ONE_GB * 100;
+ expectedNumURIs = 10000;
+ upload = ds.initiateDataRecordUpload(uploadSize, -1);
+ Assert.assertEquals(expectedNumURIs, upload.getUploadURIs().size());
+
+ uploadSize = ONE_GB * 200;
+ upload = ds.initiateDataRecordUpload(uploadSize, -1);
+ Assert.assertEquals(expectedNumURIs, upload.getUploadURIs().size());
+ }
+
+ @Test
+ public void testGetDownloadURI() throws IOException, RepositoryException {
+ DataStore ds = createDataStore();
+
+ byte[] data = new byte[dataLength];
+ randomGen.nextBytes(data);
+
+ DataRecord record = doSynchronousAddRecord(ds, new ByteArrayInputStream(data));
+ URI uri = ((DataRecordAccessProvider) ds).getDownloadURI(record.getIdentifier(),
+ DataRecordDownloadOptions.DEFAULT);
+ Assert.assertNotNull("uri is null", uri);
+
+ // Download content from the URI directly and check
+ HttpEntity entity = httpGet(uri);
+ assertStream(new ByteArrayInputStream(data), entity.getContent());
+
+ // Download with DataStore API and check
+ DataRecord getrec = ds.getRecord(record.getIdentifier());
+ Assert.assertNotNull(getrec);
+ Assert.assertEquals(data.length, getrec.getLength());
+ assertRecord(data, getrec);
+ }
+
+ @Test
+ public void testDataMigration() {
+ try {
+ String encryption = props.getProperty(S3Constants.S3_ENCRYPTION);
+
+ //manually close the setup ds and remove encryption
+ ds.close();
+ props.remove(S3Constants.S3_ENCRYPTION);
+ ds = createDataStore();
+
+ byte[] data = new byte[dataLength];
+ randomGen.nextBytes(data);
+ DataRecord rec = ds.addRecord(new ByteArrayInputStream(data));
+ Assert.assertEquals(data.length, rec.getLength());
+ assertRecord(data, rec);
+ ds.close();
+
+ // turn encryption now anc recreate datastore instance
+ props.setProperty(S3Constants.S3_ENCRYPTION, encryption);
+ props.setProperty(S3Constants.S3_RENAME_KEYS, "true");
+ ds = createDataStore();
+
+ Assert.assertNotEquals(null, ds);
+ rec = ds.getRecord(rec.getIdentifier());
+ Assert.assertNotEquals(null, rec);
+ Assert.assertEquals(data.length, rec.getLength());
+ assertRecord(data, rec);
+
+ randomGen.nextBytes(data);
+ rec = ds.addRecord(new ByteArrayInputStream(data));
+ DataRecord rec1 = ds.getRecord(rec.getIdentifier());
+ Assert.assertEquals(rec.getLength(), rec1.getLength());
+ assertRecord(data, rec);
+
+ ds.close();
+ } catch (Exception e) {
+ LOG.error("error:", e);
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testInitiateCompleteUpload() throws IOException, RepositoryException, IllegalArgumentException, DataRecordUploadException {
+
+ S3DataStore ds = (S3DataStore) createDataStore();
+ ds.setDirectUploadURIExpirySeconds(60*5);
+ ds.setDirectDownloadURIExpirySeconds(60*5);
+ ds.setDirectDownloadURICacheSize(60*5);
+
+ DataRecordUpload uploadContext = ds.initiateDataRecordUpload(ONE_GB, 1);
+ assertNotNull(uploadContext);
+
+ String uploadToken = uploadContext.getUploadToken();
+
+ byte[] data = new byte[dataLength];
+ randomGen.nextBytes(data);
+
+ // Upload directly using the URI and check
+ CloseableHttpResponse response = httpPut(uploadContext, new ByteArrayInputStream(data), data.length);
+ Assert.assertEquals(200, response.getStatusLine().getStatusCode());
+ DataRecord uploadedRecord = ds.completeDataRecordUpload(uploadToken);
+ assertNotNull(uploadedRecord);
+ Assert.assertEquals(data.length, uploadedRecord.getLength());
+ assertRecord(data, uploadedRecord);
+
+ // Retieve through DataStore API and check
+ DataRecord getrec = ds.getRecord(uploadedRecord.getIdentifier());
+ Assert.assertNotNull(getrec);
+ Assert.assertEquals(data.length, getrec.getLength());
+ assertRecord(data, getrec);
+ }
+
+ public CloseableHttpResponse httpPut(@Nullable DataRecordUpload uploadContext, InputStream inputstream, long length) throws IOException {
+ // this weird combination of @Nullable and assertNotNull() is for IDEs not warning in test methods
+ assertNotNull(uploadContext);
+
+ URI puturl = uploadContext.getUploadURIs().iterator().next();
+ HttpPut putreq = new HttpPut(puturl);
+
+ String keyId = null;
+ String encryptionType = props.getProperty(S3Constants.S3_ENCRYPTION);
+
+ if (encryptionType.equals(S3Constants.S3_ENCRYPTION_SSE_KMS)) {
+ keyId = props.getProperty(S3Constants.S3_SSE_KMS_KEYID);
+ putreq.addHeader(new BasicHeader(Headers.SERVER_SIDE_ENCRYPTION,
+ SSEAlgorithm.KMS.getAlgorithm()));
+ if(keyId != null) {
+ putreq.addHeader(new BasicHeader(Headers.SERVER_SIDE_ENCRYPTION_AWS_KMS_KEYID,
+ keyId));
+ }
+ }
+
+ putreq.setEntity(new InputStreamEntity(inputstream , length));
+ CloseableHttpClient httpclient = HttpClients.createDefault();
+ CloseableHttpResponse response = httpclient.execute(putreq);
+ return response;
+ }
+
+
+ private HttpEntity httpGet(URI uri) throws IOException {
+ HttpGet getreq = new HttpGet(uri);
+ CloseableHttpClient httpclient = HttpClients.createDefault();
+ CloseableHttpResponse res = httpclient.execute(getreq);
+ Assert.assertEquals(200, res.getStatusLine().getStatusCode());
+ return res.getEntity();
+ }
+
+ protected DataRecord doSynchronousAddRecord(DataStore ds, InputStream in) throws DataStoreException {
+ return ((S3DataStore)ds).addRecord(in, new BlobOptions().setUpload(BlobOptions.UploadType.SYNCHRONOUS));
+ }
+
+ private static void assertStream(InputStream expected, InputStream actual) throws IOException {
+ while (true) {
+ int expectedByte = expected.read();
+ int actualByte = actual.read();
+ Assert.assertEquals(expectedByte, actualByte);
+ if (expectedByte == -1) {
+ break;
+ }
+ }
+ }
+
@Override
@After
public void tearDown() {
Modified: jackrabbit/oak/branches/1.22/oak-blob-cloud/src/test/resources/aws.properties
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.22/oak-blob-cloud/src/test/resources/aws.properties?rev=1874995&r1=1874994&r2=1874995&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.22/oak-blob-cloud/src/test/resources/aws.properties (original)
+++ jackrabbit/oak/branches/1.22/oak-blob-cloud/src/test/resources/aws.properties Mon Mar 9 06:54:27 2020
@@ -14,7 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-
# AWS account ID
accessKey=
# AWS secret key
@@ -35,13 +34,14 @@ s3Region=
# S3 endpoint to be used. This parameter is optional
# and has a higher precedence over endpoint derived
# via S3 region.
-s3EndPoint=
+kmsKeyId=
connectionTimeout=120000
socketTimeout=120000
maxConnections=20
maxErrorRetry=10
+endpoint-url=
# maximum concurrent threads to write to S3.
writeThreads=10
# proxy configurations (optional)
-proxyHost=
-proxyPort=
+#proxyHost=
+#proxyPort=
Modified: jackrabbit/oak/branches/1.22/oak-doc/src/site/site.xml
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.22/oak-doc/src/site/site.xml?rev=1874995&r1=1874994&r2=1874995&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.22/oak-doc/src/site/site.xml (original)
+++ jackrabbit/oak/branches/1.22/oak-doc/src/site/site.xml Mon Mar 9 06:54:27 2020
@@ -55,6 +55,7 @@ under the License.
</item>
<item href="plugins/blobstore.html" name="Blob Storage" collapse="false">
<item href="features/direct-binary-access.html" name="Direct Binary Access" />
+ <item href="features/direct-binary-access-upload-file.html" name="Direct Binary Access Upload File" />
</item>
<item href="query/query.html" name="Query" collapse="false">
<item href="query/query-engine.html" name="Query Engine" />