You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2017/10/10 09:31:45 UTC
svn commit: r1811652 - in /jackrabbit/oak/trunk/oak-blob-cloud/src:
main/java/org/apache/jackrabbit/oak/blob/cloud/s3/
test/java/org/apache/jackrabbit/oak/blob/cloud/s3/
Author: amitj
Date: Tue Oct 10 09:31:44 2017
New Revision: 1811652
URL: http://svn.apache.org/viewvc?rev=1811652&view=rev
Log:
OAK-6802: Manage 'secret' property internally in S3/AzureDataStore
S3 implementation to manage the secret property internally and remove the need to be configured
Modified:
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStore.java
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java
Modified: jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java?rev=1811652&r1=1811651&r2=1811652&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java (original)
+++ jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java Tue Oct 10 09:31:44 2017
@@ -17,9 +17,10 @@
package org.apache.jackrabbit.oak.blob.cloud.s3;
+import java.io.ByteArrayInputStream;
import java.io.File;
+import java.io.IOException;
import java.io.InputStream;
-import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
@@ -56,6 +57,7 @@ import com.google.common.base.Strings;
import com.google.common.collect.AbstractIterator;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import org.apache.commons.io.IOUtils;
import org.apache.jackrabbit.core.data.DataIdentifier;
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
@@ -82,11 +84,13 @@ public class S3Backend extends AbstractS
private static final String META_KEY_PREFIX = "META/";
+ private static final String REF_KEY = "reference.key";
+
private AmazonS3Client s3service;
private String bucket;
- private String secret;
+ private byte[] secret;
private TransferManager tmx;
@@ -114,7 +118,6 @@ public class S3Backend extends AbstractS
bucket = properties.getProperty(S3Constants.S3_CONTAINER);
}
}
- secret = properties.getProperty("secret");
String region = properties.getProperty(S3Constants.S3_REGION);
Region s3Region;
if (StringUtils.isNullOrEmpty(region)) {
@@ -527,13 +530,36 @@ public class S3Backend extends AbstractS
@Override
public byte[] getOrCreateReferenceKey() throws DataStoreException {
try {
- if (!Strings.isNullOrEmpty(secret)) {
- return secret.getBytes("UTF-8");
+ if (secret != null && secret.length != 0) {
+ return secret;
+ } else {
+ byte[] key;
+ // Try reading from the metadata folder if it exists
+ if (metadataExists(REF_KEY)) {
+ DataRecord rec = getMetadataRecord(REF_KEY);
+ key = IOUtils.toByteArray(rec.getStream());
+ } else {
+ key = super.getOrCreateReferenceKey();
+ addMetadataRecord(new ByteArrayInputStream(key), REF_KEY);
+ }
+ secret = key;
+ return secret;
+ }
+ } catch (IOException e) {
+ throw new DataStoreException("Unable to get or create key " + e);
+ }
+ }
+
+ public boolean metadataExists(String name) {
+ ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
+ try {
+ Thread.currentThread().setContextClassLoader(
+ getClass().getClassLoader());
+ return s3service.doesObjectExist(bucket, addMetaKeyPrefix(name));
+ } finally {
+ if (contextClassLoader != null) {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
}
- LOG.warn("secret not defined");
- return super.getOrCreateReferenceKey();
- } catch (UnsupportedEncodingException e) {
- throw new DataStoreException(e);
}
}
Modified: jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStore.java?rev=1811652&r1=1811651&r2=1811652&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStore.java (original)
+++ jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStore.java Tue Oct 10 09:31:44 2017
@@ -29,10 +29,6 @@ import org.slf4j.LoggerFactory;
* Amazon S3 data store extending from {@link AbstractSharedCachingDataStore}.
*/
public class S3DataStore extends AbstractSharedCachingDataStore {
- /**
- * Logger instance.
- */
- private static final Logger LOG = LoggerFactory.getLogger(S3DataStore.class);
protected Properties properties;
@@ -41,8 +37,6 @@ public class S3DataStore extends Abstrac
*/
private int minRecordLength = 16 * 1024;
- private String secret;
-
@Override
protected AbstractSharedBackend createBackend() {
S3Backend backend = new S3Backend();
Modified: jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java?rev=1811652&r1=1811651&r2=1811652&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java (original)
+++ jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java Tue Oct 10 09:31:44 2017
@@ -18,6 +18,8 @@ package org.apache.jackrabbit.oak.blob.c
import java.io.ByteArrayInputStream;
import java.io.File;
+import java.util.Arrays;
+import java.util.Date;
import java.util.List;
import java.util.Properties;
import java.util.Random;
@@ -26,9 +28,10 @@ import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import javax.jcr.RepositoryException;
+import org.apache.commons.io.IOUtils;
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStore;
-import org.junit.Assert;
+import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -45,7 +48,7 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.isS3Configured;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeTrue;
/**
@@ -53,7 +56,7 @@ import static org.junit.Assume.assumeTru
*/
@RunWith(Parameterized.class)
public class TestS3DataStore {
- protected static final Logger LOG = LoggerFactory.getLogger(TestS3Ds.class);
+ protected static final Logger LOG = LoggerFactory.getLogger(TestS3DataStore.class);
@Rule
public ExpectedException expectedEx = ExpectedException.none();
@@ -69,14 +72,18 @@ public class TestS3DataStore {
private File dataStoreDir;
private DataStore ds;
+ private Date startTime;
@Parameterized.Parameters(name = "{index}: ({0})")
public static List<String> fixtures() {
return getFixtures();
}
+ private String bucket;
@Before
public void setUp() throws Exception {
+ startTime = new Date();
+
dataStoreDir = folder.newFolder();
props = new Properties();
}
@@ -94,42 +101,40 @@ public class TestS3DataStore {
}
@Test
- public void testNoSecretDefinedUseDefault() throws Exception {
+ public void testSecret() throws Exception {
assumeTrue(isS3Configured());
- assumeTrue(s3Class.equals(S3DataStoreUtils.S3.getName()));
Random randomGen = new Random();
props = S3DataStoreUtils.getS3Config();
ds = getS3DataStore(s3Class, props, dataStoreDir.getAbsolutePath());
- byte[] data = new byte[4096];
- randomGen.nextBytes(data);
- DataRecord rec = ds.addRecord(new ByteArrayInputStream(data));
- assertEquals(data.length, rec.getLength());
- assertNotNull(rec.getReference());
- }
+ bucket = props.getProperty(S3Constants.S3_BUCKET);
- @Test
- public void testSecretDefined() throws Exception {
- assumeTrue(isS3Configured());
-
- Random randomGen = new Random();
- props = S3DataStoreUtils.getS3Config();
- props.setProperty("secret", "123456");
- ds = getS3DataStore(s3Class, props, dataStoreDir.getAbsolutePath());
byte[] data = new byte[4096];
randomGen.nextBytes(data);
DataRecord rec = ds.addRecord(new ByteArrayInputStream(data));
assertEquals(data.length, rec.getLength());
String ref = rec.getReference();
- assertNotNull(ref);
String id = rec.getIdentifier().toString();
+ assertNotNull(ref);
+
+ S3DataStore s3 = ((S3DataStore) ds);
+ byte[] refKey = ((S3Backend) s3.getBackend()).getOrCreateReferenceKey();
+
Mac mac = Mac.getInstance("HmacSHA1");
- mac.init(new SecretKeySpec("123456".getBytes("UTF-8"), "HmacSHA1"));
+ mac.init(new SecretKeySpec(refKey, "HmacSHA1"));
byte[] hash = mac.doFinal(id.getBytes("UTF-8"));
- id = id + ':' + encodeHexString(hash);
+ String calcRef = id + ':' + encodeHexString(hash);
+
+ assertEquals("getReference() not equal", calcRef, ref);
+
+ DataRecord refRec = s3.getMetadataRecord("reference.key");
+ assertNotNull("Reference data record null", refRec);
- assertEquals(id, ref);
+ byte[] refDirectFromBackend = IOUtils.toByteArray(refRec.getStream());
+ LOG.warn("Ref direct from backend {}", refDirectFromBackend);
+ assertTrue("refKey in memory not equal to the metadata record",
+ Arrays.equals(refKey, refDirectFromBackend));
}
@Test
@@ -139,7 +144,7 @@ public class TestS3DataStore {
Random randomGen = new Random();
props = S3DataStoreUtils.getS3Config();
//Replace bucket in props with container
- String bucket = props.getProperty(S3Constants.S3_BUCKET);
+ bucket = props.getProperty(S3Constants.S3_BUCKET);
props.remove(S3Constants.S3_BUCKET);
props.put(S3Constants.S3_CONTAINER, bucket);
@@ -149,4 +154,14 @@ public class TestS3DataStore {
DataRecord rec = ds.addRecord(new ByteArrayInputStream(data));
assertEquals(data.length, rec.getLength());
}
+
+ @After
+ public void tearDown() {
+ try {
+ if (bucket != null) {
+ S3DataStoreUtils.deleteBucket(bucket, startTime);
+ }
+ } catch (Exception ignore) {
+ }
+ }
}