You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2022/07/20 06:46:26 UTC
[jackrabbit-oak] branch OAK-9807 created (now f640551c26)
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a change to branch OAK-9807
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
at f640551c26 OAK-9807: On DataStore initialization create the reference.key if absent
This branch includes the following new commits:
new f640551c26 OAK-9807: On DataStore initialization create the reference.key if absent
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[jackrabbit-oak] 01/01: OAK-9807: On DataStore initialization create the reference.key if absent
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch OAK-9807
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit f640551c26908142d4d4dde35ca77fd7a86fbd16
Author: amjain <am...@adobe.com>
AuthorDate: Wed Jul 20 12:13:10 2022 +0530
OAK-9807: On DataStore initialization create the reference.key if absent
Create reference.key on init if not present
---
.../azure/blobstorage/AzureBlobStoreBackend.java | 8 +++++
.../cloud/azure/blobstorage/AzureConstants.java | 5 +++
.../blobstorage/AzureBlobStoreBackendTest.java | 25 ++++++++++++++
.../azure/blobstorage/AzureDataStoreTest.java | 14 +++++---
.../jackrabbit/oak/blob/cloud/s3/S3Backend.java | 5 ++-
.../oak/blob/cloud/s3/TestS3DataStore.java | 39 +++++++++++-----------
6 files changed, 70 insertions(+), 26 deletions(-)
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
index e7c0ba5b74..4d94d94105 100644
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
+++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
@@ -220,6 +220,14 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
}
uploadDomainOverride = properties.getProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE, null);
downloadDomainOverride = properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE, null);
+
+ // Initialize reference key secret
+ boolean createRefSecretOnInit = PropertiesUtil.toBoolean(
+ Strings.emptyToNull(properties.getProperty(AzureConstants.AZURE_REF_ON_INIT)), true);
+
+ if (createRefSecretOnInit) {
+ getOrCreateReferenceKey();
+ }
}
catch (StorageException e) {
throw new DataStoreException(e);
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java
index 0c3956063d..58cf6160c3 100644
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java
+++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java
@@ -119,5 +119,10 @@ public final class AzureConstants {
*/
public static final String PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE = "presignedHttpUploadURIDomainOverride";
+ /**
+ * Property to enable/disable creation of reference secret on init.
+ */
+ public static final String AZURE_REF_ON_INIT = "refOnInit";
+
private AzureConstants() { }
}
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
index 9cac0cbcaa..d37c6c7237 100644
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
+++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
@@ -32,6 +32,9 @@ import java.util.EnumSet;
import java.util.Properties;
import java.util.Set;
import java.util.stream.StreamSupport;
+
+import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.core.data.DataStoreException;
import org.jetbrains.annotations.NotNull;
import org.junit.After;
import org.junit.ClassRule;
@@ -44,6 +47,8 @@ import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.READ;
import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.WRITE;
import static java.util.stream.Collectors.toSet;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class AzureBlobStoreBackendTest {
@@ -129,6 +134,15 @@ public class AzureBlobStoreBackendTest {
assertReadAccessGranted(azureBlobStoreBackend, ImmutableSet.of("file"));
}
+ @Test
+ public void initSecret() throws Exception {
+ AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
+ azureBlobStoreBackend.setProperties(getConfigurationWithConnectionString());
+
+ azureBlobStoreBackend.init();
+ assertReferenceSecret(azureBlobStoreBackend);
+ }
+
private CloudBlobContainer createBlobContainer() throws Exception {
container = azurite.getContainer("blobstore");
for (String blob : BLOBS) {
@@ -141,6 +155,7 @@ public class AzureBlobStoreBackendTest {
Properties properties = getBasicConfiguration();
properties.setProperty(AzureConstants.AZURE_SAS, sasToken);
properties.setProperty(AzureConstants.AZURE_CREATE_CONTAINER, "false");
+ properties.setProperty(AzureConstants.AZURE_REF_ON_INIT, "false");
return properties;
}
@@ -185,7 +200,9 @@ public class AzureBlobStoreBackendTest {
Set<String> actualBlobNames = StreamSupport.stream(container.listBlobs().spliterator(), false)
.map(blob -> blob.getUri().getPath())
.map(path -> path.substring(path.lastIndexOf('/') + 1))
+ .filter(path -> !path.isEmpty())
.collect(toSet());
+
Set<String> expectedBlobNames = expectedBlobs.stream().map(name -> name + ".txt").collect(toSet());
assertEquals(expectedBlobNames, actualBlobNames);
@@ -236,4 +253,12 @@ public class AzureBlobStoreBackendTest {
private static String getConnectionString() {
return Utils.getConnectionString(AzuriteDockerRule.ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_KEY, azurite.getBlobEndpoint());
}
+
+ private static void assertReferenceSecret(AzureBlobStoreBackend azureBlobStoreBackend)
+ throws DataStoreException, IOException {
+ // assert secret already created on init
+ DataRecord refRec = azureBlobStoreBackend.getMetadataRecord("reference.key");
+ assertNotNull("Reference data record null", refRec);
+ assertTrue("reference key is empty", refRec.getLength() > 0);
+ }
}
\ No newline at end of file
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreTest.java
index c1fd8416c5..f5de8e6cb6 100644
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreTest.java
+++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreTest.java
@@ -589,7 +589,9 @@ public class AzureDataStoreTest {
@Test
public void testBackendGetAllMetadataRecordsPrefixMatchesAll() throws DataStoreException {
- assertEquals(0, backend.getAllMetadataRecords("").size());
+ // reference.key initialized in backend#init() - OAK-9807, so expected 1 record
+ assertEquals(1, backend.getAllMetadataRecords("").size());
+ backend.deleteAllMetadataRecords("");
String prefixAll = "prefix1";
String prefixSome = "prefix1.prefix2";
@@ -694,7 +696,8 @@ public class AzureDataStoreTest {
@Test
public void testBackendDeleteAllMetadataRecordsNoRecordsNoChange() {
- assertEquals(0, backend.getAllMetadataRecords("").size());
+ // reference.key initialized in backend#init() - OAK-9807, so expected 1 record
+ assertEquals(1, backend.getAllMetadataRecords("").size());
backend.deleteAllMetadataRecords("");
@@ -714,6 +717,10 @@ public class AzureDataStoreTest {
@Test
public void testSecret() throws Exception {
+ // assert secret already created on init
+ DataRecord refRec = ds.getMetadataRecord("reference.key");
+ assertNotNull("Reference data record null", refRec);
+
byte[] data = new byte[4096];
randomGen.nextBytes(data);
DataRecord rec = ds.addRecord(new ByteArrayInputStream(data));
@@ -732,9 +739,6 @@ public class AzureDataStoreTest {
assertEquals("getReference() not equal", calcRef, ref);
- DataRecord refRec = ds.getMetadataRecord("reference.key");
- assertNotNull("Reference data record null", refRec);
-
byte[] refDirectFromBackend = IOUtils.toByteArray(refRec.getStream());
LOG.warn("Ref direct from backend {}", refDirectFromBackend);
assertTrue("refKey in memory not equal to the metadata record",
diff --git a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
index 0da9adf9cc..3b17c3c694 100644
--- a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
+++ b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
@@ -237,7 +237,10 @@ public class S3Backend extends AbstractSharedBackend {
presignedDownloadURIVerifyExists =
PropertiesUtil.toBoolean(properties.get(S3Constants.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS), true);
-
+
+ // Initialize reference key secret
+ getOrCreateReferenceKey();
+
LOG.debug("S3 Backend initialized in [{}] ms",
+(System.currentTimeMillis() - startTime.getTime()));
} catch (Exception e) {
diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java
index b6b54d88b9..f1e041589d 100644
--- a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java
+++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java
@@ -118,38 +118,34 @@ public class TestS3DataStore {
public void testSecret() throws Exception {
assumeTrue(isS3Configured());
+ S3DataStore s3 = getDataStore();
+ // get ref as record
+ DataRecord refRec = s3.getMetadataRecord("reference.key");
+ assertNotNull("Reference data record null", refRec);
+ byte[] refDirectFromBackend = IOUtils.toByteArray(refRec.getStream());
+ LOG.warn("Ref direct from backend {}", refDirectFromBackend);
+
+ // reference.key initialized in backend#init() - OAK-9807, so expected reference.key already created
+ byte[] refKey = ((S3Backend) s3.getBackend()).getOrCreateReferenceKey();
+ assertTrue("refKey in memory not equal to the metadata record",
+ Arrays.equals(refKey, refDirectFromBackend));
+
Random randomGen = new Random();
- props = S3DataStoreUtils.getS3Config();
- props.put("cacheSize", "0");
- ds = getS3DataStore(s3Class, props, dataStoreDir.getAbsolutePath());
- bucket = props.getProperty(S3Constants.S3_BUCKET);
-
byte[] data = new byte[4096];
randomGen.nextBytes(data);
- DataRecord rec = ds.addRecord(new ByteArrayInputStream(data));
+ DataRecord rec = s3.addRecord(new ByteArrayInputStream(data));
assertEquals(data.length, rec.getLength());
String ref = rec.getReference();
assertNotNull(ref);
String id = rec.getIdentifier().toString();
-
- S3DataStore s3 = ((S3DataStore) ds);
- byte[] refKey = ((S3Backend) s3.getBackend()).getOrCreateReferenceKey();
-
+
Mac mac = Mac.getInstance("HmacSHA1");
mac.init(new SecretKeySpec(refKey, "HmacSHA1"));
byte[] hash = mac.doFinal(id.getBytes("UTF-8"));
String calcRef = id + ':' + encodeHexString(hash);
assertEquals("getReference() not equal", calcRef, ref);
-
- DataRecord refRec = s3.getMetadataRecord("reference.key");
- assertNotNull("Reference data record null", refRec);
-
- byte[] refDirectFromBackend = IOUtils.toByteArray(refRec.getStream());
- LOG.warn("Ref direct from backend {}", refDirectFromBackend);
- assertTrue("refKey in memory not equal to the metadata record",
- Arrays.equals(refKey, refDirectFromBackend));
}
@Test
@@ -283,7 +279,9 @@ public class TestS3DataStore {
assumeTrue(isS3Configured());
S3DataStore s3ds = getDataStore();
- assertEquals(0, s3ds.getAllMetadataRecords("").size());
+ // reference.key initialized in backend#init() - OAK-9807, so expected 1 record
+ assertEquals(1, s3ds.getAllMetadataRecords("").size());
+ s3ds.deleteAllMetadataRecords("");
String prefixAll = "prefix1";
String prefixSome = "prefix1.prefix2";
@@ -402,7 +400,8 @@ public class TestS3DataStore {
assumeTrue(isS3Configured());
S3DataStore s3ds = getDataStore();
- assertEquals(0, s3ds.getAllMetadataRecords("").size());
+ // reference.key initialized in backend#init() - OAK-9807, so expected 1 record
+ assertEquals(1, s3ds.getAllMetadataRecords("").size());
s3ds.deleteAllMetadataRecords("");