You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2022/07/20 06:46:27 UTC
[jackrabbit-oak] 01/01: OAK-9807: On DataStore initialization create the reference.key if absent
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch OAK-9807
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit f640551c26908142d4d4dde35ca77fd7a86fbd16
Author: amjain <am...@adobe.com>
AuthorDate: Wed Jul 20 12:13:10 2022 +0530
OAK-9807: On DataStore initialization create the reference.key if absent
Create reference.key on init if not present
---
.../azure/blobstorage/AzureBlobStoreBackend.java | 8 +++++
.../cloud/azure/blobstorage/AzureConstants.java | 5 +++
.../blobstorage/AzureBlobStoreBackendTest.java | 25 ++++++++++++++
.../azure/blobstorage/AzureDataStoreTest.java | 14 +++++---
.../jackrabbit/oak/blob/cloud/s3/S3Backend.java | 5 ++-
.../oak/blob/cloud/s3/TestS3DataStore.java | 39 +++++++++++-----------
6 files changed, 70 insertions(+), 26 deletions(-)
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
index e7c0ba5b74..4d94d94105 100644
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
+++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
@@ -220,6 +220,14 @@ public class AzureBlobStoreBackend extends AbstractSharedBackend {
}
uploadDomainOverride = properties.getProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE, null);
downloadDomainOverride = properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE, null);
+
+ // Initialize reference key secret
+ boolean createRefSecretOnInit = PropertiesUtil.toBoolean(
+ Strings.emptyToNull(properties.getProperty(AzureConstants.AZURE_REF_ON_INIT)), true);
+
+ if (createRefSecretOnInit) {
+ getOrCreateReferenceKey();
+ }
}
catch (StorageException e) {
throw new DataStoreException(e);
diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java
index 0c3956063d..58cf6160c3 100644
--- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java
+++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java
@@ -119,5 +119,10 @@ public final class AzureConstants {
*/
public static final String PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE = "presignedHttpUploadURIDomainOverride";
+ /**
+ * Property to enable/disable creation of reference secret on init.
+ */
+ public static final String AZURE_REF_ON_INIT = "refOnInit";
+
private AzureConstants() { }
}
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
index 9cac0cbcaa..d37c6c7237 100644
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
+++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
@@ -32,6 +32,9 @@ import java.util.EnumSet;
import java.util.Properties;
import java.util.Set;
import java.util.stream.StreamSupport;
+
+import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.core.data.DataStoreException;
import org.jetbrains.annotations.NotNull;
import org.junit.After;
import org.junit.ClassRule;
@@ -44,6 +47,8 @@ import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.READ;
import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.WRITE;
import static java.util.stream.Collectors.toSet;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class AzureBlobStoreBackendTest {
@@ -129,6 +134,15 @@ public class AzureBlobStoreBackendTest {
assertReadAccessGranted(azureBlobStoreBackend, ImmutableSet.of("file"));
}
+ @Test
+ public void initSecret() throws Exception {
+ AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
+ azureBlobStoreBackend.setProperties(getConfigurationWithConnectionString());
+
+ azureBlobStoreBackend.init();
+ assertReferenceSecret(azureBlobStoreBackend);
+ }
+
private CloudBlobContainer createBlobContainer() throws Exception {
container = azurite.getContainer("blobstore");
for (String blob : BLOBS) {
@@ -141,6 +155,7 @@ public class AzureBlobStoreBackendTest {
Properties properties = getBasicConfiguration();
properties.setProperty(AzureConstants.AZURE_SAS, sasToken);
properties.setProperty(AzureConstants.AZURE_CREATE_CONTAINER, "false");
+ properties.setProperty(AzureConstants.AZURE_REF_ON_INIT, "false");
return properties;
}
@@ -185,7 +200,9 @@ public class AzureBlobStoreBackendTest {
Set<String> actualBlobNames = StreamSupport.stream(container.listBlobs().spliterator(), false)
.map(blob -> blob.getUri().getPath())
.map(path -> path.substring(path.lastIndexOf('/') + 1))
+ .filter(path -> !path.isEmpty())
.collect(toSet());
+
Set<String> expectedBlobNames = expectedBlobs.stream().map(name -> name + ".txt").collect(toSet());
assertEquals(expectedBlobNames, actualBlobNames);
@@ -236,4 +253,12 @@ public class AzureBlobStoreBackendTest {
private static String getConnectionString() {
return Utils.getConnectionString(AzuriteDockerRule.ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_KEY, azurite.getBlobEndpoint());
}
+
+ private static void assertReferenceSecret(AzureBlobStoreBackend azureBlobStoreBackend)
+ throws DataStoreException, IOException {
+ // assert secret already created on init
+ DataRecord refRec = azureBlobStoreBackend.getMetadataRecord("reference.key");
+ assertNotNull("Reference data record null", refRec);
+ assertTrue("reference key is empty", refRec.getLength() > 0);
+ }
}
\ No newline at end of file
diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreTest.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreTest.java
index c1fd8416c5..f5de8e6cb6 100644
--- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreTest.java
+++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreTest.java
@@ -589,7 +589,9 @@ public class AzureDataStoreTest {
@Test
public void testBackendGetAllMetadataRecordsPrefixMatchesAll() throws DataStoreException {
- assertEquals(0, backend.getAllMetadataRecords("").size());
+ // reference.key initialized in backend#init() - OAK-9807, so expected 1 record
+ assertEquals(1, backend.getAllMetadataRecords("").size());
+ backend.deleteAllMetadataRecords("");
String prefixAll = "prefix1";
String prefixSome = "prefix1.prefix2";
@@ -694,7 +696,8 @@ public class AzureDataStoreTest {
@Test
public void testBackendDeleteAllMetadataRecordsNoRecordsNoChange() {
- assertEquals(0, backend.getAllMetadataRecords("").size());
+ // reference.key initialized in backend#init() - OAK-9807, so expected 1 record
+ assertEquals(1, backend.getAllMetadataRecords("").size());
backend.deleteAllMetadataRecords("");
@@ -714,6 +717,10 @@ public class AzureDataStoreTest {
@Test
public void testSecret() throws Exception {
+ // assert secret already created on init
+ DataRecord refRec = ds.getMetadataRecord("reference.key");
+ assertNotNull("Reference data record null", refRec);
+
byte[] data = new byte[4096];
randomGen.nextBytes(data);
DataRecord rec = ds.addRecord(new ByteArrayInputStream(data));
@@ -732,9 +739,6 @@ public class AzureDataStoreTest {
assertEquals("getReference() not equal", calcRef, ref);
- DataRecord refRec = ds.getMetadataRecord("reference.key");
- assertNotNull("Reference data record null", refRec);
-
byte[] refDirectFromBackend = IOUtils.toByteArray(refRec.getStream());
LOG.warn("Ref direct from backend {}", refDirectFromBackend);
assertTrue("refKey in memory not equal to the metadata record",
diff --git a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
index 0da9adf9cc..3b17c3c694 100644
--- a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
+++ b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
@@ -237,7 +237,10 @@ public class S3Backend extends AbstractSharedBackend {
presignedDownloadURIVerifyExists =
PropertiesUtil.toBoolean(properties.get(S3Constants.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS), true);
-
+
+ // Initialize reference key secret
+ getOrCreateReferenceKey();
+
LOG.debug("S3 Backend initialized in [{}] ms",
+(System.currentTimeMillis() - startTime.getTime()));
} catch (Exception e) {
diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java
index b6b54d88b9..f1e041589d 100644
--- a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java
+++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java
@@ -118,38 +118,34 @@ public class TestS3DataStore {
public void testSecret() throws Exception {
assumeTrue(isS3Configured());
+ S3DataStore s3 = getDataStore();
+ // get ref as record
+ DataRecord refRec = s3.getMetadataRecord("reference.key");
+ assertNotNull("Reference data record null", refRec);
+ byte[] refDirectFromBackend = IOUtils.toByteArray(refRec.getStream());
+ LOG.warn("Ref direct from backend {}", refDirectFromBackend);
+
+ // reference.key initialized in backend#init() - OAK-9807, so expected reference.key already created
+ byte[] refKey = ((S3Backend) s3.getBackend()).getOrCreateReferenceKey();
+ assertTrue("refKey in memory not equal to the metadata record",
+ Arrays.equals(refKey, refDirectFromBackend));
+
Random randomGen = new Random();
- props = S3DataStoreUtils.getS3Config();
- props.put("cacheSize", "0");
- ds = getS3DataStore(s3Class, props, dataStoreDir.getAbsolutePath());
- bucket = props.getProperty(S3Constants.S3_BUCKET);
-
byte[] data = new byte[4096];
randomGen.nextBytes(data);
- DataRecord rec = ds.addRecord(new ByteArrayInputStream(data));
+ DataRecord rec = s3.addRecord(new ByteArrayInputStream(data));
assertEquals(data.length, rec.getLength());
String ref = rec.getReference();
assertNotNull(ref);
String id = rec.getIdentifier().toString();
-
- S3DataStore s3 = ((S3DataStore) ds);
- byte[] refKey = ((S3Backend) s3.getBackend()).getOrCreateReferenceKey();
-
+
Mac mac = Mac.getInstance("HmacSHA1");
mac.init(new SecretKeySpec(refKey, "HmacSHA1"));
byte[] hash = mac.doFinal(id.getBytes("UTF-8"));
String calcRef = id + ':' + encodeHexString(hash);
assertEquals("getReference() not equal", calcRef, ref);
-
- DataRecord refRec = s3.getMetadataRecord("reference.key");
- assertNotNull("Reference data record null", refRec);
-
- byte[] refDirectFromBackend = IOUtils.toByteArray(refRec.getStream());
- LOG.warn("Ref direct from backend {}", refDirectFromBackend);
- assertTrue("refKey in memory not equal to the metadata record",
- Arrays.equals(refKey, refDirectFromBackend));
}
@Test
@@ -283,7 +279,9 @@ public class TestS3DataStore {
assumeTrue(isS3Configured());
S3DataStore s3ds = getDataStore();
- assertEquals(0, s3ds.getAllMetadataRecords("").size());
+ // reference.key initialized in backend#init() - OAK-9807, so expected 1 record
+ assertEquals(1, s3ds.getAllMetadataRecords("").size());
+ s3ds.deleteAllMetadataRecords("");
String prefixAll = "prefix1";
String prefixSome = "prefix1.prefix2";
@@ -402,7 +400,8 @@ public class TestS3DataStore {
assumeTrue(isS3Configured());
S3DataStore s3ds = getDataStore();
- assertEquals(0, s3ds.getAllMetadataRecords("").size());
+ // reference.key initialized in backend#init() - OAK-9807, so expected 1 record
+ assertEquals(1, s3ds.getAllMetadataRecords("").size());
s3ds.deleteAllMetadataRecords("");