You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ad...@apache.org on 2024/06/18 09:04:25 UTC
(jackrabbit-oak) branch issues/OAK-10898 created (now eb73debd55)
This is an automated email from the ASF dual-hosted git repository.
adulceanu pushed a change to branch issues/OAK-10898
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
at eb73debd55 OAK-10898 - Allow AzureCheck and AzureCompact to be built directly with a CloudBlobDirectory
This branch includes the following new commits:
new eb73debd55 OAK-10898 - Allow AzureCheck and AzureCompact to be built directly with a CloudBlobDirectory
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(jackrabbit-oak) 01/01: OAK-10898 - Allow AzureCheck and AzureCompact to be built directly with a CloudBlobDirectory
Posted by ad...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
adulceanu pushed a commit to branch issues/OAK-10898
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit eb73debd5539f0cccfe6541fb674b1af74d7cb08
Author: Andrei Dulceanu <du...@adobe.com>
AuthorDate: Mon Jun 17 14:50:43 2024 +0200
OAK-10898 - Allow AzureCheck and AzureCompact to be built directly with a CloudBlobDirectory
---
.../oak/segment/azure/tool/AzureCheck.java | 27 +++++++++++++--
.../oak/segment/azure/tool/AzureCompact.java | 39 ++++++++++++++++++----
.../oak/segment/azure/tool/ToolUtils.java | 13 +++++---
3 files changed, 65 insertions(+), 14 deletions(-)
diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
index 2452e8b12b..8d0bca571e 100644
--- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
+++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
@@ -17,6 +17,8 @@
package org.apache.jackrabbit.oak.segment.azure.tool;
import com.google.common.io.Files;
+import com.microsoft.azure.storage.blob.CloudBlobDirectory;
+import org.apache.jackrabbit.oak.segment.azure.AzurePersistence;
import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
import org.apache.jackrabbit.oak.segment.file.JournalReader;
import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore;
@@ -81,6 +83,8 @@ public class AzureCheck {
private Integer persistentCacheSizeGb;
+ private CloudBlobDirectory cloudBlobDirectory;
+
private Builder() {
// Prevent external instantiation.
}
@@ -270,6 +274,17 @@ public class AzureCheck {
return this;
}
+ /**
+ * The Azure blob directory to connect to.
+ * @param cloudBlobDirectory
+ * the Azure blob directory.
+ * @return this builder
+ */
+ public Builder withCloudBlobDirectory(CloudBlobDirectory cloudBlobDirectory) {
+ this.cloudBlobDirectory = checkNotNull(cloudBlobDirectory);
+ return this;
+ }
+
/**
* Create an executable version of the {@link Check} command.
*
@@ -329,6 +344,8 @@ public class AzureCheck {
private final Integer persistentCacheSizeGb;
+ private final CloudBlobDirectory cloudBlobDirectory;
+
private AzureCheck(Builder builder) {
this.path = builder.path;
this.debugInterval = builder.debugInterval;
@@ -345,6 +362,7 @@ public class AzureCheck {
this.failFast = builder.failFast;
this.persistentCachePath = builder.persistentCachePath;
this.persistentCacheSizeGb = builder.persistentCacheSizeGb;
+ this.cloudBlobDirectory = builder.cloudBlobDirectory;
}
private static Integer revisionsToCheckCount(Integer revisionsCount) {
@@ -354,12 +372,17 @@ public class AzureCheck {
public int run() {
StatisticsIOMonitor ioMonitor = new StatisticsIOMonitor();
SegmentNodeStorePersistence persistence;
- if (persistentCachePath != null) {
- persistence = ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, path, persistentCachePath, persistentCacheSizeGb);
+
+ if (cloudBlobDirectory != null) {
+ persistence = new AzurePersistence(cloudBlobDirectory);
} else {
persistence = ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, path);
}
+ if (persistentCachePath != null) {
+ persistence = ToolUtils.decorateWithCache(persistence, persistentCachePath, persistentCacheSizeGb);
+ }
+
FileStoreBuilder builder = fileStoreBuilder(Files.createTempDir()).withCustomPersistence(persistence);
if (ioStatistics) {
diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
index d4942f63f1..ab18f4f282 100644
--- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
+++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
@@ -19,11 +19,7 @@ package org.apache.jackrabbit.oak.segment.azure.tool;
import static org.apache.jackrabbit.guava.common.base.Preconditions.checkArgument;
import static org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull;
-import static org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.createArchiveManager;
-import static org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.createCloudBlobDirectory;
-import static org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.newFileStore;
-import static org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.newSegmentNodeStorePersistence;
-import static org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.printableStopwatch;
+import static org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.*;
import org.apache.jackrabbit.guava.common.base.Stopwatch;
import org.apache.jackrabbit.guava.common.io.Files;
@@ -35,6 +31,7 @@ import com.microsoft.azure.storage.blob.CloudBlobDirectory;
import com.microsoft.azure.storage.blob.ListBlobItem;
import org.apache.jackrabbit.oak.segment.SegmentCache;
+import org.apache.jackrabbit.oak.segment.azure.AzurePersistence;
import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.SegmentStoreType;
import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.GCType;
import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType;
@@ -90,6 +87,10 @@ public class AzureCompact {
private Integer persistentCacheSizeGb;
+ private CloudBlobDirectory sourceCloudBlobDirectory;
+
+ private CloudBlobDirectory destinationCloudBlobDirectory;
+
private Builder() {
// Prevent external instantiation.
}
@@ -217,6 +218,16 @@ public class AzureCompact {
return this;
}
+ public Builder withSourceCloudBlobDirectory(CloudBlobDirectory sourceCloudBlobDirectory) {
+ this.sourceCloudBlobDirectory = checkNotNull(sourceCloudBlobDirectory);
+ return this;
+ }
+
+ public Builder withDestinationCloudBlobDirectory(CloudBlobDirectory destinationCloudBlobDirectory) {
+ this.destinationCloudBlobDirectory = checkNotNull(destinationCloudBlobDirectory);
+ return this;
+ }
+
/**
* Create an executable version of the {@link Compact} command.
*
@@ -248,6 +259,10 @@ public class AzureCompact {
private Integer persistentCacheSizeGb;
+ private CloudBlobDirectory sourceCloudBlobDirectory;
+
+ private CloudBlobDirectory destinationCloudBlobDirectory;
+
private AzureCompact(Builder builder) {
this.path = builder.path;
this.targetPath = builder.targetPath;
@@ -259,12 +274,22 @@ public class AzureCompact {
this.concurrency = builder.concurrency;
this.persistentCachePath = builder.persistentCachePath;
this.persistentCacheSizeGb = builder.persistentCacheSizeGb;
+ this.sourceCloudBlobDirectory = builder.sourceCloudBlobDirectory;
+ this.destinationCloudBlobDirectory = builder.destinationCloudBlobDirectory;
}
public int run() throws IOException, StorageException, URISyntaxException {
Stopwatch watch = Stopwatch.createStarted();
- SegmentNodeStorePersistence roPersistence = newSegmentNodeStorePersistence(SegmentStoreType.AZURE, path, persistentCachePath, persistentCacheSizeGb);
- SegmentNodeStorePersistence rwPersistence = newSegmentNodeStorePersistence(SegmentStoreType.AZURE, targetPath);
+
+ SegmentNodeStorePersistence roPersistence;
+ SegmentNodeStorePersistence rwPersistence;
+ if (sourceCloudBlobDirectory != null && destinationCloudBlobDirectory != null) {
+ roPersistence = decorateWithCache(new AzurePersistence(sourceCloudBlobDirectory), persistentCachePath, persistentCacheSizeGb);
+ rwPersistence = new AzurePersistence(destinationCloudBlobDirectory);
+ } else {
+ roPersistence = newSegmentNodeStorePersistence(SegmentStoreType.AZURE, path, persistentCachePath, persistentCacheSizeGb);
+ rwPersistence = newSegmentNodeStorePersistence(SegmentStoreType.AZURE, targetPath);
+ }
SegmentNodeStorePersistence splitPersistence = new SplitPersistence(roPersistence, rwPersistence);
diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
index 4e73a79578..f1845bc0a6 100644
--- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
+++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
@@ -134,11 +134,7 @@ public class ToolUtils {
switch (storeType) {
case AZURE:
CloudBlobDirectory cloudBlobDirectory = createCloudBlobDirectory(pathOrUri.substring(3));
- SegmentNodeStorePersistence basePersistence = new AzurePersistence(cloudBlobDirectory);
-
- PersistentCache persistentCache = new PersistentDiskCache(new File(persistentCachePath),
- persistentCacheSize * 1024, new DiskCacheIOMonitor(StatisticsProvider.NOOP));
- persistence = new CachingPersistence(persistentCache, basePersistence);
+ persistence = decorateWithCache(new AzurePersistence(cloudBlobDirectory), persistentCachePath, persistentCacheSize);
break;
default:
persistence = new TarPersistence(new File(pathOrUri));
@@ -147,6 +143,13 @@ public class ToolUtils {
return persistence;
}
+ public static SegmentNodeStorePersistence decorateWithCache(SegmentNodeStorePersistence persistence,
+ String persistentCachePath, Integer persistentCacheSize) {
+ PersistentCache persistentCache = new PersistentDiskCache(new File(persistentCachePath),
+ persistentCacheSize * 1024, new DiskCacheIOMonitor(StatisticsProvider.NOOP));
+ return new CachingPersistence(persistentCache, persistence);
+ }
+
public static SegmentNodeStorePersistence newSegmentNodeStorePersistence(SegmentStoreType storeType,
String pathOrUri) {
SegmentNodeStorePersistence persistence = null;