You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ad...@apache.org on 2024/05/30 09:30:27 UTC
(jackrabbit-oak) branch trunk updated: OAK-10807 - Add oak-run diff to Azure repositories (#1496)
This is an automated email from the ASF dual-hosted git repository.
adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 53fd887f88 OAK-10807 - Add oak-run diff to Azure repositories (#1496)
53fd887f88 is described below
commit 53fd887f886a96fb3b6122d10f5cab0b3c8d6683
Author: Andrei Dulceanu <du...@users.noreply.github.com>
AuthorDate: Thu May 30 11:30:22 2024 +0200
OAK-10807 - Add oak-run diff to Azure repositories (#1496)
---
.../jackrabbit/oak/run/FileStoreDiffCommand.java | 67 ++++++---
.../oak/segment/azure/tool/ToolUtils.java | 33 ++++-
.../apache/jackrabbit/oak/segment/tool/Diff.java | 150 ++++++++++++---------
.../jackrabbit/oak/segment/tool/Revisions.java | 20 +--
.../apache/jackrabbit/oak/segment/tool/Utils.java | 10 +-
5 files changed, 186 insertions(+), 94 deletions(-)
diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java
index 5b0871b216..3bc7d27d0c 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java
@@ -18,13 +18,19 @@
package org.apache.jackrabbit.oak.run;
import static java.util.Arrays.asList;
+import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder;
+import static org.apache.jackrabbit.oak.segment.tool.Utils.newBasicReadOnlyBlobStore;
import java.io.File;
+import com.google.common.io.Files;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;
import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils;
+import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore;
+import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence;
import org.apache.jackrabbit.oak.segment.tool.Diff;
import org.apache.jackrabbit.oak.segment.tool.Revisions;
@@ -34,7 +40,7 @@ class FileStoreDiffCommand implements Command {
public void execute(String... args) throws Exception {
OptionParser parser = new OptionParser();
OptionSpec<?> help = parser.acceptsAll(asList("h", "?", "help"), "show help").forHelp();
- OptionSpec<File> storeO = parser.nonOptions("Path to segment store (required)").ofType(File.class);
+ OptionSpec<String> pathOrURI0 = parser.nonOptions("Path/URI to segment store (required)").ofType(String.class);
OptionSpec<File> outO = parser.accepts("output", "Output file").withRequiredArg().ofType(File.class).defaultsTo(defaultOutFile());
OptionSpec<?> listOnlyO = parser.accepts("list", "Lists available revisions");
OptionSpec<String> intervalO = parser.accepts("diff", "Revision diff interval. Ex '--diff=R0..R1'. 'HEAD' can be used to reference the latest head revision, ie. '--diff=R0..HEAD'").withRequiredArg().ofType(String.class);
@@ -48,9 +54,9 @@ class FileStoreDiffCommand implements Command {
System.exit(0);
}
- File store = storeO.value(options);
+ String pathOrURI = pathOrURI0.value(options);
- if (store == null) {
+ if (pathOrURI == null) {
parser.printHelpOn(System.out);
System.exit(1);
}
@@ -65,21 +71,48 @@ class FileStoreDiffCommand implements Command {
int statusCode;
if (listOnly) {
- statusCode = Revisions.builder()
- .withPath(store)
- .withOutput(out)
- .build()
- .run();
+ if (pathOrURI.startsWith("az:")) {
+ statusCode = Revisions.builder()
+ .withPath(pathOrURI)
+ .withOutput(out)
+ .build()
+ .run(ToolUtils::readRevisions);
+ } else {
+ statusCode = Revisions.builder()
+ .withPath(pathOrURI)
+ .withOutput(out)
+ .build()
+ .run(org.apache.jackrabbit.oak.segment.tool.Utils::readRevisions);
+ }
} else {
- statusCode = Diff.builder()
- .withPath(store)
- .withOutput(out)
- .withInterval(interval)
- .withIncremental(incremental)
- .withFilter(path)
- .withIgnoreMissingSegments(ignoreSNFEs)
- .build()
- .run();
+ if (pathOrURI.startsWith("az:")) {
+ SegmentNodeStorePersistence azurePersistence = ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, pathOrURI);
+ ReadOnlyFileStore store = fileStoreBuilder(Files.createTempDir()).withCustomPersistence(azurePersistence).withBlobStore(newBasicReadOnlyBlobStore()).buildReadOnly();
+ statusCode = Diff.builder()
+ .withPath(pathOrURI)
+ .withReadOnlyFileStore(store)
+ .withOutput(out)
+ .withInterval(interval)
+ .withIncremental(incremental)
+ .withFilter(path)
+ .withIgnoreMissingSegments(ignoreSNFEs)
+ .withRevisionsProcessor(ToolUtils::readRevisions)
+ .build()
+ .run();
+ } else {
+ ReadOnlyFileStore store = fileStoreBuilder(new File(pathOrURI)).withBlobStore(newBasicReadOnlyBlobStore()).buildReadOnly();
+ statusCode = Diff.builder()
+ .withPath(pathOrURI)
+ .withReadOnlyFileStore(store)
+ .withOutput(out)
+ .withInterval(interval)
+ .withIncremental(incremental)
+ .withFilter(path)
+ .withIgnoreMissingSegments(ignoreSNFEs)
+ .withRevisionsProcessor(org.apache.jackrabbit.oak.segment.tool.Utils::readRevisions)
+ .build()
+ .run();
+ }
}
System.exit(statusCode);
}
diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
index e71097db21..4e73a79578 100644
--- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
+++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
@@ -18,6 +18,7 @@
*/
package org.apache.jackrabbit.oak.segment.azure.tool;
+import static org.apache.jackrabbit.guava.common.collect.Lists.newArrayList;
import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_ID;
import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_SECRET;
import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_SECRET_KEY;
@@ -35,6 +36,8 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.net.URISyntaxException;
import java.text.MessageFormat;
+import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@@ -44,21 +47,23 @@ import org.apache.jackrabbit.oak.segment.azure.AzurePersistence;
import org.apache.jackrabbit.oak.segment.azure.AzureUtilities;
import org.apache.jackrabbit.oak.segment.azure.util.Environment;
import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType;
-import org.apache.jackrabbit.oak.segment.file.FileStore;
-import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
-import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
+import org.apache.jackrabbit.oak.segment.file.*;
import org.apache.jackrabbit.oak.segment.file.tar.TarPersistence;
import org.apache.jackrabbit.oak.segment.remote.persistentcache.DiskCacheIOMonitor;
import org.apache.jackrabbit.oak.segment.remote.persistentcache.PersistentDiskCache;
import org.apache.jackrabbit.oak.segment.spi.monitor.FileStoreMonitorAdapter;
import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitorAdapter;
import org.apache.jackrabbit.oak.segment.spi.monitor.RemoteStoreMonitorAdapter;
+import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile;
import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveManager;
import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence;
import org.apache.jackrabbit.oak.segment.spi.persistence.persistentcache.CachingPersistence;
import org.apache.jackrabbit.oak.segment.spi.persistence.persistentcache.PersistentCache;
import org.apache.jackrabbit.guava.common.base.Stopwatch;
+import org.apache.jackrabbit.guava.common.base.Function;
+import org.apache.jackrabbit.guava.common.collect.Iterators;
+
import com.microsoft.azure.storage.StorageCredentials;
import com.microsoft.azure.storage.StorageCredentialsAccountAndKey;
import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature;
@@ -198,6 +203,28 @@ public class ToolUtils {
}
}
+ public static List<String> readRevisions(String uri) {
+ SegmentNodeStorePersistence persistence = newSegmentNodeStorePersistence(SegmentStoreType.AZURE, uri);
+ JournalFile journal = persistence.getJournalFile();
+
+ if (journal.exists()) {
+ try (JournalReader journalReader = new JournalReader(journal)) {
+ Iterator<String> revisionIterator = Iterators.transform(journalReader, new Function<JournalEntry, String>() {
+ @NotNull
+ @Override
+ public String apply(JournalEntry entry) {
+ return entry.getRevision();
+ }
+ });
+ return newArrayList(revisionIterator);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ return newArrayList();
+ }
+
@NotNull
public static StorageCredentials getStorageCredentialsFromAccountAndEnv(@NotNull String accountName) {
return getStorageCredentialsFromAccountAndEnv(accountName, ENVIRONMENT);
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Diff.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Diff.java
index 70cc2bae61..b2a929b309 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Diff.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Diff.java
@@ -21,9 +21,6 @@ import static org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull
import static org.apache.jackrabbit.guava.common.collect.Lists.reverse;
import static org.apache.jackrabbit.oak.commons.PathUtils.elements;
import static org.apache.jackrabbit.oak.segment.RecordId.fromString;
-import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder;
-import static org.apache.jackrabbit.oak.segment.tool.Utils.newBasicReadOnlyBlobStore;
-import static org.apache.jackrabbit.oak.segment.tool.Utils.readRevisions;
import java.io.File;
import java.io.PrintWriter;
@@ -55,7 +52,9 @@ public class Diff {
*/
public static class Builder {
- private File path;
+ private String path;
+
+ private ReadOnlyFileStore store;
private String interval;
@@ -67,6 +66,8 @@ public class Diff {
private boolean ignoreMissingSegments;
+ private Revisions.RevisionsProcessor processor;
+
private Builder() {
// Prevent external instantiation.
}
@@ -77,11 +78,20 @@ public class Diff {
* @param path the path to an existing segment store.
* @return this builder.
*/
- public Builder withPath(File path) {
+ public Builder withPath(String path) {
this.path = checkNotNull(path);
return this;
}
+ /**
+ * The read-only filestore at path
+ * @param store the read-only filestore to diff
+ * @return this builder.
+ */
+ public Builder withReadOnlyFileStore(ReadOnlyFileStore store) {
+ this.store = checkNotNull(store);
+ return this;
+ }
/**
* The two node records to diff specified as a record ID interval. This
* parameter is required.
@@ -155,6 +165,16 @@ public class Diff {
return this;
}
+ /**
+ * The revisions processor used to list revisions
+ * @param processor a revisions processor
+ * @return this builder.
+ */
+ public Builder withRevisionsProcessor(Revisions.RevisionsProcessor processor) {
+ this.processor = processor;
+ return this;
+ }
+
/**
* Create an executable version of the {@link Diff} command.
*
@@ -170,7 +190,7 @@ public class Diff {
}
- private final File path;
+ private final String path;
private final String interval;
@@ -182,13 +202,19 @@ public class Diff {
private final boolean ignoreMissingSegments;
+ private final ReadOnlyFileStore store;
+
+ private final Revisions.RevisionsProcessor processor;
+
private Diff(Builder builder) {
this.path = builder.path;
+ this.store = builder.store;
this.interval = builder.interval;
this.incremental = builder.incremental;
this.out = builder.out;
this.filter = builder.filter;
this.ignoreMissingSegments = builder.ignoreMissingSegments;
+ this.processor = builder.processor;
}
public int run() {
@@ -212,75 +238,73 @@ public class Diff {
return;
}
- try (ReadOnlyFileStore store = fileStoreBuilder(path).withBlobStore(newBasicReadOnlyBlobStore()).buildReadOnly()) {
- SegmentIdProvider idProvider = store.getSegmentIdProvider();
- RecordId idL;
+ SegmentIdProvider idProvider = store.getSegmentIdProvider();
+ RecordId idL;
- try {
- if (tokens[0].equalsIgnoreCase("head")) {
- idL = store.getRevisions().getHead();
- } else {
- idL = fromString(idProvider, tokens[0]);
- }
- } catch (IllegalArgumentException e) {
- System.out.println("Invalid left endpoint for interval " + interval);
- return;
+ try {
+ if (tokens[0].equalsIgnoreCase("head")) {
+ idL = store.getRevisions().getHead();
+ } else {
+ idL = fromString(idProvider, tokens[0]);
}
+ } catch (IllegalArgumentException e) {
+ System.out.println("Invalid left endpoint for interval " + interval);
+ return;
+ }
- RecordId idR;
+ RecordId idR;
- try {
- if (tokens[1].equalsIgnoreCase("head")) {
- idR = store.getRevisions().getHead();
- } else {
- idR = fromString(idProvider, tokens[1]);
- }
- } catch (IllegalArgumentException e) {
- System.out.println("Invalid left endpoint for interval " + interval);
- return;
+ try {
+ if (tokens[1].equalsIgnoreCase("head")) {
+ idR = store.getRevisions().getHead();
+ } else {
+ idR = fromString(idProvider, tokens[1]);
}
+ } catch (IllegalArgumentException e) {
+ System.out.println("Invalid left endpoint for interval " + interval);
+ return;
+ }
- long start = System.currentTimeMillis();
+ long start = System.currentTimeMillis();
- try (PrintWriter pw = new PrintWriter(out)) {
- if (incremental) {
- List<String> revs = readRevisions(path);
- System.out.println("Generating diff between " + idL + " and " + idR + " incrementally. Found " + revs.size() + " revisions.");
+ try (PrintWriter pw = new PrintWriter(out)) {
+ if (incremental) {
+ List<String> revs = processor.process(path);
+ System.out.println("Generating diff between " + idL + " and " + idR + " incrementally. Found " + revs.size() + " revisions.");
- int s = revs.indexOf(idL.toString10());
- int e = revs.indexOf(idR.toString10());
- if (s == -1 || e == -1) {
- System.out.println("Unable to match input revisions with FileStore.");
- return;
- }
- List<String> revDiffs = revs.subList(Math.min(s, e), Math.max(s, e) + 1);
- if (s > e) {
- // reverse list
- revDiffs = reverse(revDiffs);
- }
- if (revDiffs.size() < 2) {
- System.out.println("Nothing to diff: " + revDiffs);
- return;
- }
- Iterator<String> revDiffsIt = revDiffs.iterator();
- RecordId idLt = fromString(idProvider, revDiffsIt.next());
- while (revDiffsIt.hasNext()) {
- RecordId idRt = fromString(idProvider, revDiffsIt.next());
- boolean good = diff(store, idLt, idRt, pw);
- idLt = idRt;
- if (!good && !ignoreMissingSegments) {
- break;
- }
+ int s = revs.indexOf(idL.toString10());
+ int e = revs.indexOf(idR.toString10());
+ if (s == -1 || e == -1) {
+ System.out.println("Unable to match input revisions with FileStore.");
+ return;
+ }
+ List<String> revDiffs = revs.subList(Math.min(s, e), Math.max(s, e) + 1);
+ if (s > e) {
+ // reverse list
+ revDiffs = reverse(revDiffs);
+ }
+ if (revDiffs.size() < 2) {
+ System.out.println("Nothing to diff: " + revDiffs);
+ return;
+ }
+ Iterator<String> revDiffsIt = revDiffs.iterator();
+ RecordId idLt = fromString(idProvider, revDiffsIt.next());
+ while (revDiffsIt.hasNext()) {
+ RecordId idRt = fromString(idProvider, revDiffsIt.next());
+ boolean good = diff(store, idLt, idRt, pw);
+ idLt = idRt;
+ if (!good && !ignoreMissingSegments) {
+ break;
}
- } else {
- System.out.println("Generating diff between " + idL + " and " + idR);
- diff(store, idL, idR, pw);
}
+ } else {
+ System.out.println("Generating diff between " + idL + " and " + idR);
+ diff(store, idL, idR, pw);
}
-
- long dur = System.currentTimeMillis() - start;
- System.out.println("Finished in " + dur + " ms.");
}
+
+ long dur = System.currentTimeMillis() - start;
+ System.out.println("Finished in " + dur + " ms.");
}
private boolean diff(ReadOnlyFileStore store, RecordId idL, RecordId idR, PrintWriter pw) {
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Revisions.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Revisions.java
index 1a02ec2f96..61f29b91ad 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Revisions.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Revisions.java
@@ -18,7 +18,6 @@
package org.apache.jackrabbit.oak.segment.tool;
import static org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull;
-import static org.apache.jackrabbit.oak.segment.tool.Utils.readRevisions;
import java.io.File;
import java.io.IOException;
@@ -44,7 +43,7 @@ public class Revisions {
*/
public static class Builder {
- private File path;
+ private String path;
private File out;
@@ -58,7 +57,7 @@ public class Revisions {
* @param path the path to an existing segment store.
* @return this builder.
*/
- public Builder withPath(File path) {
+ public Builder withPath(String path) {
this.path = checkNotNull(path);
return this;
}
@@ -88,7 +87,7 @@ public class Revisions {
}
- private final File path;
+ private final String path;
private final File out;
@@ -97,9 +96,9 @@ public class Revisions {
this.out = builder.out;
}
- public int run() {
+ public int run(RevisionsProcessor p) {
try {
- listRevisions();
+ listRevisions(p);
return 0;
} catch (Exception e) {
e.printStackTrace(System.err);
@@ -107,11 +106,11 @@ public class Revisions {
}
}
- private void listRevisions() throws IOException {
+ private void listRevisions(RevisionsProcessor p) throws IOException {
System.out.println("Store " + path);
System.out.println("Writing revisions to " + out);
- List<String> revs = readRevisions(path);
+ List<String> revs = p.process(path);
if (revs.isEmpty()) {
System.out.println("No revisions found.");
@@ -125,4 +124,9 @@ public class Revisions {
}
}
+ @FunctionalInterface
+ public interface RevisionsProcessor {
+ List<String> process(String path);
+ }
+
}
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Utils.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Utils.java
index a14475f9a0..06ea7b87f2 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Utils.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Utils.java
@@ -41,7 +41,7 @@ import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.jetbrains.annotations.NotNull;
-final class Utils {
+public final class Utils {
private static final boolean TAR_STORAGE_MEMORY_MAPPED = Boolean.getBoolean("tar.memoryMapped");
@@ -64,11 +64,15 @@ final class Utils {
.buildReadOnly();
}
- static BlobStore newBasicReadOnlyBlobStore() {
+ public static BlobStore newBasicReadOnlyBlobStore() {
return new BasicReadOnlyBlobStore();
}
- static List<String> readRevisions(File store) {
+ public static List<String> readRevisions(String path) {
+ return readRevisions(new File(path));
+ }
+
+ public static List<String> readRevisions(File store) {
JournalFile journal = new LocalJournalFile(store, "journal.log");
if (journal.exists()) {