You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ad...@apache.org on 2020/04/07 10:52:50 UTC
svn commit: r1876227 - in /jackrabbit/oak/trunk:
oak-doc/src/site/markdown/nodestore/segment/
oak-run/src/main/java/org/apache/jackrabbit/oak/run/
oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/
oak-segment-azure/src/test/...
Author: adulceanu
Date: Tue Apr 7 10:52:50 2020
New Revision: 1876227
URL: http://svn.apache.org/viewvc?rev=1876227&view=rev
Log:
OAK-8836 - oak-run compact should offer the possibility to choose between checkpoints compactor and the classical compactor
Modified:
jackrabbit/oak/trunk/oak-doc/src/site/markdown/nodestore/segment/overview.md
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
jackrabbit/oak/trunk/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Compact.java
Modified: jackrabbit/oak/trunk/oak-doc/src/site/markdown/nodestore/segment/overview.md
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-doc/src/site/markdown/nodestore/segment/overview.md?rev=1876227&r1=1876226&r2=1876227&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-doc/src/site/markdown/nodestore/segment/overview.md (original)
+++ jackrabbit/oak/trunk/oak-doc/src/site/markdown/nodestore/segment/overview.md Tue Apr 7 10:52:50 2020
@@ -799,7 +799,7 @@ This option is optional and is disabled
### <a name="compact"/> Compact
```
-java -jar oak-run.jar compact [--force] [--mmap] PATH | cloud-prefix:URI
+java -jar oak-run.jar compact [--force] [--mmap] [--compactor] PATH | cloud-prefix:URI
```
The `compact` command performs offline compaction of the local/remote Segment Store at `PATH`/`URI`.
@@ -816,6 +816,8 @@ to `true` for memory mapped access and `
mapped access is used on 64 bit systems and file access is used on 32 bit systems. On
Windows, regular file access is always enforced and this option is ignored.
+The optional `--compactor [String]` argument can be used to pick the compactor type to be used. Valid choices are *classic* and *diff*. While the former is slower, it might be more stable, due to lack of optimisations employed by the *diff* compactor which compacts the checkpoints on top of each other. If not specified, *diff* compactor is used.
+
To enable logging during offline compaction a Logback configuration file has to be injected
via the `logback.configurationFile` property. In addition the `compaction-progress-log`
property controls the number of compacted nodes that will be logged. The default value is 150000.
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java?rev=1876227&r1=1876226&r2=1876227&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java Tue Apr 7 10:52:50 2020
@@ -25,6 +25,8 @@ import joptsimple.OptionSet;
import joptsimple.OptionSpec;
import org.apache.jackrabbit.oak.run.commons.Command;
import org.apache.jackrabbit.oak.segment.azure.tool.AzureCompact;
+import org.apache.jackrabbit.oak.segment.azure.tool.AzureCompact.Builder;
+import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType;
import org.apache.jackrabbit.oak.segment.tool.Compact;
class CompactCommand implements Command {
@@ -52,6 +54,12 @@ class CompactCommand implements Command
"which is incompatible with older versions of Oak.")
.withOptionalArg()
.ofType(Boolean.class);
+ OptionSpec<String> compactor = parser.accepts("compactor",
+ "Allow the user to control compactor type to be used. Valid choices are \"classic\" and \"diff\". " +
+ "While the former is slower, it might be more stable, due to lack of optimisations employed " +
+ "by the \"diff\" compactor which compacts the checkpoints on top of each other. If not " +
+ "specified, \"diff\" compactor is used.")
+ .withRequiredArg().ofType(String.class);
OptionSet options = parser.parse(args);
String path = directoryArg.value(options);
@@ -65,21 +73,34 @@ class CompactCommand implements Command
int code = 0;
if (path.startsWith("az:")) {
- code = AzureCompact.builder()
+ Builder azureBuilder = AzureCompact.builder()
.withPath(path)
.withForce(isTrue(forceArg.value(options)))
.withSegmentCacheSize(Integer.getInteger("cache", 256))
- .withGCLogInterval(Long.getLong("compaction-progress-log", 150000))
+ .withGCLogInterval(Long.getLong("compaction-progress-log", 150000));
+
+ if (options.has(compactor)) {
+ azureBuilder.withCompactorType(CompactorType.fromDescription(compactor.value(options)));
+ }
+
+ code = azureBuilder
.build()
.run();
+
} else {
- code = Compact.builder()
+ org.apache.jackrabbit.oak.segment.tool.Compact.Builder tarBuilder = Compact.builder()
.withPath(new File(path))
.withForce(isTrue(forceArg.value(options)))
.withMmap(mmapArg.value(options))
.withOs(StandardSystemProperty.OS_NAME.value())
.withSegmentCacheSize(Integer.getInteger("cache", 256))
- .withGCLogInterval(Long.getLong("compaction-progress-log", 150000))
+ .withGCLogInterval(Long.getLong("compaction-progress-log", 150000));
+
+ if (options.has(compactor)) {
+ tarBuilder.withCompactorType(CompactorType.fromDescription(compactor.value(options)));
+ }
+
+ code = tarBuilder
.build()
.run();
}
Modified: jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java?rev=1876227&r1=1876226&r2=1876227&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java (original)
+++ jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java Tue Apr 7 10:52:50 2020
@@ -30,6 +30,7 @@ import com.google.common.io.Files;
import org.apache.jackrabbit.oak.segment.SegmentCache;
import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.SegmentStoreType;
+import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.segment.file.JournalReader;
import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile;
@@ -70,6 +71,8 @@ public class AzureCompact {
private int segmentCacheSize = DEFAULT_SEGMENT_CACHE_MB;
+ private CompactorType compactorType = CompactorType.CHECKPOINT_COMPACTOR;
+
private Builder() {
// Prevent external instantiation.
}
@@ -131,6 +134,17 @@ public class AzureCompact {
}
/**
+ * The compactor type to be used by compaction. If not specified it defaults to
+ * "diff" compactor
+ * @param compactorType the compactor type
+ * @return this builder
+ */
+ public Builder withCompactorType(CompactorType compactorType) {
+ this.compactorType = compactorType;
+ return this;
+ }
+
+ /**
* Create an executable version of the {@link Compact} command.
*
* @return an instance of {@link Runnable}.
@@ -149,11 +163,14 @@ public class AzureCompact {
private final long gcLogInterval;
+ private final CompactorType compactorType;
+
private AzureCompact(Builder builder) {
this.path = builder.path;
this.segmentCacheSize = builder.segmentCacheSize;
this.strictVersionCheck = !builder.force;
this.gcLogInterval = builder.gcLogInterval;
+ this.compactorType = builder.compactorType;
}
public int run() {
@@ -174,7 +191,7 @@ public class AzureCompact {
System.out.printf(" -> compacting\n");
try (FileStore store = newFileStore(persistence, Files.createTempDir(), strictVersionCheck, segmentCacheSize,
- gcLogInterval)) {
+ gcLogInterval, compactorType)) {
if (!store.compactFull()) {
System.out.printf("Compaction cancelled after %s.\n", printableStopwatch(watch));
return 1;
Modified: jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java?rev=1876227&r1=1876226&r2=1876227&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java (original)
+++ jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java Tue Apr 7 10:52:50 2020
@@ -41,6 +41,7 @@ import com.microsoft.azure.storage.blob.
import org.apache.jackrabbit.oak.commons.Buffer;
import org.apache.jackrabbit.oak.segment.azure.AzurePersistence;
import org.apache.jackrabbit.oak.segment.azure.AzureUtilities;
+import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
@@ -80,12 +81,12 @@ public class ToolUtils {
}
public static FileStore newFileStore(SegmentNodeStorePersistence persistence, File directory,
- boolean strictVersionCheck, int segmentCacheSize, long gcLogInterval)
+ boolean strictVersionCheck, int segmentCacheSize, long gcLogInterval, CompactorType compactorType)
throws IOException, InvalidFileStoreVersionException, URISyntaxException, StorageException {
FileStoreBuilder builder = FileStoreBuilder.fileStoreBuilder(directory)
.withCustomPersistence(persistence).withMemoryMapping(false).withStrictVersionCheck(strictVersionCheck)
.withSegmentCacheSize(segmentCacheSize)
- .withGCOptions(defaultGCOptions().setOffline().setGCLogInterval(gcLogInterval));
+ .withGCOptions(defaultGCOptions().setOffline().setGCLogInterval(gcLogInterval).setCompactorType(compactorType));
return builder.build();
}
Modified: jackrabbit/oak/trunk/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java?rev=1876227&r1=1876226&r2=1876227&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java (original)
+++ jackrabbit/oak/trunk/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java Tue Apr 7 10:52:50 2020
@@ -37,6 +37,7 @@ import org.apache.jackrabbit.oak.segment
import org.apache.jackrabbit.oak.segment.azure.AzuriteDockerRule;
import org.apache.jackrabbit.oak.segment.azure.tool.SegmentCopy;
import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.SegmentStoreType;
+import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.segment.spi.monitor.*;
import org.apache.jackrabbit.oak.segment.spi.persistence.GCJournalFile;
@@ -103,7 +104,7 @@ public abstract class SegmentCopyTestBas
// Repeatedly add content and close FileStore to obtain a new tar file each time
for (int i = 0; i < 10; i++) {
try (FileStore fileStore = newFileStore(srcPersistence, folder.getRoot(), true,
- SegmentCache.DEFAULT_SEGMENT_CACHE_MB, 150_000L)) {
+ SegmentCache.DEFAULT_SEGMENT_CACHE_MB, 150_000L, CompactorType.CHECKPOINT_COMPACTOR)) {
SegmentNodeStore sns = SegmentNodeStoreBuilders.builder(fileStore).build();
addContent(sns, i);
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java?rev=1876227&r1=1876226&r2=1876227&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java Tue Apr 7 10:52:50 2020
@@ -50,12 +50,33 @@ public class SegmentGCOptions {
/**
* Simple compactor implementation
*/
- CLASSIC_COMPACTOR,
+ CLASSIC_COMPACTOR("classic"),
/**
* Checkpoints aware compaction implementation
*/
- CHECKPOINT_COMPACTOR
+ CHECKPOINT_COMPACTOR("diff");
+
+ private final String description;
+
+ CompactorType(String description) {
+ this.description = description;
+ }
+
+ public static CompactorType fromDescription(String description) {
+ switch (description) {
+ case "classic":
+ return CLASSIC_COMPACTOR;
+ case "diff":
+ return CHECKPOINT_COMPACTOR;
+ default:
+ throw new IllegalArgumentException("Unrecongnized compactor type " + description);
+ }
+ }
+
+ public String description() {
+ return description;
+ }
}
/**
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Compact.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Compact.java?rev=1876227&r1=1876226&r2=1876227&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Compact.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Compact.java Tue Apr 7 10:52:50 2020
@@ -37,6 +37,7 @@ import java.util.concurrent.TimeUnit;
import com.google.common.base.Stopwatch;
import org.apache.jackrabbit.oak.segment.SegmentCache;
+import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType;
import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile;
import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFileWriter;
import org.apache.jackrabbit.oak.segment.file.FileStore;
@@ -77,6 +78,8 @@ public class Compact {
private int segmentCacheSize = DEFAULT_SEGMENT_CACHE_MB;
+ private CompactorType compactorType = CompactorType.CHECKPOINT_COMPACTOR;
+
private Builder() {
// Prevent external instantiation.
}
@@ -160,6 +163,17 @@ public class Compact {
}
/**
+ * The compactor type to be used by compaction. If not specified it defaults to
+ * "diff" compactor
+ * @param compactorType the compactor type
+ * @return this builder
+ */
+ public Builder withCompactorType(CompactorType compactorType) {
+ this.compactorType = compactorType;
+ return this;
+ }
+
+ /**
* Create an executable version of the {@link Compact} command.
*
* @return an instance of {@link Runnable}.
@@ -251,6 +265,8 @@ public class Compact {
private final long gcLogInterval;
+ private final CompactorType compactorType;
+
private Compact(Builder builder) {
this.path = builder.path;
this.journal = new File(builder.path, "journal.log");
@@ -258,10 +274,11 @@ public class Compact {
this.segmentCacheSize = builder.segmentCacheSize;
this.strictVersionCheck = !builder.force;
this.gcLogInterval = builder.gcLogInterval;
+ this.compactorType = builder.compactorType;
}
public int run() {
- System.out.printf("Compacting %s with %s\n", path, fileAccessMode.description);
+ System.out.printf("Compacting %s with %s and %s compactor type\n", path, fileAccessMode.description, compactorType.description());
System.out.printf(" before\n");
Set<File> beforeFiles = listFiles(path);
printFiles(System.out, beforeFiles);
@@ -312,7 +329,8 @@ public class Compact {
.withSegmentCacheSize(segmentCacheSize)
.withGCOptions(defaultGCOptions()
.setOffline()
- .setGCLogInterval(gcLogInterval));
+ .setGCLogInterval(gcLogInterval)
+ .setCompactorType(compactorType));
if (fileAccessMode.memoryMapped != null) {
builder.withMemoryMapping(fileAccessMode.memoryMapped);
}