You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by st...@apache.org on 2024/10/21 13:02:44 UTC
(jackrabbit-oak) branch trunk updated: OAK-11203 : added support for delayFactor, progress/batch size override from oak-run for full GC (#1814)
This is an automated email from the ASF dual-hosted git repository.
stefanegli pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new ddf259361e OAK-11203 : added support for delayFactor, progress/batch size override from oak-run for full GC (#1814)
ddf259361e is described below
commit ddf259361ee7c5c90ed81c01bc310b2643a92f2d
Author: Rishabh Kumar <ri...@gmail.com>
AuthorDate: Mon Oct 21 18:32:39 2024 +0530
OAK-11203 : added support for delayFactor, progress/batch size override from oak-run for full GC (#1814)
* OAK-11203 : added support for delayFactor, progress/batch size override from oak-run for full GC
* OAK-11203 : changed default value of delayFactor to 2 to avoid load on prod system
---------
Co-authored-by: Rishabh Kumar <di...@adobe.com>
---
.../plugins/document/DocumentNodeStoreHelper.java | 15 +++---
.../jackrabbit/oak/run/RevisionsCommand.java | 60 ++++++++++++++++------
.../oak/plugins/document/RevisionsCommandTest.java | 32 +++++++++++-
.../plugins/document/VersionGarbageCollector.java | 4 +-
4 files changed, 84 insertions(+), 27 deletions(-)
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
index 85387df838..f9bda9802f 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
@@ -45,9 +45,8 @@ import com.mongodb.client.FindIterable;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.model.Filters;
-import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FGC_BATCH_SIZE;
-import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FGC_DELAY_FACTOR;
-import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FGC_PROGRESS_SIZE;
+import static org.apache.jackrabbit.oak.plugins.document.util.Utils.isEmbeddedVerificationEnabled;
+import static org.apache.jackrabbit.oak.plugins.document.util.Utils.isFullGCEnabled;
/**
* Helper class to access package private method of DocumentNodeStore and other
@@ -72,11 +71,11 @@ public class DocumentNodeStoreHelper {
System.out.println("Collected in " + sw.stop());
}
- public static VersionGarbageCollector createVersionGC(DocumentNodeStore nodeStore, VersionGCSupport gcSupport,
- boolean fullGCEnabled, boolean isFullGCDryRun,
- boolean embeddedVerification, int fullGCMode) {
- return new VersionGarbageCollector(nodeStore, gcSupport, fullGCEnabled, isFullGCDryRun, embeddedVerification,
- fullGCMode, 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE);
+ public static VersionGarbageCollector createVersionGC(final DocumentNodeStore nodeStore, final VersionGCSupport gcSupport,
+ boolean isFullGCDryRun, final DocumentNodeStoreBuilder<?> builder) {
+ return new VersionGarbageCollector(nodeStore, gcSupport, isFullGCEnabled(builder), isFullGCDryRun,
+ isEmbeddedVerificationEnabled(builder), builder.getFullGCMode(), builder.getFullGCDelayFactor(),
+ builder.getFullGCBatchSize(), builder.getFullGCProgressSize());
}
public static DocumentNodeState readNode(DocumentNodeStore documentNodeStore, Path path, RevisionVector rootRevision) {
diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
index b10ad2f38b..779dc6e078 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
@@ -73,8 +73,6 @@ import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreHelper
import static org.apache.jackrabbit.oak.plugins.document.FormatVersion.versionOf;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getIdFromPath;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getRootDocument;
-import static org.apache.jackrabbit.oak.plugins.document.util.Utils.isFullGCEnabled;
-import static org.apache.jackrabbit.oak.plugins.document.util.Utils.isEmbeddedVerificationEnabled;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.timestampToString;
import static org.apache.jackrabbit.oak.run.Utils.asCloseable;
import static org.apache.jackrabbit.oak.run.Utils.createDocumentMKBuilder;
@@ -118,9 +116,12 @@ public class RevisionsCommand implements Command {
final OptionSpec<?> once;
final OptionSpec<Integer> limit;
+ final OptionSpec<Integer> fullGcBatchSize;
+ final OptionSpec<Integer> fullGcProgressSize;
final OptionSpec<Long> timeLimit;
final OptionSpec<Long> olderThan;
final OptionSpec<Double> delay;
+ final OptionSpec<Double> fullGcDelayFactor;
final OptionSpec<?> continuous;
final OptionSpec<?> fullGCOnly;
final OptionSpec<Boolean> resetFullGC;
@@ -146,6 +147,10 @@ public class RevisionsCommand implements Command {
delay = parser
.accepts("delay", "introduce delays to reduce impact on system").withRequiredArg()
.ofType(Double.class).defaultsTo(0.0);
+ fullGcDelayFactor = parser
+ .accepts("fullGcDelayFactor", "introduce delays after each fullGc batch to reduce load " +
+ "on system")
+ .withRequiredArg().ofType(Double.class).defaultsTo(2.0);
timeLimit = parser
.accepts("timeLimit", "cancel garbage collection after n seconds").withRequiredArg()
.ofType(Long.class).defaultsTo(-1L);
@@ -180,6 +185,12 @@ public class RevisionsCommand implements Command {
resetFullGC = parser
.accepts("resetFullGC", "reset fullGC after running FullGC")
.withRequiredArg().ofType(Boolean.class).defaultsTo(FALSE);
+ fullGcBatchSize = parser.accepts("fullGcBatchSize", "The number of documents to fetch from database " +
+ "in a single query to check for Full GC.")
+ .withRequiredArg().ofType(Integer.class).defaultsTo(1000);
+ fullGcProgressSize = parser.accepts("fullGcProgressSize", "The number of documents to check for " +
+ "garbage in each Full GC cycle")
+ .withRequiredArg().ofType(Integer.class).defaultsTo(10000);
}
public RevisionsOptions parse(String[] args) {
@@ -211,10 +222,22 @@ public class RevisionsCommand implements Command {
return embeddedVerification.value(options);
}
- int fullGcMode() {
+ int getFullGcMode() {
return fullGcMode.value(options);
}
+ int getFullGcBatchSize() {
+ return fullGcBatchSize.value(options);
+ }
+
+ int getFullGcProgressSize() {
+ return fullGcProgressSize.value(options);
+ }
+
+ double getFullGcDelayFactor() {
+ return fullGcDelayFactor.value(options);
+ }
+
long getOlderThan() {
return olderThan.value(options);
}
@@ -235,11 +258,11 @@ public class RevisionsCommand implements Command {
return path.value(options);
}
- String[] includePaths() {
+ String[] getIncludePaths() {
return includePaths.value(options).split("::");
}
- String[] excludePaths() {
+ String[] getExcludePaths() {
return excludePaths.value(options).split("::");
}
@@ -247,7 +270,7 @@ public class RevisionsCommand implements Command {
return options.has(fullGCOnly);
}
- boolean resetFullGC() {
+ boolean isResetFullGC() {
return resetFullGC.value(options);
}
@@ -320,9 +343,12 @@ public class RevisionsCommand implements Command {
}
// set fullGC
builder.setFullGCEnabled(fullGCEnabled);
- builder.setFullGCIncludePaths(options.includePaths());
- builder.setFullGCExcludePaths(options.excludePaths());
- builder.setFullGCMode(options.fullGcMode());
+ builder.setFullGCIncludePaths(options.getIncludePaths());
+ builder.setFullGCExcludePaths(options.getExcludePaths());
+ builder.setFullGCMode(options.getFullGcMode());
+ builder.setFullGCDelayFactor(options.getFullGcDelayFactor());
+ builder.setFullGCBatchSize(options.getFullGcBatchSize());
+ builder.setFullGCProgressSize(options.getFullGcProgressSize());
// create a VersionGCSupport while builder is read-write
VersionGCSupport gcSupport = builder.createVersionGCSupport();
@@ -346,13 +372,15 @@ public class RevisionsCommand implements Command {
// and a GC support with a writable DocumentStore
System.out.println("DryRun is enabled : " + options.isDryRun());
System.out.println("EmbeddedVerification is enabled : " + options.isEmbeddedVerificationEnabled());
- System.out.println("ResetFullGC is enabled : " + options.resetFullGC());
+ System.out.println("ResetFullGC is enabled : " + options.isResetFullGC());
System.out.println("Compaction is enabled : " + options.doCompaction());
- System.out.println("IncludePaths are : " + Arrays.toString(options.includePaths()));
- System.out.println("ExcludePaths are : " + Arrays.toString(options.excludePaths()));
- System.out.println("FullGcMode is : " + options.fullGcMode());
- VersionGarbageCollector gc = createVersionGC(builder.build(), gcSupport, isFullGCEnabled(builder), options.isDryRun(),
- isEmbeddedVerificationEnabled(builder), builder.getFullGCMode());
+ System.out.println("IncludePaths are : " + Arrays.toString(options.getIncludePaths()));
+ System.out.println("ExcludePaths are : " + Arrays.toString(options.getExcludePaths()));
+ System.out.println("FullGcMode is : " + options.getFullGcMode());
+ System.out.println("FullGcDelayFactory is : " + options.getFullGcDelayFactor());
+ System.out.println("FullGcBatchSize is : " + options.getFullGcBatchSize());
+ System.out.println("FullGcProgressSize is : " + options.getFullGcProgressSize());
+ VersionGarbageCollector gc = createVersionGC(builder.build(), gcSupport, options.isDryRun(), builder);
VersionGCOptions gcOptions = gc.getOptions();
gcOptions = gcOptions.withDelayFactor(options.getDelay());
@@ -430,7 +458,7 @@ public class RevisionsCommand implements Command {
if (options.isDryRun()) {
gc.resetDryRun();
}
- if (options.resetFullGC()) {
+ if (options.isResetFullGC()) {
gc.resetFullGC();
}
if (options.doCompaction()) {
diff --git a/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/document/RevisionsCommandTest.java b/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/document/RevisionsCommandTest.java
index d91bb80acd..a83cf27a4b 100644
--- a/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/document/RevisionsCommandTest.java
+++ b/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/document/RevisionsCommandTest.java
@@ -177,7 +177,7 @@ public class RevisionsCommandTest {
}
@Test
- public void collect() throws Exception {
+ public void collect() {
ns.dispose();
String output = captureSystemOut(new RevisionsCmd("collect"));
@@ -196,6 +196,36 @@ public class RevisionsCommandTest {
assertTrue(output.contains("IncludePaths are : [/]"));
assertTrue(output.contains("ExcludePaths are : []"));
assertTrue(output.contains("FullGcMode is : 0"));
+ assertTrue(output.contains("FullGcDelayFactory is : 2.0"));
+ assertTrue(output.contains("FullGcBatchSize is : 1000"));
+ assertTrue(output.contains("FullGcProgressSize is : 10000"));
+ }
+
+ @Test
+ public void fullGCWithDelayFactor() {
+ ns.dispose();
+
+ String output = captureSystemOut(new RevisionsCmd("fullGC", "--fullGcDelayFactor", "2.5", "--entireRepo"));
+ assertTrue(output.contains("FullGcDelayFactory is : 2.5"));
+ assertTrue(output.contains("starting gc collect"));
+ }
+
+ @Test
+ public void fullGCWithBatchSize() {
+ ns.dispose();
+
+ String output = captureSystemOut(new RevisionsCmd("fullGC", "--fullGcBatchSize", "200", "--entireRepo"));
+ assertTrue(output.contains("FullGcBatchSize is : 200"));
+ assertTrue(output.contains("starting gc collect"));
+ }
+
+ @Test
+ public void fullGCWithProgressSize() {
+ ns.dispose();
+
+ String output = captureSystemOut(new RevisionsCmd("fullGC", "--fullGcProgressSize", "20000", "--entireRepo"));
+ assertTrue(output.contains("FullGcProgressSize is : 20000"));
+ assertTrue(output.contains("starting gc collect"));
}
@Test
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
index 37b7297b7a..39b5fa28d5 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
@@ -284,8 +284,8 @@ public class VersionGarbageCollector {
this.options = new VersionGCOptions();
setFullGcMode(fullGCMode);
- AUDIT_LOG.info("<init> VersionGarbageCollector created with fullGcMode: {}, batchSize: {}, progressSize: {}",
- fullGcMode, fullGCBatchSize, fullGCProgressSize);
+ AUDIT_LOG.info("<init> VersionGarbageCollector created with fullGcMode: {}, batchSize: {}, progressSize: {}, delayFactor: {}",
+ fullGcMode, fullGCBatchSize, fullGCProgressSize, fullGCDelayFactor);
}
/**