You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by da...@apache.org on 2025/01/29 11:35:59 UTC
(jackrabbit-oak) branch OAK-11433 created (now 84c0a8077d)
This is an automated email from the ASF dual-hosted git repository.
daim pushed a change to branch OAK-11433
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
at 84c0a8077d OAK-11433 : added OSGi config for setting max revision age for full gc
This branch includes the following new commits:
new 84c0a8077d OAK-11433 : added OSGi config for setting max revision age for full gc
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(jackrabbit-oak) 01/01: OAK-11433 : added OSGi config for setting max revision age for full gc
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
daim pushed a commit to branch OAK-11433
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 84c0a8077d57db42b43b6a96addbf33f90087be2
Author: Rishabh Kumar <di...@adobe.com>
AuthorDate: Wed Jan 29 17:05:49 2025 +0530
OAK-11433 : added OSGi config for setting max revision age for full gc
---
.../plugins/document/DocumentNodeStoreHelper.java | 2 +-
.../oak/plugins/document/Configuration.java | 8 +++
.../oak/plugins/document/DocumentNodeStore.java | 2 +-
.../plugins/document/DocumentNodeStoreBuilder.java | 21 ++++++++
.../plugins/document/DocumentNodeStoreService.java | 7 +++
.../plugins/document/VersionGCRecommendations.java | 9 ++--
.../plugins/document/VersionGarbageCollector.java | 20 +++++---
.../document/rdb/RDBDocumentNodeStoreBuilder.java | 13 +++++
.../oak/plugins/document/BranchCommitGCTest.java | 45 +++++++++--------
.../DocumentNodeStoreServiceConfigurationTest.java | 9 ++++
.../oak/plugins/document/FullGCHelper.java | 13 +++++
.../plugins/document/VersionGCDeletionTest.java | 15 +++---
.../oak/plugins/document/VersionGCTest.java | 58 +++++++++++-----------
.../document/VersionGarbageCollectorIT.java | 57 +++++++++++++++------
.../rdb/RDBDocumentNodeStoreBuilderTest.java | 7 +++
15 files changed, 204 insertions(+), 82 deletions(-)
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
index b52c853ba5..454f73e47c 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
@@ -74,7 +74,7 @@ public class DocumentNodeStoreHelper {
boolean isFullGCDryRun, final DocumentNodeStoreBuilder<?> builder) {
return new VersionGarbageCollector(nodeStore, gcSupport, isFullGCEnabled(builder), isFullGCDryRun,
isEmbeddedVerificationEnabled(builder), builder.getFullGCMode(), builder.getFullGCDelayFactor(),
- builder.getFullGCBatchSize(), builder.getFullGCProgressSize());
+ builder.getFullGCBatchSize(), builder.getFullGCProgressSize(), builder.getFullGcMaxAgeMillis());
}
public static DocumentNodeState readNode(DocumentNodeStore documentNodeStore, Path path, RevisionVector rootRevision) {
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Configuration.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Configuration.java
index 6dec39e11b..9e7921036a 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Configuration.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Configuration.java
@@ -349,6 +349,14 @@ import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreServic
"property 'oak.documentstore.fullGCEnabled'")
boolean fullGCEnabled() default DEFAULT_FULL_GC_ENABLED;
+ @AttributeDefinition(
+ name = "Full GC Max Age (in secs)",
+ description = "Version Garbage Collector (Full GC) logic will only consider those nodes for Full GC which " +
+ "are not accessed recently (currentTime - lastModifiedTime > fullGcMaxAgeInSecs). For " +
+ "example as per default only those document which have not been *updated* 24 hrs ago will be " +
+ "considered for Full GC.")
+ long fullGcMaxAgeInSecs() default DocumentNodeStoreService.DEFAULT_FULL_GC_MAX_AGE;
+
@AttributeDefinition(
name = "Document Node Store Embedded Verification for Full GC",
description = "Boolean value indicating whether Embedded Verification (i.e. verify the document after " +
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
index e056cb0f01..95bd69017a 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
@@ -656,7 +656,7 @@ public final class DocumentNodeStore
this.versionGarbageCollector = new VersionGarbageCollector(
this, builder.createVersionGCSupport(), isFullGCEnabled(builder), false,
isEmbeddedVerificationEnabled(builder), builder.getFullGCMode(), builder.getFullGCDelayFactor(),
- builder.getFullGCBatchSize(), builder.getFullGCProgressSize());
+ builder.getFullGCBatchSize(), builder.getFullGCProgressSize(), builder.getFullGcMaxAgeMillis());
this.versionGarbageCollector.setStatisticsProvider(builder.getStatisticsProvider());
this.versionGarbageCollector.setGCMonitor(builder.getGCMonitor());
this.versionGarbageCollector.setFullGCPaths(builder.getFullGCIncludePaths(), builder.getFullGCExcludePaths());
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java
index c260c82b8d..7b94796fe9 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java
@@ -181,6 +181,7 @@ public class DocumentNodeStoreBuilder<T extends DocumentNodeStoreBuilder<T>> {
private Set<String> fullGCExcludePaths = Set.of();
private boolean embeddedVerificationEnabled = DocumentNodeStoreService.DEFAULT_EMBEDDED_VERIFICATION_ENABLED;
private int fullGCMode = DocumentNodeStoreService.DEFAULT_FULL_GC_MODE;
+ private long fullGcMaxAgeMillis = TimeUnit.SECONDS.toMillis(DocumentNodeStoreService.DEFAULT_FULL_GC_MAX_AGE);
private int fullGCBatchSize = DocumentNodeStoreService.DEFAULT_FGC_BATCH_SIZE;
private int fullGCProgressSize = DocumentNodeStoreService.DEFAULT_FGC_PROGRESS_SIZE;
private double fullGCDelayFactor = DocumentNodeStoreService.DEFAULT_FGC_DELAY_FACTOR;
@@ -360,6 +361,26 @@ public class DocumentNodeStoreBuilder<T extends DocumentNodeStoreBuilder<T>> {
return this.fullGCMode;
}
+ /**
+ * The maximum age for nodes in milliseconds. Older entries are candidates for full gc
+ * @param v max age in millis
+ * @return builder object
+ */
+ public T setFullGcMaxAgeMillis(long v) {
+ this.fullGcMaxAgeMillis = v;
+ return thisBuilder();
+ }
+
+ /**
+ * The maximum age for nodes in milliseconds. Older entries
+ * are candidates for Full GC.
+ *
+ * @return maximum age for nodes entries in milliseconds.
+ */
+ public long getFullGcMaxAgeMillis() {
+ return this.fullGcMaxAgeMillis;
+ }
+
public T setFullGCBatchSize(int v) {
this.fullGCBatchSize = v;
return thisBuilder();
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
index 31e79fb61e..7e20d2f959 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
@@ -165,6 +165,11 @@ public class DocumentNodeStoreService {
*/
static final long DEFAULT_VER_GC_MAX_AGE = 24 * 60 * 60; //TimeUnit.DAYS.toSeconds(1);
+ /**
+ * Nodes older than this time would be garbage collected by Full GC
+ */
+ static final long DEFAULT_FULL_GC_MAX_AGE = 24 * 60 * 60; //TimeUnit.DAYS.toSeconds(1);
+
/**
* Blob modified before this time duration would be considered for Blob GC
@@ -523,6 +528,7 @@ public class DocumentNodeStoreService {
setFullGCExcludePaths(config.fullGCExcludePaths()).
setEmbeddedVerificationEnabled(config.embeddedVerificationEnabled()).
setFullGCMode(config.fullGCMode()).
+ setFullGcMaxAgeMillis(TimeUnit.SECONDS.toMillis(config.fullGcMaxAgeInSecs())).
setFullGCBatchSize(config.fullGCBatchSize()).
setFullGCProgressSize(config.fullGCProgressSize()).
setFullGCDelayFactor(config.fullGCDelayFactor()).
@@ -947,6 +953,7 @@ public class DocumentNodeStoreService {
Map<String, Object> props = jobPropertiesFor(RevisionGCJob.class);
props.put("scheduler.expression", expr);
long versionGcMaxAgeInSecs = config.versionGcMaxAgeInSecs();
+ long fullGcMaxAgeInSecs = config.fullGcMaxAgeInSecs();
long versionGCTimeLimitInSecs = config.versionGCTimeLimitInSecs();
double versionGCDelayFactor = config.versionGCDelayFactor();
addRegistration(scheduleWithFixedDelay(whiteboard,
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java
index 72afd946ab..3d92e4e144 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java
@@ -104,10 +104,11 @@ public class VersionGCRecommendations {
* @param gcMonitor monitor class for messages
* @param fullGCEnabled whether fullGC is enabled or not
* @param isFullGCDryRun whether fullGC is running in dryRun mode or not
+ * @param fullGcMaxAgeMs the maximum age for revisions to be collected by fullGC
*/
VersionGCRecommendations(long maxRevisionAgeMs, Checkpoints checkpoints, boolean checkpointCleanup, Clock clock,
VersionGCSupport vgc, VersionGCOptions options, GCMonitor gcMonitor,
- boolean fullGCEnabled, boolean isFullGCDryRun) {
+ boolean fullGCEnabled, boolean isFullGCDryRun, long fullGcMaxAgeMs) {
boolean ignoreDueToCheckPoint;
boolean ignoreFullGCDueToCheckPoint;
long deletedOnceCount = 0;
@@ -185,9 +186,11 @@ public class VersionGCRecommendations {
}
}
+ TimeInterval keepFullGc = new TimeInterval(clock.getTime() - fullGcMaxAgeMs, Long.MAX_VALUE);
+
TimeInterval scopeFullGC = new TimeInterval(isFullGCDryRun ? oldestModifiedDryRunDocTimeStamp.get() :
oldestModifiedDocTimeStamp.get(), MAX_VALUE);
- scopeFullGC = scopeFullGC.notLaterThan(keep.fromMs);
+ scopeFullGC = scopeFullGC.notLaterThan(keepFullGc.fromMs);
suggestedIntervalMs = (long) settings.get(SETTINGS_COLLECTION_REC_INTERVAL_PROP);
if (suggestedIntervalMs > 0) {
@@ -248,7 +251,7 @@ public class VersionGCRecommendations {
this.scopeFullGC = scopeFullGC;
this.fullGCId = isFullGCDryRun ? oldestModifiedDryRunDocId : oldestModifiedDocId;
this.scopeIsComplete = scope.toMs >= keep.fromMs;
- this.fullGCScopeIsComplete = scopeFullGC.toMs >= keep.fromMs;
+ this.fullGCScopeIsComplete = scopeFullGC.toMs >= keepFullGc.fromMs;
this.maxCollect = collectLimit;
this.suggestedIntervalMs = suggestedIntervalMs;
this.deleteCandidateCount = deletedOnceCount;
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
index 18c544a495..c3ea9f7566 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
@@ -83,6 +83,7 @@ import static org.apache.jackrabbit.oak.plugins.document.Collection.SETTINGS;
import static org.apache.jackrabbit.oak.plugins.document.Document.ID;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FGC_BATCH_SIZE;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FGC_PROGRESS_SIZE;
+import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FULL_GC_MAX_AGE;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FULL_GC_MODE;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.BRANCH_COMMITS;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.COLLISIONS;
@@ -237,6 +238,7 @@ public class VersionGarbageCollector {
private final boolean isFullGCDryRun;
private final boolean embeddedVerification;
private final double fullGCDelayFactor;
+ private long fullGcMaxAgeInMillis;
private final int fullGCBatchSize;
private final int fullGCProgressSize;
private Set<String> fullGCIncludePaths = Collections.emptySet();
@@ -254,7 +256,7 @@ public class VersionGarbageCollector {
final boolean isFullGCDryRun,
final boolean embeddedVerification) {
this(nodeStore, gcSupport, fullGCEnabled, isFullGCDryRun, embeddedVerification, DEFAULT_FULL_GC_MODE,
- 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE);
+ 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE, SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE));
}
VersionGarbageCollector(DocumentNodeStore nodeStore,
@@ -265,7 +267,8 @@ public class VersionGarbageCollector {
final int fullGCMode,
final double fullGCDelayFactor,
final int fullGCBatchSize,
- final int fullGCProgressSize) {
+ final int fullGCProgressSize,
+ final long fullGcMaxAgeInMillis) {
this.nodeStore = nodeStore;
this.versionStore = gcSupport;
this.ds = gcSupport.getDocumentStore();
@@ -273,13 +276,14 @@ public class VersionGarbageCollector {
this.isFullGCDryRun = isFullGCDryRun;
this.embeddedVerification = embeddedVerification;
this.fullGCDelayFactor = fullGCDelayFactor;
+ this.fullGcMaxAgeInMillis = fullGcMaxAgeInMillis;
this.fullGCBatchSize = Math.min(fullGCBatchSize, fullGCProgressSize);
this.fullGCProgressSize = fullGCProgressSize;
this.options = new VersionGCOptions();
setFullGcMode(fullGCMode);
- AUDIT_LOG.info("<init> VersionGarbageCollector created with fullGcMode: {}, batchSize: {}, progressSize: {}, delayFactor: {}",
- fullGcMode, fullGCBatchSize, fullGCProgressSize, fullGCDelayFactor);
+ AUDIT_LOG.info("<init> VersionGarbageCollector created with fullGcMode: {}, maxFullGcAgeInMillis: {}, batchSize: {}, progressSize: {}, delayFactor: {}",
+ fullGcMode, fullGcMaxAgeInMillis, fullGCBatchSize, fullGCProgressSize, fullGCDelayFactor);
}
/**
@@ -297,6 +301,10 @@ public class VersionGarbageCollector {
AUDIT_LOG.info("Full GC paths set to include: {} and exclude: {} in mode {}", includes, excludes, fullGcMode);
}
+ void setFullGcMaxAge(final long fullGcMaxAge, final TimeUnit unit) {
+ this.fullGcMaxAgeInMillis = unit.toMillis(fullGcMaxAge);
+ }
+
public void setStatisticsProvider(StatisticsProvider provider) {
this.gcStats = new RevisionGCStats(provider);
this.fullGCStats = new FullGCStatsCollectorImpl(provider);
@@ -411,7 +419,7 @@ public class VersionGarbageCollector {
long now = nodeStore.getClock().getTime();
VersionGCRecommendations rec = new VersionGCRecommendations(maxRevisionAgeInMillis, nodeStore.getCheckpoints(),
!nodeStore.isReadOnlyMode(), nodeStore.getClock(), versionStore, options, gcMonitor, fullGCEnabled,
- isFullGCDryRun);
+ isFullGCDryRun, fullGcMaxAgeInMillis);
int estimatedIterations = -1;
if (rec.suggestedIntervalMs > 0) {
estimatedIterations = (int)Math.ceil((double) (now - rec.scope.toMs) / rec.suggestedIntervalMs);
@@ -792,7 +800,7 @@ public class VersionGarbageCollector {
stats.active.start();
VersionGCRecommendations rec = new VersionGCRecommendations(maxRevisionAgeInMillis, nodeStore.getCheckpoints(),
!nodeStore.isReadOnlyMode(), nodeStore.getClock(), versionStore, options, gcMonitor, fullGCEnabled,
- isFullGCDryRun);
+ isFullGCDryRun, fullGcMaxAgeInMillis);
GCPhases phases = new GCPhases(cancel, stats, gcMonitor);
try {
if (!isFullGCDryRun) {
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilder.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilder.java
index 69ef254408..18cfb61f60 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilder.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilder.java
@@ -170,6 +170,19 @@ public class RDBDocumentNodeStoreBuilder
// fullGC modes are not supported for RDB
return 0;
}
+
+ @Override
+ public RDBDocumentNodeStoreBuilder setFullGcMaxAgeMillis(long v) {
+ // fullGC modes are not supported for RDB
+ log.warn("FullGC Max Age is not supported for RDB");
+ return thisBuilder();
+ }
+
+ @Override
+ public long getFullGcMaxAgeMillis() {
+ // fullGC max age is not supported for RDB
+ return 0;
+ }
@Override
public RDBDocumentNodeStoreBuilder setDocStoreFullGCFeature(@Nullable Feature docStoreFullGC) {
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BranchCommitGCTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BranchCommitGCTest.java
index fc8921048e..6bc5ed5c75 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BranchCommitGCTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BranchCommitGCTest.java
@@ -21,6 +21,7 @@ import static org.apache.commons.lang3.reflect.FieldUtils.writeField;
import static org.apache.commons.lang3.reflect.FieldUtils.writeStaticField;
import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.assertBranchRevisionRemovedFromAllDocuments;
import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.build;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.gc;
import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.allOrphProp;
import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.assertStatsCountsEqual;
import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.assertStatsCountsZero;
@@ -145,7 +146,7 @@ public class BranchCommitGCTest {
// wait two hours
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
// clean everything older than one hour
- VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS);
+ VersionGarbageCollector.VersionGCStats stats = gc(gc,1, HOURS);
assertStatsCountsEqual(stats,
gapOrphOnly(),
@@ -160,7 +161,7 @@ public class BranchCommitGCTest {
// now do another gc - should not have anything left to clean up though
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
- stats = gc.gc(1, HOURS);
+ stats = gc(gc,1, HOURS);
assertStatsCountsZero(stats);
if (!isModeOneOf(FullGCMode.NONE, FullGCMode.GAP_ORPHANS, FullGCMode.GAP_ORPHANS_EMPTYPROPS, FullGCMode.EMPTYPROPS)) {
assertNotExists("1:/a");
@@ -204,7 +205,7 @@ public class BranchCommitGCTest {
// wait two hours
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
// clean everything older than one hour
- VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS);
+ VersionGarbageCollector.VersionGCStats stats = gc(gc, 1, HOURS);
if (collisionsBeforeGC == 1) {
// expects a collision to have happened - which was cleaned up - hence a _bc (but not the _revision I guess)
@@ -241,7 +242,7 @@ public class BranchCommitGCTest {
assertNotExists("1:/b");
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
- stats = gc.gc(1, HOURS);
+ stats = gc(gc, 1, HOURS);
assertEquals(0, stats.updatedFullGCDocsCount);
assertEquals(0, stats.deletedDocGCCount);
assertEquals(0, stats.deletedUnmergedBCCount);
@@ -288,7 +289,7 @@ public class BranchCommitGCTest {
// wait two hours
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
// clean everything older than one hour
- VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS);
+ VersionGarbageCollector.VersionGCStats stats = gc(gc, 1, HOURS);
// 6 deleted props: 0:/[_collisions], 1:/foo[p, a], 1:/bar[_bc,prop,_revisions]
assertStatsCountsEqual(stats,
@@ -331,7 +332,7 @@ public class BranchCommitGCTest {
// wait two hours
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
// clean everything older than one hour
- VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS);
+ VersionGarbageCollector.VersionGCStats stats = gc(gc, 1, HOURS);
assertStatsCountsEqual(stats,
gapOrphOnly(),
@@ -351,7 +352,7 @@ public class BranchCommitGCTest {
// now do another gc - should not have anything left to clean up though
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
- stats = gc.gc(1, HOURS);
+ stats = gc(gc, 1, HOURS);
assertStatsCountsZero(stats);
if (!isModeOneOf(FullGCMode.NONE)) {
assertBranchRevisionRemovedFromAllDocuments(store, br1);
@@ -387,7 +388,7 @@ public class BranchCommitGCTest {
// wait two hours
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
// clean everything older than one hour
- VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS);
+ VersionGarbageCollector.VersionGCStats stats = gc(gc, 1, HOURS);
assertStatsCountsEqual(stats,
gapOrphOnly(),
@@ -404,7 +405,7 @@ public class BranchCommitGCTest {
// now do another gc to get those documents actually deleted
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
- stats = gc.gc(1, HOURS);
+ stats = gc(gc, 1, HOURS);
assertStatsCountsZero(stats);
if (!isModeOneOf(FullGCMode.NONE)) {
assertBranchRevisionRemovedFromAllDocuments(store, br1);
@@ -452,7 +453,7 @@ public class BranchCommitGCTest {
// wait two hours
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
// clean everything older than one hour
- VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS);
+ VersionGarbageCollector.VersionGCStats stats = gc(gc, 1, HOURS);
assertStatsCountsEqual(stats,
empPropOnly(0, 0, 0, 0, 0, 0, 0),
@@ -469,7 +470,7 @@ public class BranchCommitGCTest {
// now do another gc to get those documents actually deleted
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
- stats = gc.gc(1, HOURS);
+ stats = gc(gc, 1, HOURS);
assertStatsCountsZero(stats);
if (!isModeOneOf(FullGCMode.NONE)) {
assertBranchRevisionRemovedFromAllDocuments(store, br1);
@@ -494,7 +495,7 @@ public class BranchCommitGCTest {
// wait two hours
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
// clean everything older than one hour
- VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS);
+ VersionGarbageCollector.VersionGCStats stats = gc(gc, 1, HOURS);
// first gc round now deletes it, via orphaned node deletion
assertStatsCountsEqual(stats,
@@ -511,7 +512,7 @@ public class BranchCommitGCTest {
// wait two hours
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
// now do second gc round - should not have anything left to clean up though
- stats = gc.gc(1, HOURS);
+ stats = gc(gc, 1, HOURS);
assertStatsCountsZero(stats);
if (!isModeOneOf(FullGCMode.NONE)) {
assertBranchRevisionRemovedFromAllDocuments(store, br);
@@ -524,7 +525,7 @@ public class BranchCommitGCTest {
mergedBranchCommit(b -> b.child("foo").setProperty("a", "b"));
// do a gc without waiting, to check that works fine
store.runBackgroundOperations();
- stats = gc.gc(1, HOURS);
+ stats = gc(gc, 1, HOURS);
assertEquals(0, stats.updatedFullGCDocsCount);
RevisionVector br = unmergedBranchCommit(b -> b.child("foo").removeProperty("a"));
@@ -534,7 +535,7 @@ public class BranchCommitGCTest {
// wait two hours
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
// clean everything older than one hour
- stats = gc.gc(1, HOURS);
+ stats = gc(gc, 1, HOURS);
assertStatsCountsEqual(stats,
gapOrphOnly(),
@@ -560,7 +561,7 @@ public class BranchCommitGCTest {
// wait two hours
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
// clean everything older than one hour
- stats = gc.gc(1, HOURS);
+ stats = gc(gc, 1, HOURS);
// 1 deleted prop: 1:/foo[a]
assertStatsCountsEqual(stats,
@@ -589,7 +590,7 @@ public class BranchCommitGCTest {
});
// do a gc without waiting, to check that works fine
store.runBackgroundOperations();
- stats = gc.gc(1, HOURS);
+ stats = gc(gc, 1, HOURS);
assertEquals(0, stats.updatedFullGCDocsCount);
assertEquals(0, stats.deletedUnmergedBCCount);
@@ -605,7 +606,7 @@ public class BranchCommitGCTest {
// wait two hours
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
// clean everything older than one hour
- stats = gc.gc(1, HOURS);
+ stats = gc(gc, 1, HOURS);
assertStatsCountsEqual(stats,
gapOrphOnly(0, 0, 0, 0, 0, 0, 0),
@@ -638,7 +639,7 @@ public class BranchCommitGCTest {
});
// do a gc without waiting, to check that works fine
store.runBackgroundOperations();
- stats = gc.gc(1, HOURS);
+ stats = gc(gc, 1, HOURS);
assertEquals(0, stats.updatedFullGCDocsCount);
assertEquals(0, stats.deletedUnmergedBCCount);
@@ -655,7 +656,7 @@ public class BranchCommitGCTest {
// wait two hours
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
// clean everything older than one hour
- stats = gc.gc(1, HOURS);
+ stats = gc(gc, 1, HOURS);
assertStatsCountsEqual(stats,
gapOrphOnly(0, 0, 0, 0, 0, 0, 0),
@@ -681,7 +682,7 @@ public class BranchCommitGCTest {
mergedBranchCommit(b -> b.child("foo").setProperty("c", "d"));
// do a gc without waiting, to check that works fine
store.runBackgroundOperations();
- stats = gc.gc(1, HOURS);
+ stats = gc(gc, 1, HOURS);
assertEquals(0, stats.updatedFullGCDocsCount);
assertEquals(0, stats.deletedUnmergedBCCount);
assertEquals(0, stats.deletedPropsCount);
@@ -712,7 +713,7 @@ public class BranchCommitGCTest {
// wait two hours
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
// clean everything older than one hour
- stats = gc.gc(1, HOURS);
+ stats = gc(gc, 1, HOURS);
// deleted props: 0:/[rootProp], 1:/foo[a]
// deleted internal prop : 0:/ _collision
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceConfigurationTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceConfigurationTest.java
index f713148420..398f650174 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceConfigurationTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceConfigurationTest.java
@@ -101,6 +101,7 @@ public class DocumentNodeStoreServiceConfigurationTest {
assertEquals(DEFAULT_FGC_PROGRESS_SIZE, config.fullGCProgressSize());
assertEquals(DEFAULT_FULL_GC_ENABLED, config.fullGCEnabled());
assertEquals(DEFAULT_EMBEDDED_VERIFICATION_ENABLED, config.embeddedVerificationEnabled());
+ assertEquals(DocumentNodeStoreService.DEFAULT_FULL_GC_MAX_AGE, config.fullGcMaxAgeInSecs());
assertEquals(CommitQueue.DEFAULT_SUSPEND_TIMEOUT, config.suspendTimeoutMillis());
}
@@ -184,6 +185,14 @@ public class DocumentNodeStoreServiceConfigurationTest {
assertEquals(fullGCDelayFactor, config.fullGCDelayFactor(), 0.01);
}
+ @Test
+ public void fullGcMaxAgeInSecs() throws Exception {
+ long fullGcMaxAgeInSecs = 30 * 24 * 60 * 60; // 30 days
+ addConfigurationEntry(preset, "fullGcMaxAgeInSecs", fullGcMaxAgeInSecs);
+ Configuration config = createConfiguration();
+ assertEquals(fullGcMaxAgeInSecs, config.fullGcMaxAgeInSecs());
+ }
+
@Test
public void presetSocketKeepAlive() throws Exception {
boolean keepAlive = !DocumentNodeStoreService.DEFAULT_SO_KEEP_ALIVE;
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/FullGCHelper.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/FullGCHelper.java
index f75f180fe4..6af0fbc4db 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/FullGCHelper.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/FullGCHelper.java
@@ -26,12 +26,15 @@ import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import java.io.IOException;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import static org.apache.commons.lang3.reflect.FieldUtils.writeField;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -60,6 +63,16 @@ public class FullGCHelper {
}
}
+ public static VersionGarbageCollector.VersionGCStats gc(final VersionGarbageCollector gc, final long maxRevisionAge, final TimeUnit unit) throws IOException {
+ gc.setFullGcMaxAge(maxRevisionAge, unit); // set full gc max age
+ final VersionGarbageCollector.VersionGCStats stats = gc.gc(maxRevisionAge, unit);
+ if (stats.skippedFullGCDocsCount != 0) {
+ (new Exception("here: " + stats.skippedFullGCDocsCount)).printStackTrace(System.out);
+ }
+ assertEquals(0, stats.skippedFullGCDocsCount);
+ return stats;
+ }
+
public static RevisionVector mergedBranchCommit(final NodeStore store, final Consumer<NodeBuilder> buildFunction) throws Exception {
return build(store, true, true, buildFunction);
}
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCDeletionTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCDeletionTest.java
index aff4a6b324..55cf764243 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCDeletionTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCDeletionTest.java
@@ -21,6 +21,7 @@ package org.apache.jackrabbit.oak.plugins.document;
import static java.util.concurrent.Executors.newSingleThreadExecutor;
import static java.util.concurrent.TimeUnit.HOURS;
import static java.util.concurrent.TimeUnit.MINUTES;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.gc;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -109,14 +110,14 @@ public class VersionGCDeletionTest {
//sorted again by VersionGC and then /x would always come after /x/y
try {
ts.throwException = true;
- gc.gc(maxAge * 2, HOURS);
+ gc(gc, maxAge * 2, HOURS);
fail("Exception should be thrown");
} catch (AssertionError ignore) {
}
ts.throwException = false;
- gc.gc(maxAge * 2, HOURS);
+ gc(gc, maxAge * 2, HOURS);
assertNull(ts.find(Collection.NODES, "2:/x/y"));
assertNull(ts.find(Collection.NODES, "1:/x"));
}
@@ -162,7 +163,7 @@ public class VersionGCDeletionTest {
clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge * 2) + delta);
VersionGarbageCollector gc = store.getVersionGarbageCollector();
- VersionGCStats stats = gc.gc(maxAge * 2, HOURS);
+ VersionGCStats stats = gc(gc, maxAge * 2, HOURS);
assertEquals(1, stats.updateResurrectedGCCount);
NodeDocument d4 = ts.find(Collection.NODES, id, 0);
assertNotNull(d4);
@@ -202,7 +203,7 @@ public class VersionGCDeletionTest {
VersionGarbageCollector gc = store.getVersionGarbageCollector();
gc.setOptions(gc.getOptions().withOverflowToDiskThreshold(100));
- VersionGCStats stats = gc.gc(maxAge * 2, HOURS);
+ VersionGCStats stats = gc(gc, maxAge * 2, HOURS);
assertEquals(noOfDocsToDelete * 2 + 1, stats.deletedDocGCCount);
assertEquals(noOfDocsToDelete, stats.deletedLeafDocGCCount);
@@ -249,7 +250,7 @@ public class VersionGCDeletionTest {
VersionGarbageCollector gc = store.getVersionGarbageCollector();
gc.setOptions(gc.getOptions().withOverflowToDiskThreshold(100));
- VersionGCStats stats = gc.gc(maxAge * 2, HOURS);
+ VersionGCStats stats = gc(gc, maxAge * 2, HOURS);
assertEquals(noOfDocsToDelete * 2 + 1, stats.deletedDocGCCount);
assertEquals(noOfDocsToDelete, stats.deletedLeafDocGCCount);
}
@@ -297,7 +298,7 @@ public class VersionGCDeletionTest {
//Pass some time and run GC
clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge * 2) + delta);
VersionGarbageCollector gc = store.getVersionGarbageCollector();
- VersionGCStats stats = gc.gc(maxAge * 2, HOURS);
+ VersionGCStats stats = gc(gc, maxAge * 2, HOURS);
//Asset GC stats
assertEquals(2, stats.deletedDocGCCount);
@@ -382,7 +383,7 @@ public class VersionGCDeletionTest {
// run GC once the reader thread is collecting documents
ready.await();
VersionGarbageCollector gc = store.getVersionGarbageCollector();
- VersionGCStats stats = gc.gc(30, MINUTES);
+ VersionGCStats stats = gc(gc, 30, MINUTES);
assertEquals(90, stats.deletedDocGCCount);
assertEquals(90, stats.deletedLeafDocGCCount);
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
index edb4bfd692..1b87fd5d46 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
@@ -55,6 +55,7 @@ import static org.apache.commons.lang3.reflect.FieldUtils.readDeclaredField;
import static org.apache.jackrabbit.oak.plugins.document.Collection.SETTINGS;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FGC_BATCH_SIZE;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FGC_PROGRESS_SIZE;
+import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FULL_GC_MAX_AGE;
import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.disableFullGC;
import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.disableFullGCDryRun;
import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.enableFullGC;
@@ -136,7 +137,7 @@ public class VersionGCTest {
assertTrue(gcBlocked);
// now try to trigger another GC
try {
- gc.gc(30, TimeUnit.MINUTES);
+ FullGCHelper.gc(gc, 30, TimeUnit.MINUTES);
fail("must throw an IOException");
} catch (IOException e) {
assertTrue(e.getMessage().contains("already running"));
@@ -209,7 +210,7 @@ public class VersionGCTest {
String versionGCId = SETTINGS_COLLECTION_ID;
String fullGCTimestamp = SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP;
enableFullGC(gc);
- gc.gc(30, SECONDS);
+ FullGCHelper.gc(gc, 30, SECONDS);
Document statusBefore = store.find(SETTINGS, versionGCId);
// block gc call
store.semaphore.acquireUninterruptibly();
@@ -247,7 +248,7 @@ public class VersionGCTest {
String versionGCId = SETTINGS_COLLECTION_ID;
String oldestModifiedDocId = SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP;
enableFullGC(gc);
- gc.gc(30, SECONDS);
+ FullGCHelper.gc(gc, 30, SECONDS);
Document statusBefore = store.find(SETTINGS, versionGCId);
// block gc call
store.semaphore.acquireUninterruptibly();
@@ -288,12 +289,12 @@ public class VersionGCTest {
String versionGCId = SETTINGS_COLLECTION_ID;
String fullGCTimestamp = SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP;
enableFullGC(gc);
- gc.gc(30, SECONDS);
+ FullGCHelper.gc(gc, 30, SECONDS);
Document statusBefore = store.find(SETTINGS, versionGCId);
// now run GC in dryRun mode
enableFullGCDryRun(gc);
- gc.gc(30, SECONDS);
+ FullGCHelper.gc(gc, 30, SECONDS);
// ensure a dryRun GC doesn't update that versionGC SETTINGS entries
Document statusAfter = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
@@ -309,11 +310,11 @@ public class VersionGCTest {
String versionGCId = SETTINGS_COLLECTION_ID;
String oldestModifiedDocId = SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP;
enableFullGC(gc);
- gc.gc(30, SECONDS);
+ FullGCHelper.gc(gc, 30, SECONDS);
final Document statusBefore = store.find(SETTINGS, versionGCId);
// now run GC in dryRun mode
enableFullGCDryRun(gc);
- gc.gc(30, SECONDS);
+ FullGCHelper.gc(gc, 30, SECONDS);
// ensure a dryRun GC doesn't update that versionGC SETTINGS entry
final Document statusAfter = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
assertNotNull(statusAfter);
@@ -326,7 +327,7 @@ public class VersionGCTest {
@Test
public void getInfo() throws Exception {
- gc.gc(1, TimeUnit.HOURS);
+ FullGCHelper.gc(gc, 1, TimeUnit.HOURS);
gc.getInfo(1, TimeUnit.HOURS);
}
@@ -336,7 +337,7 @@ public class VersionGCTest {
TestGCMonitor monitor = new TestGCMonitor();
gc.setGCMonitor(monitor);
- gc.gc(30, TimeUnit.MINUTES);
+ FullGCHelper.gc(gc, 30, TimeUnit.MINUTES);
List<String> expected = List.of("INITIALIZING",
"COLLECTING", "CHECKING", "COLLECTING", "DELETING", "SORTING",
@@ -349,7 +350,7 @@ public class VersionGCTest {
TestGCMonitor monitor = new TestGCMonitor();
gc.setGCMonitor(monitor);
- gc.gc(2, TimeUnit.HOURS);
+ FullGCHelper.gc(gc, 2, TimeUnit.HOURS);
List<String> infoMessages = monitor.getInfoMessages();
assertEquals(3, infoMessages.size());
@@ -361,7 +362,7 @@ public class VersionGCTest {
@Test
public void findVersionGC() throws Exception {
store.findVersionGC.set(0);
- gc.gc(1, TimeUnit.HOURS);
+ FullGCHelper.gc(gc, 1, TimeUnit.HOURS);
// must only read once
assertEquals(1, store.findVersionGC.get());
}
@@ -377,7 +378,7 @@ public class VersionGCTest {
VersionGCSupport localgcsupport = fakeVersionGCSupport(ns.getDocumentStore(), oneYearAgo, twelveTimesTheLimit);
VersionGCRecommendations rec = new VersionGCRecommendations(secondsPerDay, ns.getCheckpoints(), true, ns.getClock(),
- localgcsupport, options, new TestGCMonitor(), false, false);
+ localgcsupport, options, new TestGCMonitor(), false, false, SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE));
// should select a duration of roughly one month
long duration= rec.scope.getDurationMs();
@@ -391,7 +392,7 @@ public class VersionGCTest {
assertTrue(stats.needRepeat);
rec = new VersionGCRecommendations(secondsPerDay, ns.getCheckpoints(), true, ns.getClock(), localgcsupport,
- options, new TestGCMonitor(), false, false);
+ options, new TestGCMonitor(), false, false, SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE));
// new duration should be half
long nduration = rec.scope.getDurationMs();
@@ -420,7 +421,7 @@ public class VersionGCTest {
// loop until the recommended interval is at 60s (precisionMS)
do {
rec = new VersionGCRecommendations(secondsPerDay, ns.getCheckpoints(), true, ns.getClock(), localgcsupport,
- options, testmonitor, false, false);
+ options, testmonitor, false, false, SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE));
stats = new VersionGCStats();
stats.limitExceeded = true;
rec.evaluate(stats);
@@ -437,7 +438,7 @@ public class VersionGCTest {
deletedCount -= deleted;
localgcsupport = fakeVersionGCSupport(ns.getDocumentStore(), oldestDeleted, deletedCount);
rec = new VersionGCRecommendations(secondsPerDay, ns.getCheckpoints(), true, ns.getClock(), localgcsupport,
- options, testmonitor, false, false);
+ options, testmonitor, false, false, SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE));
stats = new VersionGCStats();
stats.limitExceeded = false;
stats.deletedDocGCCount = deleted;
@@ -464,7 +465,7 @@ public class VersionGCTest {
}, false, false, false);
// run first RGC
- gc.gc(1, TimeUnit.HOURS);
+ FullGCHelper.gc(gc, 1, TimeUnit.HOURS);
// afterwards there should be no more calls to getDeletedOnceCount()
deletedOnceCountCalls.set(0);
@@ -472,7 +473,7 @@ public class VersionGCTest {
for (int i = 0; i < 10; i++) {
advanceClock(5, SECONDS);
- gc.gc(1, TimeUnit.HOURS);
+ FullGCHelper.gc(gc, 1, TimeUnit.HOURS);
assertEquals(0, deletedOnceCountCalls.get());
}
}
@@ -481,7 +482,7 @@ public class VersionGCTest {
@Test
public void testFullGCDocumentRead_disabled() throws Exception {
disableFullGC(gc);
- VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
+ VersionGCStats stats = FullGCHelper.gc(gc, 30, TimeUnit.MINUTES);
assertNotNull(stats);
assertEquals(0, stats.fullGCDocsElapsed);
}
@@ -489,7 +490,8 @@ public class VersionGCTest {
@Test
public void testFullGCDocumentRead_enabled() throws Exception {
enableFullGC(gc);
- VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
+ gc.setFullGcMaxAge(30, MINUTES);
+ VersionGCStats stats = FullGCHelper.gc(gc, 30, TimeUnit.MINUTES);
assertNotNull(stats);
assertNotEquals(0, stats.fullGCDocsElapsed);
}
@@ -501,7 +503,7 @@ public class VersionGCTest {
public void testFullGCDryRunModeEnabled() throws Exception {
enableFullGC(gc);
enableFullGCDryRun(gc);
- VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
+ VersionGCStats stats = FullGCHelper.gc(gc, 30, TimeUnit.MINUTES);
assertNotNull(stats);
assertTrue(stats.fullGCDryRunMode);
}
@@ -510,7 +512,7 @@ public class VersionGCTest {
public void testResetFullGCDryRunMode() throws Exception {
enableFullGC(gc);
enableFullGCDryRun(gc);
- VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
+ VersionGCStats stats = FullGCHelper.gc(gc, 30, TimeUnit.MINUTES);
assertNotNull(stats);
// add dryRun fields data
@@ -539,7 +541,7 @@ public class VersionGCTest {
public void testVGCWithBatchSizeSmallerThanProgressSize() throws IllegalAccessException {
VersionGarbageCollector vgc = new VersionGarbageCollector(
ns, new VersionGCSupport(store), true, false, false,
- 0, 0, 1000, 5000);
+ 0, 0, 1000, 5000, TimeUnit.SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE));
assertEquals(1000, readDeclaredField(vgc, "fullGCBatchSize", true));
assertEquals(5000, readDeclaredField(vgc, "fullGCProgressSize", true));
@@ -549,7 +551,7 @@ public class VersionGCTest {
public void testVGCWithBatchSizeGreaterThanProgressSize() throws IllegalAccessException {
VersionGarbageCollector vgc = new VersionGarbageCollector(
ns, new VersionGCSupport(store), true, false, false,
- 0, 0, 20000, 15000);
+ 0, 0, 20000, 15000, TimeUnit.SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE));
assertEquals(15000, readDeclaredField(vgc, "fullGCBatchSize", true));
assertEquals(15000, readDeclaredField(vgc, "fullGCProgressSize", true));
@@ -570,7 +572,7 @@ public class VersionGCTest {
// reinitialize VersionGarbageCollector with not allowed value
VersionGarbageCollector gc = new VersionGarbageCollector(
ns, new VersionGCSupport(store), true, false, false,
- fullGcModeNotAllowedValue, 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE);
+ fullGcModeNotAllowedValue, 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE, TimeUnit.SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE));
assertEquals("Starting VersionGarbageCollector with not applicable / not allowed value" +
"will set fullGcMode to default NONE", VersionGarbageCollector.FullGCMode.NONE, VersionGarbageCollector.getFullGcMode());
@@ -581,7 +583,7 @@ public class VersionGCTest {
int fullGcModeNone = 0;
VersionGarbageCollector gc = new VersionGarbageCollector(
ns, new VersionGCSupport(store), true, false, false,
- fullGcModeNone, 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE);
+ fullGcModeNone, 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE, TimeUnit.SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE));
assertEquals(VersionGarbageCollector.FullGCMode.NONE, VersionGarbageCollector.getFullGcMode());
}
@@ -591,7 +593,7 @@ public class VersionGCTest {
int fullGcModeGapOrphans = 2;
VersionGarbageCollector gc = new VersionGarbageCollector(
ns, new VersionGCSupport(store), true, false, false,
- fullGcModeGapOrphans, 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE);
+ fullGcModeGapOrphans, 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE, TimeUnit.SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE));
assertEquals(VersionGarbageCollector.FullGCMode.GAP_ORPHANS, VersionGarbageCollector.getFullGcMode());
}
@@ -601,7 +603,7 @@ public class VersionGCTest {
int fullGcModeGapOrphansEmptyProperties = 3;
VersionGarbageCollector gc = new VersionGarbageCollector(
ns, new VersionGCSupport(store), true, false, false,
- fullGcModeGapOrphansEmptyProperties, 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE);
+ fullGcModeGapOrphansEmptyProperties, 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE, TimeUnit.SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE));
assertEquals(VersionGarbageCollector.FullGCMode.GAP_ORPHANS_EMPTYPROPS, VersionGarbageCollector.getFullGcMode());
}
@@ -613,7 +615,7 @@ public class VersionGCTest {
return execService.submit(new Callable<VersionGCStats>() {
@Override
public VersionGCStats call() throws Exception {
- return gc.gc(30, TimeUnit.MINUTES);
+ return FullGCHelper.gc(gc, 30, TimeUnit.MINUTES);
}
});
}
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
index 08aaea9cac..a2ad70d691 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
@@ -64,10 +64,12 @@ import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
import static org.apache.jackrabbit.oak.plugins.document.Collection.SETTINGS;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FGC_BATCH_SIZE;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FGC_PROGRESS_SIZE;
+import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FULL_GC_MAX_AGE;
import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.assertBranchRevisionNotRemovedFromAllDocuments;
import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.assertBranchRevisionRemovedFromAllDocuments;
import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.enableFullGC;
import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.enableFullGCDryRun;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.gc;
import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.mergedBranchCommit;
import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.unmergedBranchCommit;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MIN_ID_VALUE;
@@ -461,15 +463,6 @@ public class VersionGarbageCollectorIT {
gcSplitDocsInternal("sub".repeat(120));
}
- private VersionGCStats gc(VersionGarbageCollector gc, long maxRevisionAge, TimeUnit unit) throws IOException {
- final VersionGCStats stats = gc.gc(maxRevisionAge, unit);
- if (stats.skippedFullGCDocsCount != 0) {
- (new Exception("here: " + stats.skippedFullGCDocsCount)).printStackTrace(System.out);
- }
- assertEquals(0, stats.skippedFullGCDocsCount);
- return stats;
- }
-
// OAK-10199
@Test
public void testFullGCNeedRepeat() throws Exception {
@@ -588,6 +581,43 @@ public class VersionGarbageCollectorIT {
assertTrue(stats.canceled);
}
+ // OAK-11433
+ @Test
+ public void testFullGCNotIgnoredButRevisionGCIgnoredForCheckpoint() throws Exception {
+ long expiryTime = 100, maxRevisionGCAge = 20;
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+
+ // create a bunch of garbage
+ NodeBuilder b1 = store1.getRoot().builder();
+ for (int i = 0; i < 100; i++) {
+ b1.child("c" + i).setProperty("test", "t", STRING);
+ }
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ for (int i = 0; i < 100; i++) {
+ b1.child("c" + i).removeProperty("test");
+ }
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ // create a checkpoint 1 hour in the past
+ String checkpoint = store1.checkpoint(TimeUnit.HOURS.toMillis(1));
+
+ // Fast forward time to future such that we are past the checkpoint
+ clock.waitUntil(clock.getTime() + expiryTime);
+ gc.setFullGcMaxAge(2, HOURS);
+ VersionGCStats stats = gc.gc(maxRevisionGCAge, TimeUnit.MILLISECONDS);
+
+ // FullGC should not be ignored
+ assertFalse("Full GC should be performed", stats.ignoredFullGCDueToCheckPoint);
+ // RevisionGC should be ignored
+ assertTrue("Revision GC should be ignored due to checkpoint", stats.ignoredGCDueToCheckPoint);
+ assertFalse(stats.canceled);
+ assertFalse(stats.needRepeat);
+ }
+
+ // OAK-11433 - END
+
@Test
public void testGCDeletedLongPathPropsInclExcl_excludes() throws Exception {
String longName = "p".repeat(PATH_LONG + 1);
@@ -1691,7 +1721,7 @@ public class VersionGarbageCollectorIT {
//3. Check that deleted property does get collected post maxAge
clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
- stats = gcRef.get().gc(maxAge*2, HOURS);
+ stats = gc(gcRef.get(), maxAge*2, HOURS);
assertTrue(stats.canceled);
assertEquals(0, stats.updatedFullGCDocsCount);
assertEquals(0, stats.deletedPropsCount);
@@ -1759,7 +1789,7 @@ public class VersionGarbageCollectorIT {
}
};
- gcRef.set(new VersionGarbageCollector(store1, gcSupport, true, false, false, 3, 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE));
+ gcRef.set(new VersionGarbageCollector(store1, gcSupport, true, false, false, 3, 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE, TimeUnit.SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE)));
//3. Check that deleted property does get collected post maxAge
clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
@@ -1769,7 +1799,7 @@ public class VersionGarbageCollectorIT {
assertNotNull(document.get(SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP));
assertNotNull(document.get(SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP));
- stats = gcRef.get().gc(maxAge*2, HOURS);
+ stats = gc(gcRef.get(), maxAge*2, HOURS);
document = store1.getDocumentStore().find(SETTINGS, SETTINGS_COLLECTION_ID);
assertEquals(5, stats.updatedFullGCDocsCount);
@@ -4183,11 +4213,10 @@ public class VersionGarbageCollectorIT {
* each one in a separate merge
* @param orphans the nodes that should be created inproperly -
* each one in a separate late-write way
- * @param expectedNumOrphanedDocs the expected number of orphan documents that
- * FullGC should cleanup
* @param unrelatedPath an unrelated path that should be merged after
* late-write - ensures lastRev is updated on
* root to allow detecting late-writes as such
+ * @param counts the expected counts of deleted documents array
*/
private void doLateWriteCreateChildrenGC(Collection<String> parents,
Collection<String> orphans, String unrelatedPath, GCCounts... counts)
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilderTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilderTest.java
index f7577454b1..1aa9d47c6d 100755
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilderTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilderTest.java
@@ -114,4 +114,11 @@ public class RDBDocumentNodeStoreBuilderTest {
builder.setFullGCMode(3);
assertEquals(0, builder.getFullGCMode());
}
+
+ @Test
+ public void fullGcMaxAgeInSecsHasDefaultValue() {
+ RDBDocumentNodeStoreBuilder builder = new RDBDocumentNodeStoreBuilder();
+ builder.setFullGcMaxAgeMillis(30 * 24 * 60 * 60 * 1000L);
+ assertEquals(0, builder.getFullGcMaxAgeMillis());
+ }
}