You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by da...@apache.org on 2023/04/26 15:47:45 UTC
[jackrabbit-oak] 03/04: OAK-10199 : provided support for feature toggle & osgi config for detailed gc
This is an automated email from the ASF dual-hosted git repository.
daim pushed a commit to branch OAK-10199
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 5024987989903ee76e39cdc2eea4fc5d4c123725
Author: Rishabh Kumar <di...@adobe.com>
AuthorDate: Mon Apr 24 13:07:44 2023 +0530
OAK-10199 : provided support for feature toggle & osgi config for detailed gc
---
.gitignore | 1 +
.../plugins/document/DocumentNodeStoreHelper.java | 4 +-
.../jackrabbit/oak/run/RevisionsCommand.java | 3 +-
.../oak/plugins/document/Configuration.java | 9 +
.../oak/plugins/document/DocumentNodeStore.java | 3 +-
.../plugins/document/DocumentNodeStoreBuilder.java | 21 +++
.../plugins/document/DocumentNodeStoreService.java | 10 ++
.../oak/plugins/document/NodeDocument.java | 16 ++
.../plugins/document/VersionGCRecommendations.java | 43 ++++-
.../oak/plugins/document/VersionGCSupport.java | 98 ++++++-----
.../plugins/document/VersionGarbageCollector.java | 183 ++++++++++++---------
.../document/mongo/MongoVersionGCSupport.java | 42 ++++-
.../oak/plugins/document/util/Utils.java | 11 ++
.../DocumentNodeStoreServiceConfigurationTest.java | 10 ++
.../oak/plugins/document/VersionGCQueryTest.java | 4 +-
.../oak/plugins/document/VersionGCTest.java | 11 +-
.../document/VersionGarbageCollectorIT.java | 8 +-
.../mongo/MongoDocumentNodeStoreBuilderTest.java | 12 ++
.../oak/plugins/document/util/UtilsTest.java | 41 ++++-
19 files changed, 385 insertions(+), 145 deletions(-)
diff --git a/.gitignore b/.gitignore
index faa5bb5b11..210e226f41 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,3 +10,4 @@ oak-run/*.csv
atlassian-ide-plugin.xml
.checkstyle
derby.log
+.java-version
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
index 03fb649ef2..d401c8cd0a 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
@@ -72,8 +72,8 @@ public class DocumentNodeStoreHelper {
}
public static VersionGarbageCollector createVersionGC(
- DocumentNodeStore nodeStore, VersionGCSupport gcSupport) {
- return new VersionGarbageCollector(nodeStore, gcSupport);
+ DocumentNodeStore nodeStore, VersionGCSupport gcSupport, final boolean detailedGCEnabled) {
+ return new VersionGarbageCollector(nodeStore, gcSupport, detailedGCEnabled);
}
private static Iterable<BlobReferences> scan(DocumentNodeStore store,
diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
index 21e5aff706..c9cc6b3d4d 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
@@ -60,6 +60,7 @@ import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreHelper.createVersionGC;
import static org.apache.jackrabbit.oak.plugins.document.FormatVersion.versionOf;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getRootDocument;
+import static org.apache.jackrabbit.oak.plugins.document.util.Utils.isDetailedGCEnabled;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.timestampToString;
import static org.apache.jackrabbit.oak.run.Utils.asCloseable;
import static org.apache.jackrabbit.oak.run.Utils.createDocumentMKBuilder;
@@ -226,7 +227,7 @@ public class RevisionsCommand implements Command {
useMemoryBlobStore(builder);
// create a version GC that operates on a read-only DocumentNodeStore
// and a GC support with a writable DocumentStore
- VersionGarbageCollector gc = createVersionGC(builder.build(), gcSupport);
+ VersionGarbageCollector gc = createVersionGC(builder.build(), gcSupport, isDetailedGCEnabled(builder));
VersionGCOptions gcOptions = gc.getOptions();
gcOptions = gcOptions.withDelayFactor(options.getDelay());
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Configuration.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Configuration.java
index fc5043de49..0bca9397ef 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Configuration.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Configuration.java
@@ -31,6 +31,7 @@ import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilde
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder.DEFAULT_NODE_CACHE_PERCENTAGE;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder.DEFAULT_PREV_DOC_CACHE_PERCENTAGE;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder.DEFAULT_UPDATE_LIMIT;
+import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_DETAILED_GC_ENABLED;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_THROTTLING_ENABLED;
@ObjectClassDefinition(
@@ -281,4 +282,12 @@ import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreServic
name = "Document Node Store Compression",
description = "Select compressor type for collections. 'Snappy' is the default supported compression.")
String collectionCompressionType() default "snappy";
+
+ @AttributeDefinition(
+ name = "Document Node Store Detailed GC",
+ description = "Boolean value indicating whether Detailed GC should be enabled for " +
+ "document node store or not. The Default value is " + DEFAULT_DETAILED_GC_ENABLED +
+ ". Note that this value can be overridden via framework " +
+ "property 'oak.documentstore.detailedGCEnabled'")
+ boolean detailedGCEnabled() default DEFAULT_DETAILED_GC_ENABLED;
}
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
index e0809ccd1a..1117b0a4d9 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
@@ -38,6 +38,7 @@ import static org.apache.jackrabbit.oak.plugins.document.Path.ROOT;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.alignWithExternalRevisions;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getIdFromPath;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getModuleVersion;
+import static org.apache.jackrabbit.oak.plugins.document.util.Utils.isDetailedGCEnabled;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.isThrottlingEnabled;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.pathToId;
import static org.apache.jackrabbit.oak.spi.observation.ChangeSet.COMMIT_CONTEXT_OBSERVATION_CHANGESET;
@@ -641,7 +642,7 @@ public final class DocumentNodeStore
this.branches = new UnmergedBranches();
this.asyncDelay = builder.getAsyncDelay();
this.versionGarbageCollector = new VersionGarbageCollector(
- this, builder.createVersionGCSupport());
+ this, builder.createVersionGCSupport(), isDetailedGCEnabled(builder));
this.versionGarbageCollector.setStatisticsProvider(builder.getStatisticsProvider());
this.versionGarbageCollector.setGCMonitor(builder.getGCMonitor());
this.journalGarbageCollector = new JournalGarbageCollector(
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java
index 27ee381b7a..009b54bc84 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java
@@ -124,6 +124,7 @@ public class DocumentNodeStoreBuilder<T extends DocumentNodeStoreBuilder<T>> {
private boolean isReadOnlyMode = false;
private Feature prefetchFeature;
private Feature docStoreThrottlingFeature;
+ private Feature docStoreDetailedGCFeature;
private Weigher<CacheValue, CacheValue> weigher = new EmpiricalWeigher();
private long memoryCacheSize = DEFAULT_MEMORY_CACHE_SIZE;
private int nodeCachePercentage = DEFAULT_NODE_CACHE_PERCENTAGE;
@@ -162,6 +163,7 @@ public class DocumentNodeStoreBuilder<T extends DocumentNodeStoreBuilder<T>> {
private Predicate<Path> nodeCachePredicate = Predicates.alwaysTrue();
private boolean clusterInvisible;
private boolean throttlingEnabled;
+ private boolean detailedGCEnabled;
/**
* @return a new {@link DocumentNodeStoreBuilder}.
@@ -284,6 +286,15 @@ public class DocumentNodeStoreBuilder<T extends DocumentNodeStoreBuilder<T>> {
return this.throttlingEnabled;
}
+ public T setDetailedGCEnabled(boolean b) {
+ this.detailedGCEnabled = b;
+ return thisBuilder();
+ }
+
+ public boolean isDetailedGCEnabled() {
+ return this.detailedGCEnabled;
+ }
+
public T setReadOnlyMode() {
this.isReadOnlyMode = true;
return thisBuilder();
@@ -313,6 +324,16 @@ public class DocumentNodeStoreBuilder<T extends DocumentNodeStoreBuilder<T>> {
return docStoreThrottlingFeature;
}
+ public T setDocStoreDetailedGCFeature(@Nullable Feature docStoreDetailedGC) {
+ this.docStoreDetailedGCFeature = docStoreDetailedGC;
+ return thisBuilder();
+ }
+
+ @Nullable
+ public Feature getDocStoreDetailedGCFeature() {
+ return docStoreDetailedGCFeature;
+ }
+
public T setLeaseFailureHandler(LeaseFailureHandler leaseFailureHandler) {
this.leaseFailureHandler = leaseFailureHandler;
return thisBuilder();
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
index 4b9ff4a0ad..65e8c6956c 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
@@ -138,6 +138,7 @@ public class DocumentNodeStoreService {
static final String DEFAULT_DB = "oak";
static final boolean DEFAULT_SO_KEEP_ALIVE = true;
static final boolean DEFAULT_THROTTLING_ENABLED = false;
+ static final boolean DEFAULT_DETAILED_GC_ENABLED = false;
static final int DEFAULT_MONGO_LEASE_SO_TIMEOUT_MILLIS = 30000;
static final String DEFAULT_PERSISTENT_CACHE = "cache";
static final String DEFAULT_JOURNAL_CACHE = "diff-cache";
@@ -181,6 +182,11 @@ public class DocumentNodeStoreService {
*/
private static final String FT_NAME_DOC_STORE_THROTTLING = "FT_THROTTLING_OAK-9909";
+ /**
+ * Feature toggle name to enable detailed GC for Mongo Document Store
+ */
+ private static final String FT_NAME_DEATILED_GC = "FT_DETAILED_GC_OAK-10199";
+
// property name constants - values can come from framework properties or OSGi config
public static final String CUSTOM_BLOB_STORE = "customBlobStore";
public static final String PROP_REV_RECOVERY_INTERVAL = "lastRevRecoveryJobIntervalInSecs";
@@ -216,6 +222,7 @@ public class DocumentNodeStoreService {
private JournalPropertyHandlerFactory journalPropertyHandlerFactory = new JournalPropertyHandlerFactory();
private Feature prefetchFeature;
private Feature docStoreThrottlingFeature;
+ private Feature docStoreDetailedGCFeature;
private ComponentContext context;
private Whiteboard whiteboard;
private long deactivationTimestamp = 0;
@@ -250,6 +257,7 @@ public class DocumentNodeStoreService {
documentStoreType = DocumentStoreType.fromString(this.config.documentStoreType());
prefetchFeature = Feature.newFeature(FT_NAME_PREFETCH, whiteboard);
docStoreThrottlingFeature = Feature.newFeature(FT_NAME_DOC_STORE_THROTTLING, whiteboard);
+ docStoreDetailedGCFeature = Feature.newFeature(FT_NAME_DEATILED_GC, whiteboard);
registerNodeStoreIfPossible();
}
@@ -465,7 +473,9 @@ public class DocumentNodeStoreService {
setLeaseCheckMode(ClusterNodeInfo.DEFAULT_LEASE_CHECK_DISABLED ? LeaseCheckMode.DISABLED : LeaseCheckMode.valueOf(config.leaseCheckMode())).
setPrefetchFeature(prefetchFeature).
setDocStoreThrottlingFeature(docStoreThrottlingFeature).
+ setDocStoreDetailedGCFeature(docStoreDetailedGCFeature).
setThrottlingEnabled(config.throttlingEnabled()).
+ setDetailedGCEnabled(config.detailedGCEnabled()).
setLeaseFailureHandler(new LeaseFailureHandler() {
private final LeaseFailureHandler defaultLeaseFailureHandler = createDefaultLeaseFailureHandler();
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java
index c79c6adfc4..71abba0a2e 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java
@@ -32,6 +32,7 @@ import java.util.SortedSet;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
import org.apache.jackrabbit.guava.common.base.Function;
import org.apache.jackrabbit.guava.common.base.Predicate;
@@ -66,6 +67,7 @@ import static org.apache.jackrabbit.guava.common.collect.Iterables.filter;
import static org.apache.jackrabbit.guava.common.collect.Iterables.mergeSorted;
import static org.apache.jackrabbit.guava.common.collect.Iterables.transform;
import static java.util.Objects.requireNonNull;
+import static java.util.stream.Collectors.toMap;
import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
import static org.apache.jackrabbit.oak.plugins.document.StableRevisionComparator.REVERSE;
import static org.apache.jackrabbit.oak.plugins.document.UpdateOp.Key;
@@ -1669,6 +1671,20 @@ public final class NodeDocument extends Document {
return map;
}
+ /**
+ * Returns all the properties on this document
+ * @return Map of all properties along with their values
+ */
+ @NotNull
+ Map<String, SortedMap<Revision, String>> getProperties() {
+ return data
+ .keySet()
+ .stream()
+ .filter(Utils::isPropertyName)
+ .map(o -> Map.entry(o, getLocalMap(o)))
+ .collect(toMap(Entry::getKey, Entry::getValue));
+ }
+
/**
* @return the {@link #REVISIONS} stored on this document.
*/
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java
index ac47cc69d8..d8b091261d 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java
@@ -21,6 +21,7 @@ package org.apache.jackrabbit.oak.plugins.document;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import com.google.common.collect.ImmutableMap;
import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCStats;
import org.apache.jackrabbit.oak.plugins.document.util.TimeInterval;
import org.apache.jackrabbit.oak.plugins.document.util.Utils;
@@ -31,6 +32,9 @@ import org.slf4j.LoggerFactory;
import org.apache.jackrabbit.guava.common.collect.Maps;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP;
+
/**
* Gives a recommendation about parameters for the next revision garbage collection run.
*/
@@ -43,6 +47,7 @@ public class VersionGCRecommendations {
final boolean ignoreDueToCheckPoint;
final TimeInterval scope;
+ final TimeInterval scopeFullGC;
final long maxCollect;
final long deleteCandidateCount;
final long lastOldestTimestamp;
@@ -81,6 +86,7 @@ public class VersionGCRecommendations {
long deletedOnceCount = 0;
long suggestedIntervalMs;
long oldestPossible;
+ long oldestPossibleFullGC;
long collectLimit = options.collectLimit;
this.vgc = vgc;
@@ -90,7 +96,7 @@ public class VersionGCRecommendations {
TimeInterval keep = new TimeInterval(clock.getTime() - maxRevisionAgeMs, Long.MAX_VALUE);
Map<String, Long> settings = getLongSettings();
- lastOldestTimestamp = settings.get(VersionGarbageCollector.SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP);
+ lastOldestTimestamp = settings.get(SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP);
if (lastOldestTimestamp == 0) {
log.debug("No lastOldestTimestamp found, querying for the oldest deletedOnce candidate");
oldestPossible = vgc.getOldestDeletedOnceTimestamp(clock, options.precisionMs) - 1;
@@ -102,7 +108,21 @@ public class VersionGCRecommendations {
TimeInterval scope = new TimeInterval(oldestPossible, Long.MAX_VALUE);
scope = scope.notLaterThan(keep.fromMs);
- fullDetailGCTimestamp = settings.get(VersionGarbageCollector.SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP);
+ fullDetailGCTimestamp = settings.get(SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP);
+ if (fullDetailGCTimestamp == 0) {
+ if (log.isDebugEnabled()) {
+ log.debug("No fullDetailGCTimestamp found, querying for the oldest deletedOnce candidate");
+ }
+ oldestPossibleFullGC = vgc.getOldestModifiedTimestamp(clock) - 1;
+ if (log.isDebugEnabled()) {
+ log.debug("fullDetailGCTimestamp found: {}", Utils.timestampToString(oldestPossibleFullGC));
+ }
+ } else {
+ oldestPossibleFullGC = fullDetailGCTimestamp - 1;
+ }
+
+ TimeInterval scopeFullGC = new TimeInterval(oldestPossibleFullGC, Long.MAX_VALUE);
+ scopeFullGC = scopeFullGC.notLaterThan(keep.fromMs);
suggestedIntervalMs = settings.get(VersionGarbageCollector.SETTINGS_COLLECTION_REC_INTERVAL_PROP);
if (suggestedIntervalMs > 0) {
@@ -162,6 +182,7 @@ public class VersionGCRecommendations {
this.precisionMs = options.precisionMs;
this.ignoreDueToCheckPoint = ignoreDueToCheckPoint;
this.scope = scope;
+ this.scopeFullGC = scopeFullGC;
this.scopeIsComplete = scope.toMs >= keep.fromMs;
this.maxCollect = collectLimit;
this.suggestedIntervalMs = suggestedIntervalMs;
@@ -185,7 +206,10 @@ public class VersionGCRecommendations {
stats.needRepeat = true;
} else if (!stats.canceled && !stats.ignoredGCDueToCheckPoint) {
// success, we would not expect to encounter revisions older than this in the future
- setLongSetting(VersionGarbageCollector.SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP, scope.toMs);
+// setLongSetting(SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP, scope.toMs);
+// setLongSetting(SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP, stats.oldestModifiedGced);
+ setLongSetting(ImmutableMap.of(SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP, scope.toMs,
+ SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP, stats.oldestModifiedGced));
int count = stats.deletedDocGCCount - stats.deletedLeafDocGCCount;
double usedFraction;
@@ -218,9 +242,9 @@ public class VersionGCRecommendations {
Document versionGCDoc = vgc.getDocumentStore().find(Collection.SETTINGS, VersionGarbageCollector.SETTINGS_COLLECTION_ID, 0);
Map<String, Long> settings = Maps.newHashMap();
// default values
- settings.put(VersionGarbageCollector.SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP, 0L);
+ settings.put(SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP, 0L);
settings.put(VersionGarbageCollector.SETTINGS_COLLECTION_REC_INTERVAL_PROP, 0L);
- settings.put(VersionGarbageCollector.SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP, -1L);
+ settings.put(SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP, 0L);
if (versionGCDoc != null) {
for (String k : versionGCDoc.keySet()) {
Object value = versionGCDoc.get(k);
@@ -233,8 +257,15 @@ public class VersionGCRecommendations {
}
void setLongSetting(String propName, long val) {
+ setLongSetting(Map.of(propName, val));
+// UpdateOp updateOp = new UpdateOp(VersionGarbageCollector.SETTINGS_COLLECTION_ID, true);
+// updateOp.set(propName, val);
+// vgc.getDocumentStore().createOrUpdate(Collection.SETTINGS, updateOp);
+ }
+
+ void setLongSetting(final Map<String, Long> propValMap) {
UpdateOp updateOp = new UpdateOp(VersionGarbageCollector.SETTINGS_COLLECTION_ID, true);
- updateOp.set(propName, val);
+ propValMap.forEach(updateOp::set);
vgc.getDocumentStore().createOrUpdate(Collection.SETTINGS, updateOp);
}
}
\ No newline at end of file
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupport.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupport.java
index 0e5c26c83d..f23340acbc 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupport.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupport.java
@@ -20,11 +20,14 @@
package org.apache.jackrabbit.oak.plugins.document;
import static org.apache.jackrabbit.guava.common.collect.Iterables.filter;
+import static java.util.stream.Collectors.toList;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MODIFIED_IN_SECS;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.getModifiedInSecs;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getAllDocuments;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getSelectedDocuments;
import java.util.Set;
+import java.util.stream.StreamSupport;
import org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType;
import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCStats;
@@ -59,53 +62,40 @@ public class VersionGCSupport {
* @param toModified the upper bound modified timestamp (exclusive)
* @return matching documents.
*/
- public Iterable<NodeDocument> getPossiblyDeletedDocs(final long fromModified,
- final long toModified) {
- return filter(getSelectedDocuments(store, NodeDocument.DELETED_ONCE, 1), new Predicate<NodeDocument>() {
- @Override
- public boolean apply(NodeDocument input) {
- return input.wasDeletedOnce()
- && modifiedGreaterThanEquals(input, fromModified)
- && modifiedLessThan(input, toModified);
- }
-
- private boolean modifiedGreaterThanEquals(NodeDocument doc,
- long time) {
- Long modified = doc.getModified();
- return modified != null && modified.compareTo(getModifiedInSecs(time)) >= 0;
- }
-
- private boolean modifiedLessThan(NodeDocument doc,
- long time) {
- Long modified = doc.getModified();
- return modified != null && modified.compareTo(getModifiedInSecs(time)) < 0;
- }
- });
+ public Iterable<NodeDocument> getPossiblyDeletedDocs(final long fromModified, final long toModified) {
+ return StreamSupport
+ .stream(getSelectedDocuments(store, NodeDocument.DELETED_ONCE, 1).spliterator(), false)
+ .filter(input -> input.wasDeletedOnce() && modifiedGreaterThanEquals(input, fromModified) && modifiedLessThan(input, toModified))
+ .collect(toList());
}
/**
- * TODO: document me!
+ * Returns documents that have a {@link NodeDocument#MODIFIED_IN_SECS} value
+ * within the given range .The two passed modified timestamps are in milliseconds
+ * since the epoch and the implementation will convert them to seconds at
+ * the granularity of the {@link NodeDocument#MODIFIED_IN_SECS} field and
+ * then perform the comparison.
+ *
+ * @param fromModified the lower bound modified timestamp (inclusive)
+ * @param toModified the upper bound modified timestamp (exclusive)
+ * @param limit the limit of documents to return
+ * @return matching documents.
*/
- public Iterable<NodeDocument> getModifiedDocs(final long fromModified, final long toModified) {
- return filter(getSelectedDocuments(store, NodeDocument.MODIFIED_IN_SECS, fromModified), new Predicate<NodeDocument>() {
- @Override
- public boolean apply(NodeDocument input) {
- return modifiedGreaterThanEquals(input, fromModified)
- && modifiedLessThan(input, toModified);
- }
-
- private boolean modifiedGreaterThanEquals(NodeDocument doc,
- long time) {
- Long modified = doc.getModified();
- return modified != null && modified.compareTo(getModifiedInSecs(time)) >= 0;
- }
+ public Iterable<NodeDocument> getModifiedDocs(final long fromModified, final long toModified, final int limit) {
+ return StreamSupport
+ .stream(getSelectedDocuments(store, MODIFIED_IN_SECS, fromModified).spliterator(), false)
+ .filter(input -> modifiedGreaterThanEquals(input, fromModified) && modifiedLessThan(input, toModified))
+ .limit(limit)
+ .collect(toList());
+ }
- private boolean modifiedLessThan(NodeDocument doc,
- long time) {
- Long modified = doc.getModified();
- return modified != null && modified.compareTo(getModifiedInSecs(time)) < 0;
- }
- });
+ private boolean modifiedGreaterThanEquals(final NodeDocument doc, final long time) {
+ Long modified = doc.getModified();
+ return modified != null && modified.compareTo(getModifiedInSecs(time)) >= 0;
+ }
+ private boolean modifiedLessThan(final NodeDocument doc, final long time) {
+ Long modified = doc.getModified();
+ return modified != null && modified.compareTo(getModifiedInSecs(time)) < 0;
}
/**
@@ -185,6 +175,30 @@ public class VersionGCSupport {
return ts;
}
+ /**
+ * Retrieve the time of the oldest modified document.
+ *
+ * @return the timestamp of the oldest modified document.
+ */
+ public long getOldestModifiedTimestamp(final Clock clock) {
+ long ts = 0;
+ long now = clock.getTime();
+ Iterable<NodeDocument> docs = null;
+
+ LOG.info("find oldest modified document");
+ try {
+ docs = getModifiedDocs(ts, now, 1);
+ if (docs.iterator().hasNext()) {
+ Long modified = docs.iterator().next().getModified();
+ return modified != null ? modified : 0L;
+ }
+ } finally {
+ Utils.closeIfCloseable(docs);
+ }
+ LOG.info("find oldest modified document to be {}", Utils.timestampToString(ts));
+ return ts;
+ }
+
public long getDeletedOnceCount() throws UnsupportedOperationException {
throw new UnsupportedOperationException("getDeletedOnceCount()");
}
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
index e7442a7d15..608ba02398 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
@@ -27,6 +27,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.SortedMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
@@ -54,7 +55,7 @@ import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
import static org.apache.jackrabbit.guava.common.base.StandardSystemProperty.LINE_SEPARATOR;
import static org.apache.jackrabbit.guava.common.collect.Iterables.all;
import static org.apache.jackrabbit.guava.common.collect.Iterators.partition;
@@ -66,6 +67,7 @@ import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MODIFIED_I
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType.COMMIT_ROOT_ONLY;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType.DEFAULT_LEAF;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType.DEFAULT_NO_BRANCH;
+import static org.apache.jackrabbit.oak.plugins.document.StableRevisionComparator.REVERSE;
import static org.slf4j.helpers.MessageFormatter.arrayFormat;
public class VersionGarbageCollector {
@@ -115,6 +117,7 @@ public class VersionGarbageCollector {
private final DocumentNodeStore nodeStore;
private final DocumentStore ds;
+ private final boolean detailedGCEnabled;
private final VersionGCSupport versionStore;
private final AtomicReference<GCJob> collector = newReference();
private VersionGCOptions options;
@@ -122,10 +125,12 @@ public class VersionGarbageCollector {
private RevisionGCStats gcStats = new RevisionGCStats(StatisticsProvider.NOOP);
VersionGarbageCollector(DocumentNodeStore nodeStore,
- VersionGCSupport gcSupport) {
+ VersionGCSupport gcSupport,
+ final boolean detailedGCEnabled) {
this.nodeStore = nodeStore;
this.versionStore = gcSupport;
this.ds = gcSupport.getDocumentStore();
+ this.detailedGCEnabled = detailedGCEnabled;
this.options = new VersionGCOptions();
}
@@ -201,7 +206,7 @@ public class VersionGarbageCollector {
}
public void setGCMonitor(@NotNull GCMonitor gcMonitor) {
- this.gcMonitor = checkNotNull(gcMonitor);
+ this.gcMonitor = requireNonNull(gcMonitor);
}
public VersionGCOptions getOptions() {
@@ -259,6 +264,7 @@ public class VersionGarbageCollector {
}
public static class VersionGCStats {
+ public long oldestModifiedGced;
boolean ignoredGCDueToCheckPoint;
boolean canceled;
boolean success = true;
@@ -274,14 +280,14 @@ public class VersionGarbageCollector {
final Stopwatch active = Stopwatch.createUnstarted();
final Stopwatch collectDeletedDocs = Stopwatch.createUnstarted();
final Stopwatch checkDeletedDocs = Stopwatch.createUnstarted();
- final Stopwatch detailGcDocs = Stopwatch.createUnstarted();
+ final Stopwatch detailedGcDocs = Stopwatch.createUnstarted();
final Stopwatch deleteDeletedDocs = Stopwatch.createUnstarted();
final Stopwatch collectAndDeleteSplitDocs = Stopwatch.createUnstarted();
final Stopwatch deleteSplitDocs = Stopwatch.createUnstarted();
final Stopwatch sortDocIds = Stopwatch.createUnstarted();
final Stopwatch updateResurrectedDocuments = Stopwatch.createUnstarted();
long activeElapsed, collectDeletedDocsElapsed, checkDeletedDocsElapsed, deleteDeletedDocsElapsed, collectAndDeleteSplitDocsElapsed,
- deleteSplitDocsElapsed, sortDocIdsElapsed, updateResurrectedDocumentsElapsed, detailGcDocsElapsed;
+ deleteSplitDocsElapsed, sortDocIdsElapsed, updateResurrectedDocumentsElapsed, detailedGcDocsElapsed;
@Override
public String toString() {
@@ -318,6 +324,7 @@ public class VersionGarbageCollector {
return "VersionGCStats{" +
"ignoredGCDueToCheckPoint=" + ignoredGCDueToCheckPoint +
+ ", oldestModifiedGced=" + oldestModifiedGced +
", canceled=" + canceled +
", deletedDocGCCount=" + deletedDocGCCount + " (of which leaf: " + deletedLeafDocGCCount + ")" +
", updateResurrectedGCCount=" + updateResurrectedGCCount +
@@ -331,6 +338,7 @@ public class VersionGarbageCollector {
void addRun(VersionGCStats run) {
++iterationCount;
this.ignoredGCDueToCheckPoint = run.ignoredGCDueToCheckPoint;
+ this.oldestModifiedGced = run.oldestModifiedGced;
this.canceled = run.canceled;
this.success = run.success;
this.limitExceeded = run.limitExceeded;
@@ -350,7 +358,7 @@ public class VersionGarbageCollector {
this.deleteSplitDocsElapsed += run.deleteSplitDocsElapsed;
this.sortDocIdsElapsed += run.sortDocIdsElapsed;
this.updateResurrectedDocumentsElapsed += run.updateResurrectedDocumentsElapsed;
- this.detailGcDocsElapsed += run.detailGcDocsElapsed;
+ this.detailedGcDocsElapsed += run.detailedGcDocsElapsed;
} else {
// single run -> read from stop watches
this.activeElapsed += run.active.elapsed(MICROSECONDS);
@@ -361,7 +369,7 @@ public class VersionGarbageCollector {
this.deleteSplitDocsElapsed += run.deleteSplitDocs.elapsed(MICROSECONDS);
this.sortDocIdsElapsed += run.sortDocIds.elapsed(MICROSECONDS);
this.updateResurrectedDocumentsElapsed += run.updateResurrectedDocuments.elapsed(MICROSECONDS);
- this.detailGcDocsElapsed += run.detailGcDocs.elapsed(MICROSECONDS);
+ this.detailedGcDocsElapsed += run.detailedGcDocs.elapsed(MICROSECONDS);
}
}
}
@@ -370,7 +378,7 @@ public class VersionGarbageCollector {
NONE,
COLLECTING,
CHECKING,
- DETAILGC,
+ DETAILED_GC,
DELETING,
SORTING,
SPLITS_CLEANUP,
@@ -398,7 +406,7 @@ public class VersionGarbageCollector {
this.watches.put(GCPhase.NONE, Stopwatch.createStarted());
this.watches.put(GCPhase.COLLECTING, stats.collectDeletedDocs);
this.watches.put(GCPhase.CHECKING, stats.checkDeletedDocs);
- this.watches.put(GCPhase.DETAILGC, stats.detailGcDocs);
+ this.watches.put(GCPhase.DETAILED_GC, stats.detailedGcDocs);
this.watches.put(GCPhase.DELETING, stats.deleteDeletedDocs);
this.watches.put(GCPhase.SORTING, stats.sortDocIds);
this.watches.put(GCPhase.SPLITS_CLEANUP, stats.collectAndDeleteSplitDocs);
@@ -525,7 +533,10 @@ public class VersionGarbageCollector {
collectDeletedDocuments(phases, headRevision, rec);
collectSplitDocuments(phases, sweepRevisions, rec);
- collectDetailGarbage(phases, headRevision, rec);
+ if (detailedGCEnabled) {
+ // run only if enabled
+ collectDetailedGarbage(phases, headRevision, rec);
+ }
}
} catch (LimitExceededException ex) {
stats.limitExceeded = true;
@@ -555,36 +566,33 @@ public class VersionGarbageCollector {
* followed by voluntary paused (aka throttling) to avoid excessive load on the
* system. The full repository scan does not have to finish particularly fast,
* it is okay that it takes a considerable amount of time.
- *
- * @param headRevision
+ *
+ * @param phases {@link GCPhases}
+ * @param headRevision the current head revision of
* @throws IOException
* @throws LimitExceededException
*/
- private void collectDetailGarbage(GCPhases phases, RevisionVector headRevision, VersionGCRecommendations rec)
+ private void collectDetailedGarbage(final GCPhases phases, final RevisionVector headRevision, final VersionGCRecommendations rec)
throws IOException, LimitExceededException {
- if (!DETAIL_GC_ENABLED) {
- // TODO: this toggling should be done nicer asap
- return;
- }
int docsTraversed = 0;
- DetailGC gc = new DetailGC(headRevision, monitor);
- try {
- final long fromModified;
- final long toModified;
- if (rec.fullDetailGCTimestamp == -1) {
- // then full detail-gc is disabled or over - use regular scope then
- fromModified = rec.scope.fromMs;
- toModified = rec.scope.toMs;
- } else {
- // then full detail-gc is enabled - use it then
- fromModified = rec.fullDetailGCTimestamp; // TODO: once we're passed rec.scope.fromMs we should
- // disable fullgc
- toModified = rec.scope.toMs; // the 'to' here is the max. it will process only eg 1 batch
- }
- long oldestGced = fromModified;
- boolean foundAnything = false;
+ long oldestModifiedGced = rec.scopeFullGC.fromMs;
+ try (DetailedGC gc = new DetailedGC(headRevision, monitor, cancel)) {
+ final long fromModified = rec.scopeFullGC.fromMs;
+ final long toModified = rec.scopeFullGC.toMs;
+// if (rec.fullDetailGCTimestamp == -1) {
+// // then full detail-gc is disabled or over - use regular scope then
+// fromModified = rec.scope.fromMs;
+// toModified = rec.scope.toMs;
+// } else {
+// // then full detail-gc is enabled - use it then
+// fromModified = rec.fullDetailGCTimestamp; // TODO: once we're passed rec.scope.fromMs we should
+// // disable fullgc
+// toModified = rec.scope.toMs; // the 'to' here is the max. it will process only eg 1 batch
+// }
+ // TODO : remove me
+ boolean foundAnything = false; // I think this flag is redundant
if (phases.start(GCPhase.COLLECTING)) {
- Iterable<NodeDocument> itr = versionStore.getModifiedDocs(fromModified, toModified);
+ Iterable<NodeDocument> itr = versionStore.getModifiedDocs(oldestModifiedGced, toModified, 2000);
final Stopwatch timer = Stopwatch.createUnstarted();
timer.reset().start();
try {
@@ -593,44 +601,54 @@ public class VersionGarbageCollector {
if (cancel.get()) {
break;
}
- foundAnything = true;
- if (phases.start(GCPhase.DETAILGC)) {
- gc.detailGC(doc, phases);
- phases.stop(GCPhase.DETAILGC);
+ if (phases.start(GCPhase.DETAILED_GC)) {
+ gc.detailedGC(doc, phases);
+ phases.stop(GCPhase.DETAILED_GC);
}
+
+ // TODO : remove this code, I don't think its possible to fetch these documents
+ // who doesn't have _modified field
final Long modified = doc.getModified();
if (modified == null) {
monitor.warn("collectDetailGarbage : document has no _modified property : {}",
doc.getId());
- } else if (modified < oldestGced) {
+ } else if (modified < oldestModifiedGced) {
monitor.warn(
"collectDetailGarbage : document has older _modified than query boundary : {} (from: {}, to: {})",
modified, fromModified, toModified);
} else {
- oldestGced = modified;
+ oldestModifiedGced = modified;
}
+ foundAnything = true;
docsTraversed++;
if (docsTraversed % PROGRESS_BATCH_SIZE == 0) {
monitor.info("Iterated through {} documents so far. {} had detail garbage",
docsTraversed, gc.getNumDocuments());
}
+ // this would never hit, since we are only fetching the oldest 2000 element in batches of 1000
+ // TODO: remove this if above mentioned logic is fine
if (rec.maxCollect > 0 && gc.getNumDocuments() > rec.maxCollect) {
// TODO: how would we recover from this?
+ // If we don't want above solution, then one of the another solution is to use lower time duration
+ // as done in document deletion process or use lower limit value or
+ // we should perform all the update ops in 1 go
throw new LimitExceededException();
}
+ oldestModifiedGced = modified == null ? fromModified : modified;
}
} finally {
Utils.closeIfCloseable(itr);
+ // why do we need to stop this here, we are already stopping the original gc run.
+ // can this be removed
delayOnModifications(timer.stop().elapsed(TimeUnit.MILLISECONDS));
+ phases.stats.oldestModifiedGced = oldestModifiedGced;
}
phases.stop(GCPhase.COLLECTING);
- if (!cancel.get() && foundAnything) {
- // TODO: move to evaluate()
- rec.setLongSetting(SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP, oldestGced + 1);
- }
+// if (!cancel.get() && foundAnything) {
+// // TODO: move to evaluate()
+// rec.setLongSetting(SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP, oldestModifiedGced + 1);
+// }
}
- } finally {
- gc.close();
}
}
@@ -665,8 +683,7 @@ public class VersionGarbageCollector {
VersionGCRecommendations rec)
throws IOException, LimitExceededException {
int docsTraversed = 0;
- DeletedDocsGC gc = new DeletedDocsGC(headRevision, cancel, options, monitor);
- try {
+ try (DeletedDocsGC gc = new DeletedDocsGC(headRevision, cancel, options, monitor)) {
if (phases.start(GCPhase.COLLECTING)) {
Iterable<NodeDocument> itr = versionStore.getPossiblyDeletedDocs(rec.scope.fromMs, rec.scope.toMs);
try {
@@ -731,53 +748,69 @@ public class VersionGarbageCollector {
gc.updateResurrectedDocuments(phases.stats);
phases.stop(GCPhase.UPDATING);
}
- } finally {
- gc.close();
}
}
}
- private class DetailGC implements Closeable {
+ private static class DetailedGC implements Closeable {
private final RevisionVector headRevision;
private final GCMonitor monitor;
+ private final AtomicBoolean cancel;
private int count;
- public DetailGC(@NotNull RevisionVector headRevision, @NotNull GCMonitor monitor) {
- this.headRevision = checkNotNull(headRevision);
+ public DetailedGC(@NotNull RevisionVector headRevision, @NotNull GCMonitor monitor, @NotNull AtomicBoolean cancel) {
+ this.headRevision = requireNonNull(headRevision);
this.monitor = monitor;
+ this.cancel = cancel;
}
- public void detailGC(NodeDocument doc, GCPhases phases) {
- deleteSample(doc, phases);
- deleteUnmergedBranchCommitDocument(doc, phases);
- deleteDeletedProperties(doc, phases);
- deleteOldRevisions(doc, phases);
+ public void detailedGC(NodeDocument doc, GCPhases phases) {
+// deleteSample(doc, phases);
+ UpdateOp updateOp = new UpdateOp(requireNonNull(doc.getId()), false);
+ deleteDeletedProperties(doc, phases, updateOp);
+ deleteUnmergedBranchCommitDocument(doc, phases, updateOp);
+ deleteOldRevisions(doc, phases, updateOp);
}
/** TODO remove, this is just a skeleton sample */
- private void deleteSample(NodeDocument doc, GCPhases phases) {
- if (doc.getId().contains("should_delete")) {
- if (phases.start(GCPhase.DELETING)) {
- monitor.info("deleteSample: should do the deletion now, but this is demo only. I'm still learning");
- System.out.println("do the actual deletion");
- count++;
- phases.stop(GCPhase.DELETING);
- }
- }
- }
-
- private void deleteUnmergedBranchCommitDocument(NodeDocument doc, GCPhases phases) {
+// private void deleteSample(NodeDocument doc, GCPhases phases) {
+// if (doc.getId().contains("should_delete")) {
+// if (phases.start(GCPhase.DELETING)) {
+// monitor.info("deleteSample: should do the deletion now, but this is demo only. I'm still learning");
+// System.out.println("do the actual deletion");
+// count++;
+// phases.stop(GCPhase.DELETING);
+// }
+// }
+// }
+
+ private void deleteUnmergedBranchCommitDocument(NodeDocument doc, GCPhases phases, UpdateOp updateOp) {
// TODO Auto-generated method stub
}
- private void deleteDeletedProperties(NodeDocument doc, GCPhases phases) {
- // TODO Auto-generated method stub
+ private void deleteDeletedProperties(final NodeDocument doc, final GCPhases phases, final UpdateOp updateOp) {
+
+ // get Map of all properties along with their values
+ final Map<String, SortedMap<Revision, String>> properties = doc.getProperties();
+ // find all the properties which can be removed from document
+ // All the properties whose value is null in their respective
+ // latest revision are eligible to be garbage collected.
+ properties.forEach((propName, revisionStringSortedMap) -> {
+ if (revisionStringSortedMap.keySet()
+ .stream()
+ .sorted(REVERSE)
+ .limit(1)
+ .anyMatch(revision -> revisionStringSortedMap.get(revision) == null)) {
+ // set this property for removal
+ updateOp.remove(propName);
+ }
+ });
}
- private void deleteOldRevisions(NodeDocument doc, GCPhases phases) {
+ private void deleteOldRevisions(NodeDocument doc, GCPhases phases, UpdateOp updateOp) {
// TODO Auto-generated method stub
}
@@ -813,8 +846,8 @@ public class VersionGarbageCollector {
@NotNull AtomicBoolean cancel,
@NotNull VersionGCOptions options,
@NotNull GCMonitor monitor) {
- this.headRevision = checkNotNull(headRevision);
- this.cancel = checkNotNull(cancel);
+ this.headRevision = requireNonNull(headRevision);
+ this.cancel = requireNonNull(cancel);
this.timer = Stopwatch.createUnstarted();
this.options = options;
this.monitor = monitor;
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java
index 93f22f5202..e34d8f36b0 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java
@@ -22,6 +22,9 @@ package org.apache.jackrabbit.oak.plugins.document.mongo;
import static org.apache.jackrabbit.guava.common.collect.Iterables.concat;
import static org.apache.jackrabbit.guava.common.collect.Iterables.filter;
import static org.apache.jackrabbit.guava.common.collect.Iterables.transform;
+import static com.mongodb.client.model.Filters.and;
+import static com.mongodb.client.model.Filters.gte;
+import static com.mongodb.client.model.Filters.lt;
import static java.util.Collections.emptyList;
import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
import static org.apache.jackrabbit.oak.plugins.document.Document.ID;
@@ -108,10 +111,10 @@ public class MongoVersionGCSupport extends VersionGCSupport {
@Override
public CloseableIterable<NodeDocument> getPossiblyDeletedDocs(final long fromModified, final long toModified) {
//_deletedOnce == true && _modified >= fromModified && _modified < toModified
- Bson query = Filters.and(
+ Bson query = and(
Filters.eq(DELETED_ONCE, true),
- Filters.gte(MODIFIED_IN_SECS, getModifiedInSecs(fromModified)),
- Filters.lt(MODIFIED_IN_SECS, getModifiedInSecs(toModified))
+ gte(MODIFIED_IN_SECS, getModifiedInSecs(fromModified)),
+ lt(MODIFIED_IN_SECS, getModifiedInSecs(toModified))
);
FindIterable<BasicDBObject> cursor = getNodeCollection()
.find(query).batchSize(batchSize);
@@ -120,6 +123,29 @@ public class MongoVersionGCSupport extends VersionGCSupport {
input -> store.convertFromDBObject(NODES, input)));
}
+ /**
+ * Returns documents that have a {@link NodeDocument#MODIFIED_IN_SECS} value
+ * within the given range in sorted order. The two passed modified timestamps
+ * are in milliseconds since the epoch and the implementation will convert them
+ * to seconds at the granularity of the {@link NodeDocument#MODIFIED_IN_SECS}
+ * field and then perform the comparison.
+ *
+ * @param fromModified the lower bound modified timestamp (inclusive)
+ * @param toModified the upper bound modified timestamp (exclusive)
+ * @return matching documents in sorted order of {@link NodeDocument#MODIFIED_IN_SECS}
+ */
+ @Override
+ public Iterable<NodeDocument> getModifiedDocs(final long fromModified, final long toModified, final int limit) {
+ // _modified >= fromModified && _modified < toModified
+ final Bson query = and(gte(MODIFIED_IN_SECS, getModifiedInSecs(fromModified)),
+ lt(MODIFIED_IN_SECS, getModifiedInSecs(toModified)));
+ final FindIterable<BasicDBObject> cursor = getNodeCollection()
+ .find(query)
+ .sort(new org.bson.Document(MODIFIED_IN_SECS, 1))
+ .limit(limit);
+ return CloseableIterable.wrap(transform(cursor, input -> store.convertFromDBObject(NODES, input)));
+ }
+
@Override
public long getDeletedOnceCount() {
Bson query = Filters.eq(DELETED_ONCE, Boolean.TRUE);
@@ -207,9 +233,9 @@ public class MongoVersionGCSupport extends VersionGCSupport {
}
// OAK-8351: this (last) query only contains SD_TYPE and SD_MAX_REV_TIME_IN_SECS
// so mongodb should really use that _sdType_1__sdMaxRevTime_1 index
- result.add(Filters.and(
+ result.add(and(
Filters.or(orClauses),
- Filters.lt(SD_MAX_REV_TIME_IN_SECS, getModifiedInSecs(oldestRevTimeStamp))
+ lt(SD_MAX_REV_TIME_IN_SECS, getModifiedInSecs(oldestRevTimeStamp))
));
return result;
@@ -240,16 +266,16 @@ public class MongoVersionGCSupport extends VersionGCSupport {
Bson idPathClause = Filters.or(
Filters.regex(ID, Pattern.compile(".*" + idSuffix)),
// previous documents with long paths do not have a '-' in the id
- Filters.and(
+ and(
Filters.regex(ID, Pattern.compile("[^-]*")),
Filters.regex(PATH, Pattern.compile(".*" + idSuffix))
)
);
long minMaxRevTimeInSecs = Math.min(maxRevTimeInSecs, getModifiedInSecs(r.getTimestamp()));
- result.add(Filters.and(
+ result.add(and(
Filters.eq(SD_TYPE, DEFAULT_NO_BRANCH.typeCode()),
- Filters.lt(SD_MAX_REV_TIME_IN_SECS, minMaxRevTimeInSecs),
+ lt(SD_MAX_REV_TIME_IN_SECS, minMaxRevTimeInSecs),
idPathClause
));
}
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java
index 793e3a9433..c9428429bc 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java
@@ -894,6 +894,17 @@ public class Utils {
return builder.isThrottlingEnabled() || (docStoreThrottlingFeature != null && docStoreThrottlingFeature.isEnabled());
}
+ /**
+ * Check whether detailed GC is enabled or not for document store.
+ *
+ * @param builder instance for DocumentNodeStoreBuilder
+ * @return true if detailed GC is enabled else false
+ */
+ public static boolean isDetailedGCEnabled(final DocumentNodeStoreBuilder<?> builder) {
+ final Feature docStoreDetailedGCFeature = builder.getDocStoreDetailedGCFeature();
+ return builder.isDetailedGCEnabled() || (docStoreDetailedGCFeature != null && docStoreDetailedGCFeature.isEnabled());
+ }
+
/**
* Returns true if all the revisions in the {@code a} greater or equals
* to their counterparts in {@code b}. If {@code b} contains revisions
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceConfigurationTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceConfigurationTest.java
index 29710938c0..86559c1241 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceConfigurationTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceConfigurationTest.java
@@ -33,6 +33,7 @@ import org.osgi.framework.BundleContext;
import org.osgi.service.cm.ConfigurationAdmin;
import org.osgi.service.component.ComponentContext;
+import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_DETAILED_GC_ENABLED;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_THROTTLING_ENABLED;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
@@ -84,6 +85,7 @@ public class DocumentNodeStoreServiceConfigurationTest {
assertEquals(Arrays.asList("/"), Arrays.asList(config.persistentCacheIncludes()));
assertEquals("STRICT", config.leaseCheckMode());
assertEquals(DEFAULT_THROTTLING_ENABLED, config.throttlingEnabled());
+ assertEquals(DEFAULT_DETAILED_GC_ENABLED, config.detailedGCEnabled());
}
@Test
@@ -102,6 +104,14 @@ public class DocumentNodeStoreServiceConfigurationTest {
assertEquals(throttleDocStore, config.throttlingEnabled());
}
+ @Test
+ public void detailedGCEnabled() throws Exception {
+ boolean detailedGCDocStore = true;
+ addConfigurationEntry(preset, "detailedGCEnabled", detailedGCDocStore);
+ Configuration config = createConfiguration();
+ assertEquals(detailedGCDocStore, config.detailedGCEnabled());
+ }
+
@Test
public void presetSocketKeepAlive() throws Exception {
boolean keepAlive = !DocumentNodeStoreService.DEFAULT_SO_KEEP_ALIVE;
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCQueryTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCQueryTest.java
index 73eda05759..27333b57cf 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCQueryTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCQueryTest.java
@@ -107,7 +107,7 @@ public class VersionGCQueryTest {
clock.waitUntil(clock.getTime() + TimeUnit.HOURS.toMillis(1));
VersionGarbageCollector gc = new VersionGarbageCollector(
- ns, new VersionGCSupport(store));
+ ns, new VersionGCSupport(store), false);
prevDocIds.clear();
VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
assertEquals(11, stats.deletedDocGCCount);
@@ -140,7 +140,7 @@ public class VersionGCQueryTest {
clock.waitUntil(clock.getTime() + TimeUnit.HOURS.toMillis(1));
VersionGarbageCollector gc = new VersionGarbageCollector(
- ns, new VersionGCSupport(store));
+ ns, new VersionGCSupport(store), false);
prevDocIds.clear();
VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
assertEquals(1, stats.deletedDocGCCount);
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
index 445e7c4275..48f3f362ce 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
@@ -44,6 +44,7 @@ import org.jetbrains.annotations.NotNull;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
@@ -324,7 +325,7 @@ public class VersionGCTest {
deletedOnceCountCalls.incrementAndGet();
return Iterables.size(Utils.getSelectedDocuments(store, NodeDocument.DELETED_ONCE, 1));
}
- });
+ }, false);
// run first RGC
gc.gc(1, TimeUnit.HOURS);
@@ -342,21 +343,25 @@ public class VersionGCTest {
// OAK-10199
@Test
+ @Ignore
public void testDetailGcDocumentRead_disabled() throws Exception {
DetailGCHelper.disableDetailGC(ns);
VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
assertNotNull(stats);
- assertEquals(0, stats.detailGcDocsElapsed);
+ assertEquals(0, stats.detailedGcDocsElapsed);
}
@Test
+ @Ignore
public void testDetailGcDocumentRead_enabled() throws Exception {
DetailGCHelper.enableDetailGC(ns);
VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
assertNotNull(stats);
- assertNotEquals(0, stats.detailGcDocsElapsed);
+ assertNotEquals(0, stats.detailedGcDocsElapsed);
}
+ // OAK-10199
+
private Future<VersionGCStats> gc() {
// run gc in a separate thread
return execService.submit(new Callable<VersionGCStats>() {
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
index e58741733a..d33cc8c7de 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
@@ -497,7 +497,7 @@ public class VersionGarbageCollectorIT {
});
}
};
- final VersionGarbageCollector gc = new VersionGarbageCollector(store, gcSupport);
+ final VersionGarbageCollector gc = new VersionGarbageCollector(store, gcSupport, false);
// start GC -> will try to remove /foo and /bar
Future<VersionGCStats> f = execService.submit(new Callable<VersionGCStats>() {
@Override
@@ -658,7 +658,7 @@ public class VersionGarbageCollectorIT {
return super.getPossiblyDeletedDocs(fromModified, toModified);
}
};
- gcRef.set(new VersionGarbageCollector(store, gcSupport));
+ gcRef.set(new VersionGarbageCollector(store, gcSupport, false));
VersionGCStats stats = gcRef.get().gc(30, TimeUnit.MINUTES);
assertTrue(stats.canceled);
assertEquals(0, stats.deletedDocGCCount);
@@ -710,7 +710,7 @@ public class VersionGarbageCollectorIT {
return super.getPossiblyDeletedDocs(prevLastModifiedTime, lastModifiedTime).iterator();
}
};
- gcRef.set(new VersionGarbageCollector(store, gcSupport));
+ gcRef.set(new VersionGarbageCollector(store, gcSupport, false));
VersionGCStats stats = gcRef.get().gc(30, TimeUnit.MINUTES);
assertTrue(stats.canceled);
assertEquals(0, stats.deletedDocGCCount);
@@ -739,7 +739,7 @@ public class VersionGarbageCollectorIT {
});
}
};
- final VersionGarbageCollector gc = new VersionGarbageCollector(store, nonReportingGcSupport);
+ final VersionGarbageCollector gc = new VersionGarbageCollector(store, nonReportingGcSupport, false);
final long maxAgeHours = 1;
final long clockDelta = HOURS.toMillis(maxAgeHours) + MINUTES.toMillis(5);
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentNodeStoreBuilderTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentNodeStoreBuilderTest.java
index a78aef904e..a08abe05d3 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentNodeStoreBuilderTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentNodeStoreBuilderTest.java
@@ -48,6 +48,18 @@ public class MongoDocumentNodeStoreBuilderTest {
assertNull(builder.getDocStoreThrottlingFeature());
}
+ @Test
+ public void detailedGCDisabled() {
+ MongoDocumentNodeStoreBuilder builder = new MongoDocumentNodeStoreBuilder();
+ assertFalse(builder.isDetailedGCEnabled());
+ }
+
+ @Test
+ public void detailedGCFeatureToggleDisabled() {
+ MongoDocumentNodeStoreBuilder builder = new MongoDocumentNodeStoreBuilder();
+ assertNull(builder.getDocStoreDetailedGCFeature());
+ }
+
@Test
public void collectionCompressionDisabled() {
MongoDocumentNodeStoreBuilder builder = new MongoDocumentNodeStoreBuilder();
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/util/UtilsTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/util/UtilsTest.java
index 2b01f90b34..6041a41724 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/util/UtilsTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/util/UtilsTest.java
@@ -50,13 +50,13 @@ import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.toggle.Feature;
import org.apache.jackrabbit.oak.stats.Clock;
-import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import org.mockito.Mockito;
import org.slf4j.event.Level;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder.newDocumentNodeStoreBuilder;
+import static org.apache.jackrabbit.oak.plugins.document.util.Utils.isDetailedGCEnabled;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.isThrottlingEnabled;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.is;
@@ -184,6 +184,45 @@ public class UtilsTest {
assertTrue("Throttling is enabled via Feature Toggle", throttlingEnabled);
}
+ @Test
+ public void detailedGCEnabledDefaultValue() {
+ boolean detailedGCEnabled = isDetailedGCEnabled(newDocumentNodeStoreBuilder());
+ assertFalse("Detailed GC is disabled by default", detailedGCEnabled);
+ }
+
+ @Test
+ public void detailedGCExplicitlyDisabled() {
+ DocumentNodeStoreBuilder<?> builder = newDocumentNodeStoreBuilder();
+ builder.setDetailedGCEnabled(false);
+ Feature docStoreDetailedGCFeature = mock(Feature.class);
+ when(docStoreDetailedGCFeature.isEnabled()).thenReturn(false);
+ builder.setDocStoreDetailedGCFeature(docStoreDetailedGCFeature);
+ boolean detailedGCEnabled = isDetailedGCEnabled(builder);
+ assertFalse("Detailed GC is disabled explicitly", detailedGCEnabled);
+ }
+
+ @Test
+ public void detailedGCEnabledViaConfiguration() {
+ DocumentNodeStoreBuilder<?> builder = newDocumentNodeStoreBuilder();
+ builder.setDetailedGCEnabled(true);
+ Feature docStoreDetailedGCFeature = mock(Feature.class);
+ when(docStoreDetailedGCFeature.isEnabled()).thenReturn(false);
+ builder.setDocStoreDetailedGCFeature(docStoreDetailedGCFeature);
+ boolean detailedGCEnabled = isDetailedGCEnabled(builder);
+ assertTrue("Detailed GC is enabled via configuration", detailedGCEnabled);
+ }
+
+ @Test
+ public void detailedGCEnabledViaFeatureToggle() {
+ DocumentNodeStoreBuilder<?> builder = newDocumentNodeStoreBuilder();
+ builder.setDetailedGCEnabled(false);
+ Feature docStoreDetailedGCFeature = mock(Feature.class);
+ when(docStoreDetailedGCFeature.isEnabled()).thenReturn(true);
+ builder.setDocStoreDetailedGCFeature(docStoreDetailedGCFeature);
+ boolean detailedGCEnabled = isDetailedGCEnabled(builder);
+ assertTrue("Detailed GC is enabled via Feature Toggle", detailedGCEnabled);
+ }
+
@Test
public void getDepthFromId() throws Exception{
assertEquals(1, Utils.getDepthFromId("1:/x"));