You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by da...@apache.org on 2023/04/26 15:47:42 UTC
[jackrabbit-oak] branch OAK-10199 updated (8f848f1403 -> 7dfef758db)
This is an automated email from the ASF dual-hosted git repository.
daim pushed a change to branch OAK-10199
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
discard 8f848f1403 OAK-10199 : provided support for feature toggle & osgi config for detailed gc
discard 0a2ebf62b7 OAK-10199 : disable the detailGc in tearDown to avoid side-effects
discard cd6658e942 OAK-10199 : initial sketch of detail gc skeleton
add dbe8155afa OAK-10207: oak-jcr: remove unused test dependency zip4j (#909)
new bd2a1f69a0 OAK-10199 : initial sketch of detail gc skeleton
new 4e840d4e30 OAK-10199 : disable the detailGc in tearDown to avoid side-effects
new 5024987989 OAK-10199 : provided support for feature toggle & osgi config for detailed gc
new 7dfef758db OAK-10199 : override for RDB and added unit cases for deletedProps
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (8f848f1403)
\
N -- N -- N refs/heads/OAK-10199 (7dfef758db)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
oak-jcr/pom.xml | 6 -
.../oak/plugins/document/NodeDocument.java | 12 +-
.../plugins/document/VersionGCRecommendations.java | 20 +-
.../plugins/document/VersionGarbageCollector.java | 374 ++++++++++++---------
.../plugins/document/mongo/MongoDocumentStore.java | 2 +-
.../plugins/document/rdb/RDBDocumentStoreJDBC.java | 1 +
.../plugins/document/rdb/RDBVersionGCSupport.java | 35 +-
.../oak/plugins/document/DetailGCHelper.java | 22 +-
.../oak/plugins/document/NodeDocumentTest.java | 32 ++
.../oak/plugins/document/VersionGCInitTest.java | 17 +
.../oak/plugins/document/VersionGCStatsTest.java | 15 +
.../oak/plugins/document/VersionGCTest.java | 13 +-
.../document/VersionGarbageCollectorIT.java | 66 ++++
13 files changed, 407 insertions(+), 208 deletions(-)
[jackrabbit-oak] 03/04: OAK-10199 : provided support for feature toggle & osgi config for detailed gc
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
daim pushed a commit to branch OAK-10199
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 5024987989903ee76e39cdc2eea4fc5d4c123725
Author: Rishabh Kumar <di...@adobe.com>
AuthorDate: Mon Apr 24 13:07:44 2023 +0530
OAK-10199 : provided support for feature toggle & osgi config for detailed gc
---
.gitignore | 1 +
.../plugins/document/DocumentNodeStoreHelper.java | 4 +-
.../jackrabbit/oak/run/RevisionsCommand.java | 3 +-
.../oak/plugins/document/Configuration.java | 9 +
.../oak/plugins/document/DocumentNodeStore.java | 3 +-
.../plugins/document/DocumentNodeStoreBuilder.java | 21 +++
.../plugins/document/DocumentNodeStoreService.java | 10 ++
.../oak/plugins/document/NodeDocument.java | 16 ++
.../plugins/document/VersionGCRecommendations.java | 43 ++++-
.../oak/plugins/document/VersionGCSupport.java | 98 ++++++-----
.../plugins/document/VersionGarbageCollector.java | 183 ++++++++++++---------
.../document/mongo/MongoVersionGCSupport.java | 42 ++++-
.../oak/plugins/document/util/Utils.java | 11 ++
.../DocumentNodeStoreServiceConfigurationTest.java | 10 ++
.../oak/plugins/document/VersionGCQueryTest.java | 4 +-
.../oak/plugins/document/VersionGCTest.java | 11 +-
.../document/VersionGarbageCollectorIT.java | 8 +-
.../mongo/MongoDocumentNodeStoreBuilderTest.java | 12 ++
.../oak/plugins/document/util/UtilsTest.java | 41 ++++-
19 files changed, 385 insertions(+), 145 deletions(-)
diff --git a/.gitignore b/.gitignore
index faa5bb5b11..210e226f41 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,3 +10,4 @@ oak-run/*.csv
atlassian-ide-plugin.xml
.checkstyle
derby.log
+.java-version
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
index 03fb649ef2..d401c8cd0a 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
@@ -72,8 +72,8 @@ public class DocumentNodeStoreHelper {
}
public static VersionGarbageCollector createVersionGC(
- DocumentNodeStore nodeStore, VersionGCSupport gcSupport) {
- return new VersionGarbageCollector(nodeStore, gcSupport);
+ DocumentNodeStore nodeStore, VersionGCSupport gcSupport, final boolean detailedGCEnabled) {
+ return new VersionGarbageCollector(nodeStore, gcSupport, detailedGCEnabled);
}
private static Iterable<BlobReferences> scan(DocumentNodeStore store,
diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
index 21e5aff706..c9cc6b3d4d 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
@@ -60,6 +60,7 @@ import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreHelper.createVersionGC;
import static org.apache.jackrabbit.oak.plugins.document.FormatVersion.versionOf;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getRootDocument;
+import static org.apache.jackrabbit.oak.plugins.document.util.Utils.isDetailedGCEnabled;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.timestampToString;
import static org.apache.jackrabbit.oak.run.Utils.asCloseable;
import static org.apache.jackrabbit.oak.run.Utils.createDocumentMKBuilder;
@@ -226,7 +227,7 @@ public class RevisionsCommand implements Command {
useMemoryBlobStore(builder);
// create a version GC that operates on a read-only DocumentNodeStore
// and a GC support with a writable DocumentStore
- VersionGarbageCollector gc = createVersionGC(builder.build(), gcSupport);
+ VersionGarbageCollector gc = createVersionGC(builder.build(), gcSupport, isDetailedGCEnabled(builder));
VersionGCOptions gcOptions = gc.getOptions();
gcOptions = gcOptions.withDelayFactor(options.getDelay());
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Configuration.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Configuration.java
index fc5043de49..0bca9397ef 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Configuration.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Configuration.java
@@ -31,6 +31,7 @@ import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilde
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder.DEFAULT_NODE_CACHE_PERCENTAGE;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder.DEFAULT_PREV_DOC_CACHE_PERCENTAGE;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder.DEFAULT_UPDATE_LIMIT;
+import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_DETAILED_GC_ENABLED;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_THROTTLING_ENABLED;
@ObjectClassDefinition(
@@ -281,4 +282,12 @@ import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreServic
name = "Document Node Store Compression",
description = "Select compressor type for collections. 'Snappy' is the default supported compression.")
String collectionCompressionType() default "snappy";
+
+ @AttributeDefinition(
+ name = "Document Node Store Detailed GC",
+ description = "Boolean value indicating whether Detailed GC should be enabled for " +
+ "document node store or not. The Default value is " + DEFAULT_DETAILED_GC_ENABLED +
+ ". Note that this value can be overridden via framework " +
+ "property 'oak.documentstore.detailedGCEnabled'")
+ boolean detailedGCEnabled() default DEFAULT_DETAILED_GC_ENABLED;
}
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
index e0809ccd1a..1117b0a4d9 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
@@ -38,6 +38,7 @@ import static org.apache.jackrabbit.oak.plugins.document.Path.ROOT;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.alignWithExternalRevisions;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getIdFromPath;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getModuleVersion;
+import static org.apache.jackrabbit.oak.plugins.document.util.Utils.isDetailedGCEnabled;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.isThrottlingEnabled;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.pathToId;
import static org.apache.jackrabbit.oak.spi.observation.ChangeSet.COMMIT_CONTEXT_OBSERVATION_CHANGESET;
@@ -641,7 +642,7 @@ public final class DocumentNodeStore
this.branches = new UnmergedBranches();
this.asyncDelay = builder.getAsyncDelay();
this.versionGarbageCollector = new VersionGarbageCollector(
- this, builder.createVersionGCSupport());
+ this, builder.createVersionGCSupport(), isDetailedGCEnabled(builder));
this.versionGarbageCollector.setStatisticsProvider(builder.getStatisticsProvider());
this.versionGarbageCollector.setGCMonitor(builder.getGCMonitor());
this.journalGarbageCollector = new JournalGarbageCollector(
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java
index 27ee381b7a..009b54bc84 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java
@@ -124,6 +124,7 @@ public class DocumentNodeStoreBuilder<T extends DocumentNodeStoreBuilder<T>> {
private boolean isReadOnlyMode = false;
private Feature prefetchFeature;
private Feature docStoreThrottlingFeature;
+ private Feature docStoreDetailedGCFeature;
private Weigher<CacheValue, CacheValue> weigher = new EmpiricalWeigher();
private long memoryCacheSize = DEFAULT_MEMORY_CACHE_SIZE;
private int nodeCachePercentage = DEFAULT_NODE_CACHE_PERCENTAGE;
@@ -162,6 +163,7 @@ public class DocumentNodeStoreBuilder<T extends DocumentNodeStoreBuilder<T>> {
private Predicate<Path> nodeCachePredicate = Predicates.alwaysTrue();
private boolean clusterInvisible;
private boolean throttlingEnabled;
+ private boolean detailedGCEnabled;
/**
* @return a new {@link DocumentNodeStoreBuilder}.
@@ -284,6 +286,15 @@ public class DocumentNodeStoreBuilder<T extends DocumentNodeStoreBuilder<T>> {
return this.throttlingEnabled;
}
+ public T setDetailedGCEnabled(boolean b) {
+ this.detailedGCEnabled = b;
+ return thisBuilder();
+ }
+
+ public boolean isDetailedGCEnabled() {
+ return this.detailedGCEnabled;
+ }
+
public T setReadOnlyMode() {
this.isReadOnlyMode = true;
return thisBuilder();
@@ -313,6 +324,16 @@ public class DocumentNodeStoreBuilder<T extends DocumentNodeStoreBuilder<T>> {
return docStoreThrottlingFeature;
}
+ public T setDocStoreDetailedGCFeature(@Nullable Feature docStoreDetailedGC) {
+ this.docStoreDetailedGCFeature = docStoreDetailedGC;
+ return thisBuilder();
+ }
+
+ @Nullable
+ public Feature getDocStoreDetailedGCFeature() {
+ return docStoreDetailedGCFeature;
+ }
+
public T setLeaseFailureHandler(LeaseFailureHandler leaseFailureHandler) {
this.leaseFailureHandler = leaseFailureHandler;
return thisBuilder();
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
index 4b9ff4a0ad..65e8c6956c 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
@@ -138,6 +138,7 @@ public class DocumentNodeStoreService {
static final String DEFAULT_DB = "oak";
static final boolean DEFAULT_SO_KEEP_ALIVE = true;
static final boolean DEFAULT_THROTTLING_ENABLED = false;
+ static final boolean DEFAULT_DETAILED_GC_ENABLED = false;
static final int DEFAULT_MONGO_LEASE_SO_TIMEOUT_MILLIS = 30000;
static final String DEFAULT_PERSISTENT_CACHE = "cache";
static final String DEFAULT_JOURNAL_CACHE = "diff-cache";
@@ -181,6 +182,11 @@ public class DocumentNodeStoreService {
*/
private static final String FT_NAME_DOC_STORE_THROTTLING = "FT_THROTTLING_OAK-9909";
+ /**
+ * Feature toggle name to enable detailed GC for Mongo Document Store
+ */
+ private static final String FT_NAME_DEATILED_GC = "FT_DETAILED_GC_OAK-10199";
+
// property name constants - values can come from framework properties or OSGi config
public static final String CUSTOM_BLOB_STORE = "customBlobStore";
public static final String PROP_REV_RECOVERY_INTERVAL = "lastRevRecoveryJobIntervalInSecs";
@@ -216,6 +222,7 @@ public class DocumentNodeStoreService {
private JournalPropertyHandlerFactory journalPropertyHandlerFactory = new JournalPropertyHandlerFactory();
private Feature prefetchFeature;
private Feature docStoreThrottlingFeature;
+ private Feature docStoreDetailedGCFeature;
private ComponentContext context;
private Whiteboard whiteboard;
private long deactivationTimestamp = 0;
@@ -250,6 +257,7 @@ public class DocumentNodeStoreService {
documentStoreType = DocumentStoreType.fromString(this.config.documentStoreType());
prefetchFeature = Feature.newFeature(FT_NAME_PREFETCH, whiteboard);
docStoreThrottlingFeature = Feature.newFeature(FT_NAME_DOC_STORE_THROTTLING, whiteboard);
+ docStoreDetailedGCFeature = Feature.newFeature(FT_NAME_DEATILED_GC, whiteboard);
registerNodeStoreIfPossible();
}
@@ -465,7 +473,9 @@ public class DocumentNodeStoreService {
setLeaseCheckMode(ClusterNodeInfo.DEFAULT_LEASE_CHECK_DISABLED ? LeaseCheckMode.DISABLED : LeaseCheckMode.valueOf(config.leaseCheckMode())).
setPrefetchFeature(prefetchFeature).
setDocStoreThrottlingFeature(docStoreThrottlingFeature).
+ setDocStoreDetailedGCFeature(docStoreDetailedGCFeature).
setThrottlingEnabled(config.throttlingEnabled()).
+ setDetailedGCEnabled(config.detailedGCEnabled()).
setLeaseFailureHandler(new LeaseFailureHandler() {
private final LeaseFailureHandler defaultLeaseFailureHandler = createDefaultLeaseFailureHandler();
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java
index c79c6adfc4..71abba0a2e 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java
@@ -32,6 +32,7 @@ import java.util.SortedSet;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
import org.apache.jackrabbit.guava.common.base.Function;
import org.apache.jackrabbit.guava.common.base.Predicate;
@@ -66,6 +67,7 @@ import static org.apache.jackrabbit.guava.common.collect.Iterables.filter;
import static org.apache.jackrabbit.guava.common.collect.Iterables.mergeSorted;
import static org.apache.jackrabbit.guava.common.collect.Iterables.transform;
import static java.util.Objects.requireNonNull;
+import static java.util.stream.Collectors.toMap;
import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
import static org.apache.jackrabbit.oak.plugins.document.StableRevisionComparator.REVERSE;
import static org.apache.jackrabbit.oak.plugins.document.UpdateOp.Key;
@@ -1669,6 +1671,20 @@ public final class NodeDocument extends Document {
return map;
}
+ /**
+ * Returns all the properties on this document
+ * @return Map of all properties along with their values
+ */
+ @NotNull
+ Map<String, SortedMap<Revision, String>> getProperties() {
+ return data
+ .keySet()
+ .stream()
+ .filter(Utils::isPropertyName)
+ .map(o -> Map.entry(o, getLocalMap(o)))
+ .collect(toMap(Entry::getKey, Entry::getValue));
+ }
+
/**
* @return the {@link #REVISIONS} stored on this document.
*/
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java
index ac47cc69d8..d8b091261d 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java
@@ -21,6 +21,7 @@ package org.apache.jackrabbit.oak.plugins.document;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import com.google.common.collect.ImmutableMap;
import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCStats;
import org.apache.jackrabbit.oak.plugins.document.util.TimeInterval;
import org.apache.jackrabbit.oak.plugins.document.util.Utils;
@@ -31,6 +32,9 @@ import org.slf4j.LoggerFactory;
import org.apache.jackrabbit.guava.common.collect.Maps;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP;
+
/**
* Gives a recommendation about parameters for the next revision garbage collection run.
*/
@@ -43,6 +47,7 @@ public class VersionGCRecommendations {
final boolean ignoreDueToCheckPoint;
final TimeInterval scope;
+ final TimeInterval scopeFullGC;
final long maxCollect;
final long deleteCandidateCount;
final long lastOldestTimestamp;
@@ -81,6 +86,7 @@ public class VersionGCRecommendations {
long deletedOnceCount = 0;
long suggestedIntervalMs;
long oldestPossible;
+ long oldestPossibleFullGC;
long collectLimit = options.collectLimit;
this.vgc = vgc;
@@ -90,7 +96,7 @@ public class VersionGCRecommendations {
TimeInterval keep = new TimeInterval(clock.getTime() - maxRevisionAgeMs, Long.MAX_VALUE);
Map<String, Long> settings = getLongSettings();
- lastOldestTimestamp = settings.get(VersionGarbageCollector.SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP);
+ lastOldestTimestamp = settings.get(SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP);
if (lastOldestTimestamp == 0) {
log.debug("No lastOldestTimestamp found, querying for the oldest deletedOnce candidate");
oldestPossible = vgc.getOldestDeletedOnceTimestamp(clock, options.precisionMs) - 1;
@@ -102,7 +108,21 @@ public class VersionGCRecommendations {
TimeInterval scope = new TimeInterval(oldestPossible, Long.MAX_VALUE);
scope = scope.notLaterThan(keep.fromMs);
- fullDetailGCTimestamp = settings.get(VersionGarbageCollector.SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP);
+ fullDetailGCTimestamp = settings.get(SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP);
+ if (fullDetailGCTimestamp == 0) {
+ if (log.isDebugEnabled()) {
+ log.debug("No fullDetailGCTimestamp found, querying for the oldest deletedOnce candidate");
+ }
+ oldestPossibleFullGC = vgc.getOldestModifiedTimestamp(clock) - 1;
+ if (log.isDebugEnabled()) {
+ log.debug("fullDetailGCTimestamp found: {}", Utils.timestampToString(oldestPossibleFullGC));
+ }
+ } else {
+ oldestPossibleFullGC = fullDetailGCTimestamp - 1;
+ }
+
+ TimeInterval scopeFullGC = new TimeInterval(oldestPossibleFullGC, Long.MAX_VALUE);
+ scopeFullGC = scopeFullGC.notLaterThan(keep.fromMs);
suggestedIntervalMs = settings.get(VersionGarbageCollector.SETTINGS_COLLECTION_REC_INTERVAL_PROP);
if (suggestedIntervalMs > 0) {
@@ -162,6 +182,7 @@ public class VersionGCRecommendations {
this.precisionMs = options.precisionMs;
this.ignoreDueToCheckPoint = ignoreDueToCheckPoint;
this.scope = scope;
+ this.scopeFullGC = scopeFullGC;
this.scopeIsComplete = scope.toMs >= keep.fromMs;
this.maxCollect = collectLimit;
this.suggestedIntervalMs = suggestedIntervalMs;
@@ -185,7 +206,10 @@ public class VersionGCRecommendations {
stats.needRepeat = true;
} else if (!stats.canceled && !stats.ignoredGCDueToCheckPoint) {
// success, we would not expect to encounter revisions older than this in the future
- setLongSetting(VersionGarbageCollector.SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP, scope.toMs);
+// setLongSetting(SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP, scope.toMs);
+// setLongSetting(SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP, stats.oldestModifiedGced);
+ setLongSetting(ImmutableMap.of(SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP, scope.toMs,
+ SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP, stats.oldestModifiedGced));
int count = stats.deletedDocGCCount - stats.deletedLeafDocGCCount;
double usedFraction;
@@ -218,9 +242,9 @@ public class VersionGCRecommendations {
Document versionGCDoc = vgc.getDocumentStore().find(Collection.SETTINGS, VersionGarbageCollector.SETTINGS_COLLECTION_ID, 0);
Map<String, Long> settings = Maps.newHashMap();
// default values
- settings.put(VersionGarbageCollector.SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP, 0L);
+ settings.put(SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP, 0L);
settings.put(VersionGarbageCollector.SETTINGS_COLLECTION_REC_INTERVAL_PROP, 0L);
- settings.put(VersionGarbageCollector.SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP, -1L);
+ settings.put(SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP, 0L);
if (versionGCDoc != null) {
for (String k : versionGCDoc.keySet()) {
Object value = versionGCDoc.get(k);
@@ -233,8 +257,15 @@ public class VersionGCRecommendations {
}
void setLongSetting(String propName, long val) {
+ setLongSetting(Map.of(propName, val));
+// UpdateOp updateOp = new UpdateOp(VersionGarbageCollector.SETTINGS_COLLECTION_ID, true);
+// updateOp.set(propName, val);
+// vgc.getDocumentStore().createOrUpdate(Collection.SETTINGS, updateOp);
+ }
+
+ void setLongSetting(final Map<String, Long> propValMap) {
UpdateOp updateOp = new UpdateOp(VersionGarbageCollector.SETTINGS_COLLECTION_ID, true);
- updateOp.set(propName, val);
+ propValMap.forEach(updateOp::set);
vgc.getDocumentStore().createOrUpdate(Collection.SETTINGS, updateOp);
}
}
\ No newline at end of file
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupport.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupport.java
index 0e5c26c83d..f23340acbc 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupport.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupport.java
@@ -20,11 +20,14 @@
package org.apache.jackrabbit.oak.plugins.document;
import static org.apache.jackrabbit.guava.common.collect.Iterables.filter;
+import static java.util.stream.Collectors.toList;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MODIFIED_IN_SECS;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.getModifiedInSecs;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getAllDocuments;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getSelectedDocuments;
import java.util.Set;
+import java.util.stream.StreamSupport;
import org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType;
import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCStats;
@@ -59,53 +62,40 @@ public class VersionGCSupport {
* @param toModified the upper bound modified timestamp (exclusive)
* @return matching documents.
*/
- public Iterable<NodeDocument> getPossiblyDeletedDocs(final long fromModified,
- final long toModified) {
- return filter(getSelectedDocuments(store, NodeDocument.DELETED_ONCE, 1), new Predicate<NodeDocument>() {
- @Override
- public boolean apply(NodeDocument input) {
- return input.wasDeletedOnce()
- && modifiedGreaterThanEquals(input, fromModified)
- && modifiedLessThan(input, toModified);
- }
-
- private boolean modifiedGreaterThanEquals(NodeDocument doc,
- long time) {
- Long modified = doc.getModified();
- return modified != null && modified.compareTo(getModifiedInSecs(time)) >= 0;
- }
-
- private boolean modifiedLessThan(NodeDocument doc,
- long time) {
- Long modified = doc.getModified();
- return modified != null && modified.compareTo(getModifiedInSecs(time)) < 0;
- }
- });
+ public Iterable<NodeDocument> getPossiblyDeletedDocs(final long fromModified, final long toModified) {
+ return StreamSupport
+ .stream(getSelectedDocuments(store, NodeDocument.DELETED_ONCE, 1).spliterator(), false)
+ .filter(input -> input.wasDeletedOnce() && modifiedGreaterThanEquals(input, fromModified) && modifiedLessThan(input, toModified))
+ .collect(toList());
}
/**
- * TODO: document me!
+ * Returns documents that have a {@link NodeDocument#MODIFIED_IN_SECS} value
+ * within the given range .The two passed modified timestamps are in milliseconds
+ * since the epoch and the implementation will convert them to seconds at
+ * the granularity of the {@link NodeDocument#MODIFIED_IN_SECS} field and
+ * then perform the comparison.
+ *
+ * @param fromModified the lower bound modified timestamp (inclusive)
+ * @param toModified the upper bound modified timestamp (exclusive)
+ * @param limit the limit of documents to return
+ * @return matching documents.
*/
- public Iterable<NodeDocument> getModifiedDocs(final long fromModified, final long toModified) {
- return filter(getSelectedDocuments(store, NodeDocument.MODIFIED_IN_SECS, fromModified), new Predicate<NodeDocument>() {
- @Override
- public boolean apply(NodeDocument input) {
- return modifiedGreaterThanEquals(input, fromModified)
- && modifiedLessThan(input, toModified);
- }
-
- private boolean modifiedGreaterThanEquals(NodeDocument doc,
- long time) {
- Long modified = doc.getModified();
- return modified != null && modified.compareTo(getModifiedInSecs(time)) >= 0;
- }
+ public Iterable<NodeDocument> getModifiedDocs(final long fromModified, final long toModified, final int limit) {
+ return StreamSupport
+ .stream(getSelectedDocuments(store, MODIFIED_IN_SECS, fromModified).spliterator(), false)
+ .filter(input -> modifiedGreaterThanEquals(input, fromModified) && modifiedLessThan(input, toModified))
+ .limit(limit)
+ .collect(toList());
+ }
- private boolean modifiedLessThan(NodeDocument doc,
- long time) {
- Long modified = doc.getModified();
- return modified != null && modified.compareTo(getModifiedInSecs(time)) < 0;
- }
- });
+ private boolean modifiedGreaterThanEquals(final NodeDocument doc, final long time) {
+ Long modified = doc.getModified();
+ return modified != null && modified.compareTo(getModifiedInSecs(time)) >= 0;
+ }
+ private boolean modifiedLessThan(final NodeDocument doc, final long time) {
+ Long modified = doc.getModified();
+ return modified != null && modified.compareTo(getModifiedInSecs(time)) < 0;
}
/**
@@ -185,6 +175,30 @@ public class VersionGCSupport {
return ts;
}
+ /**
+ * Retrieve the time of the oldest modified document.
+ *
+ * @return the timestamp of the oldest modified document.
+ */
+ public long getOldestModifiedTimestamp(final Clock clock) {
+ long ts = 0;
+ long now = clock.getTime();
+ Iterable<NodeDocument> docs = null;
+
+ LOG.info("find oldest modified document");
+ try {
+ docs = getModifiedDocs(ts, now, 1);
+ if (docs.iterator().hasNext()) {
+ Long modified = docs.iterator().next().getModified();
+ return modified != null ? modified : 0L;
+ }
+ } finally {
+ Utils.closeIfCloseable(docs);
+ }
+ LOG.info("find oldest modified document to be {}", Utils.timestampToString(ts));
+ return ts;
+ }
+
public long getDeletedOnceCount() throws UnsupportedOperationException {
throw new UnsupportedOperationException("getDeletedOnceCount()");
}
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
index e7442a7d15..608ba02398 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
@@ -27,6 +27,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.SortedMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
@@ -54,7 +55,7 @@ import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
import static org.apache.jackrabbit.guava.common.base.StandardSystemProperty.LINE_SEPARATOR;
import static org.apache.jackrabbit.guava.common.collect.Iterables.all;
import static org.apache.jackrabbit.guava.common.collect.Iterators.partition;
@@ -66,6 +67,7 @@ import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MODIFIED_I
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType.COMMIT_ROOT_ONLY;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType.DEFAULT_LEAF;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType.DEFAULT_NO_BRANCH;
+import static org.apache.jackrabbit.oak.plugins.document.StableRevisionComparator.REVERSE;
import static org.slf4j.helpers.MessageFormatter.arrayFormat;
public class VersionGarbageCollector {
@@ -115,6 +117,7 @@ public class VersionGarbageCollector {
private final DocumentNodeStore nodeStore;
private final DocumentStore ds;
+ private final boolean detailedGCEnabled;
private final VersionGCSupport versionStore;
private final AtomicReference<GCJob> collector = newReference();
private VersionGCOptions options;
@@ -122,10 +125,12 @@ public class VersionGarbageCollector {
private RevisionGCStats gcStats = new RevisionGCStats(StatisticsProvider.NOOP);
VersionGarbageCollector(DocumentNodeStore nodeStore,
- VersionGCSupport gcSupport) {
+ VersionGCSupport gcSupport,
+ final boolean detailedGCEnabled) {
this.nodeStore = nodeStore;
this.versionStore = gcSupport;
this.ds = gcSupport.getDocumentStore();
+ this.detailedGCEnabled = detailedGCEnabled;
this.options = new VersionGCOptions();
}
@@ -201,7 +206,7 @@ public class VersionGarbageCollector {
}
public void setGCMonitor(@NotNull GCMonitor gcMonitor) {
- this.gcMonitor = checkNotNull(gcMonitor);
+ this.gcMonitor = requireNonNull(gcMonitor);
}
public VersionGCOptions getOptions() {
@@ -259,6 +264,7 @@ public class VersionGarbageCollector {
}
public static class VersionGCStats {
+ public long oldestModifiedGced;
boolean ignoredGCDueToCheckPoint;
boolean canceled;
boolean success = true;
@@ -274,14 +280,14 @@ public class VersionGarbageCollector {
final Stopwatch active = Stopwatch.createUnstarted();
final Stopwatch collectDeletedDocs = Stopwatch.createUnstarted();
final Stopwatch checkDeletedDocs = Stopwatch.createUnstarted();
- final Stopwatch detailGcDocs = Stopwatch.createUnstarted();
+ final Stopwatch detailedGcDocs = Stopwatch.createUnstarted();
final Stopwatch deleteDeletedDocs = Stopwatch.createUnstarted();
final Stopwatch collectAndDeleteSplitDocs = Stopwatch.createUnstarted();
final Stopwatch deleteSplitDocs = Stopwatch.createUnstarted();
final Stopwatch sortDocIds = Stopwatch.createUnstarted();
final Stopwatch updateResurrectedDocuments = Stopwatch.createUnstarted();
long activeElapsed, collectDeletedDocsElapsed, checkDeletedDocsElapsed, deleteDeletedDocsElapsed, collectAndDeleteSplitDocsElapsed,
- deleteSplitDocsElapsed, sortDocIdsElapsed, updateResurrectedDocumentsElapsed, detailGcDocsElapsed;
+ deleteSplitDocsElapsed, sortDocIdsElapsed, updateResurrectedDocumentsElapsed, detailedGcDocsElapsed;
@Override
public String toString() {
@@ -318,6 +324,7 @@ public class VersionGarbageCollector {
return "VersionGCStats{" +
"ignoredGCDueToCheckPoint=" + ignoredGCDueToCheckPoint +
+ ", oldestModifiedGced=" + oldestModifiedGced +
", canceled=" + canceled +
", deletedDocGCCount=" + deletedDocGCCount + " (of which leaf: " + deletedLeafDocGCCount + ")" +
", updateResurrectedGCCount=" + updateResurrectedGCCount +
@@ -331,6 +338,7 @@ public class VersionGarbageCollector {
void addRun(VersionGCStats run) {
++iterationCount;
this.ignoredGCDueToCheckPoint = run.ignoredGCDueToCheckPoint;
+ this.oldestModifiedGced = run.oldestModifiedGced;
this.canceled = run.canceled;
this.success = run.success;
this.limitExceeded = run.limitExceeded;
@@ -350,7 +358,7 @@ public class VersionGarbageCollector {
this.deleteSplitDocsElapsed += run.deleteSplitDocsElapsed;
this.sortDocIdsElapsed += run.sortDocIdsElapsed;
this.updateResurrectedDocumentsElapsed += run.updateResurrectedDocumentsElapsed;
- this.detailGcDocsElapsed += run.detailGcDocsElapsed;
+ this.detailedGcDocsElapsed += run.detailedGcDocsElapsed;
} else {
// single run -> read from stop watches
this.activeElapsed += run.active.elapsed(MICROSECONDS);
@@ -361,7 +369,7 @@ public class VersionGarbageCollector {
this.deleteSplitDocsElapsed += run.deleteSplitDocs.elapsed(MICROSECONDS);
this.sortDocIdsElapsed += run.sortDocIds.elapsed(MICROSECONDS);
this.updateResurrectedDocumentsElapsed += run.updateResurrectedDocuments.elapsed(MICROSECONDS);
- this.detailGcDocsElapsed += run.detailGcDocs.elapsed(MICROSECONDS);
+ this.detailedGcDocsElapsed += run.detailedGcDocs.elapsed(MICROSECONDS);
}
}
}
@@ -370,7 +378,7 @@ public class VersionGarbageCollector {
NONE,
COLLECTING,
CHECKING,
- DETAILGC,
+ DETAILED_GC,
DELETING,
SORTING,
SPLITS_CLEANUP,
@@ -398,7 +406,7 @@ public class VersionGarbageCollector {
this.watches.put(GCPhase.NONE, Stopwatch.createStarted());
this.watches.put(GCPhase.COLLECTING, stats.collectDeletedDocs);
this.watches.put(GCPhase.CHECKING, stats.checkDeletedDocs);
- this.watches.put(GCPhase.DETAILGC, stats.detailGcDocs);
+ this.watches.put(GCPhase.DETAILED_GC, stats.detailedGcDocs);
this.watches.put(GCPhase.DELETING, stats.deleteDeletedDocs);
this.watches.put(GCPhase.SORTING, stats.sortDocIds);
this.watches.put(GCPhase.SPLITS_CLEANUP, stats.collectAndDeleteSplitDocs);
@@ -525,7 +533,10 @@ public class VersionGarbageCollector {
collectDeletedDocuments(phases, headRevision, rec);
collectSplitDocuments(phases, sweepRevisions, rec);
- collectDetailGarbage(phases, headRevision, rec);
+ if (detailedGCEnabled) {
+ // run only if enabled
+ collectDetailedGarbage(phases, headRevision, rec);
+ }
}
} catch (LimitExceededException ex) {
stats.limitExceeded = true;
@@ -555,36 +566,33 @@ public class VersionGarbageCollector {
* followed by voluntary paused (aka throttling) to avoid excessive load on the
* system. The full repository scan does not have to finish particularly fast,
* it is okay that it takes a considerable amount of time.
- *
- * @param headRevision
+ *
+ * @param phases {@link GCPhases}
+ * @param headRevision the current head revision of
* @throws IOException
* @throws LimitExceededException
*/
- private void collectDetailGarbage(GCPhases phases, RevisionVector headRevision, VersionGCRecommendations rec)
+ private void collectDetailedGarbage(final GCPhases phases, final RevisionVector headRevision, final VersionGCRecommendations rec)
throws IOException, LimitExceededException {
- if (!DETAIL_GC_ENABLED) {
- // TODO: this toggling should be done nicer asap
- return;
- }
int docsTraversed = 0;
- DetailGC gc = new DetailGC(headRevision, monitor);
- try {
- final long fromModified;
- final long toModified;
- if (rec.fullDetailGCTimestamp == -1) {
- // then full detail-gc is disabled or over - use regular scope then
- fromModified = rec.scope.fromMs;
- toModified = rec.scope.toMs;
- } else {
- // then full detail-gc is enabled - use it then
- fromModified = rec.fullDetailGCTimestamp; // TODO: once we're passed rec.scope.fromMs we should
- // disable fullgc
- toModified = rec.scope.toMs; // the 'to' here is the max. it will process only eg 1 batch
- }
- long oldestGced = fromModified;
- boolean foundAnything = false;
+ long oldestModifiedGced = rec.scopeFullGC.fromMs;
+ try (DetailedGC gc = new DetailedGC(headRevision, monitor, cancel)) {
+ final long fromModified = rec.scopeFullGC.fromMs;
+ final long toModified = rec.scopeFullGC.toMs;
+// if (rec.fullDetailGCTimestamp == -1) {
+// // then full detail-gc is disabled or over - use regular scope then
+// fromModified = rec.scope.fromMs;
+// toModified = rec.scope.toMs;
+// } else {
+// // then full detail-gc is enabled - use it then
+// fromModified = rec.fullDetailGCTimestamp; // TODO: once we're passed rec.scope.fromMs we should
+// // disable fullgc
+// toModified = rec.scope.toMs; // the 'to' here is the max. it will process only eg 1 batch
+// }
+ // TODO : remove me
+ boolean foundAnything = false; // I think this flag is redundant
if (phases.start(GCPhase.COLLECTING)) {
- Iterable<NodeDocument> itr = versionStore.getModifiedDocs(fromModified, toModified);
+ Iterable<NodeDocument> itr = versionStore.getModifiedDocs(oldestModifiedGced, toModified, 2000);
final Stopwatch timer = Stopwatch.createUnstarted();
timer.reset().start();
try {
@@ -593,44 +601,54 @@ public class VersionGarbageCollector {
if (cancel.get()) {
break;
}
- foundAnything = true;
- if (phases.start(GCPhase.DETAILGC)) {
- gc.detailGC(doc, phases);
- phases.stop(GCPhase.DETAILGC);
+ if (phases.start(GCPhase.DETAILED_GC)) {
+ gc.detailedGC(doc, phases);
+ phases.stop(GCPhase.DETAILED_GC);
}
+
+ // TODO : remove this code, I don't think its possible to fetch these documents
+ // who doesn't have _modified field
final Long modified = doc.getModified();
if (modified == null) {
monitor.warn("collectDetailGarbage : document has no _modified property : {}",
doc.getId());
- } else if (modified < oldestGced) {
+ } else if (modified < oldestModifiedGced) {
monitor.warn(
"collectDetailGarbage : document has older _modified than query boundary : {} (from: {}, to: {})",
modified, fromModified, toModified);
} else {
- oldestGced = modified;
+ oldestModifiedGced = modified;
}
+ foundAnything = true;
docsTraversed++;
if (docsTraversed % PROGRESS_BATCH_SIZE == 0) {
monitor.info("Iterated through {} documents so far. {} had detail garbage",
docsTraversed, gc.getNumDocuments());
}
+ // this would never hit, since we are only fetching the oldest 2000 element in batches of 1000
+ // TODO: remove this if above mentioned logic is fine
if (rec.maxCollect > 0 && gc.getNumDocuments() > rec.maxCollect) {
// TODO: how would we recover from this?
+ // If we don't want above solution, then one of the another solution is to use lower time duration
+ // as done in document deletion process or use lower limit value or
+ // we should perform all the update ops in 1 go
throw new LimitExceededException();
}
+ oldestModifiedGced = modified == null ? fromModified : modified;
}
} finally {
Utils.closeIfCloseable(itr);
+ // why do we need to stop this here, we are already stopping the original gc run.
+ // can this be removed
delayOnModifications(timer.stop().elapsed(TimeUnit.MILLISECONDS));
+ phases.stats.oldestModifiedGced = oldestModifiedGced;
}
phases.stop(GCPhase.COLLECTING);
- if (!cancel.get() && foundAnything) {
- // TODO: move to evaluate()
- rec.setLongSetting(SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP, oldestGced + 1);
- }
+// if (!cancel.get() && foundAnything) {
+// // TODO: move to evaluate()
+// rec.setLongSetting(SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP, oldestModifiedGced + 1);
+// }
}
- } finally {
- gc.close();
}
}
@@ -665,8 +683,7 @@ public class VersionGarbageCollector {
VersionGCRecommendations rec)
throws IOException, LimitExceededException {
int docsTraversed = 0;
- DeletedDocsGC gc = new DeletedDocsGC(headRevision, cancel, options, monitor);
- try {
+ try (DeletedDocsGC gc = new DeletedDocsGC(headRevision, cancel, options, monitor)) {
if (phases.start(GCPhase.COLLECTING)) {
Iterable<NodeDocument> itr = versionStore.getPossiblyDeletedDocs(rec.scope.fromMs, rec.scope.toMs);
try {
@@ -731,53 +748,69 @@ public class VersionGarbageCollector {
gc.updateResurrectedDocuments(phases.stats);
phases.stop(GCPhase.UPDATING);
}
- } finally {
- gc.close();
}
}
}
- private class DetailGC implements Closeable {
+ private static class DetailedGC implements Closeable {
private final RevisionVector headRevision;
private final GCMonitor monitor;
+ private final AtomicBoolean cancel;
private int count;
- public DetailGC(@NotNull RevisionVector headRevision, @NotNull GCMonitor monitor) {
- this.headRevision = checkNotNull(headRevision);
+ public DetailedGC(@NotNull RevisionVector headRevision, @NotNull GCMonitor monitor, @NotNull AtomicBoolean cancel) {
+ this.headRevision = requireNonNull(headRevision);
this.monitor = monitor;
+ this.cancel = cancel;
}
- public void detailGC(NodeDocument doc, GCPhases phases) {
- deleteSample(doc, phases);
- deleteUnmergedBranchCommitDocument(doc, phases);
- deleteDeletedProperties(doc, phases);
- deleteOldRevisions(doc, phases);
+ public void detailedGC(NodeDocument doc, GCPhases phases) {
+// deleteSample(doc, phases);
+ UpdateOp updateOp = new UpdateOp(requireNonNull(doc.getId()), false);
+ deleteDeletedProperties(doc, phases, updateOp);
+ deleteUnmergedBranchCommitDocument(doc, phases, updateOp);
+ deleteOldRevisions(doc, phases, updateOp);
}
/** TODO remove, this is just a skeleton sample */
- private void deleteSample(NodeDocument doc, GCPhases phases) {
- if (doc.getId().contains("should_delete")) {
- if (phases.start(GCPhase.DELETING)) {
- monitor.info("deleteSample: should do the deletion now, but this is demo only. I'm still learning");
- System.out.println("do the actual deletion");
- count++;
- phases.stop(GCPhase.DELETING);
- }
- }
- }
-
- private void deleteUnmergedBranchCommitDocument(NodeDocument doc, GCPhases phases) {
+// private void deleteSample(NodeDocument doc, GCPhases phases) {
+// if (doc.getId().contains("should_delete")) {
+// if (phases.start(GCPhase.DELETING)) {
+// monitor.info("deleteSample: should do the deletion now, but this is demo only. I'm still learning");
+// System.out.println("do the actual deletion");
+// count++;
+// phases.stop(GCPhase.DELETING);
+// }
+// }
+// }
+
+ private void deleteUnmergedBranchCommitDocument(NodeDocument doc, GCPhases phases, UpdateOp updateOp) {
// TODO Auto-generated method stub
}
- private void deleteDeletedProperties(NodeDocument doc, GCPhases phases) {
- // TODO Auto-generated method stub
+ private void deleteDeletedProperties(final NodeDocument doc, final GCPhases phases, final UpdateOp updateOp) {
+
+ // get Map of all properties along with their values
+ final Map<String, SortedMap<Revision, String>> properties = doc.getProperties();
+ // find all the properties which can be removed from document
+ // All the properties whose value is null in their respective
+ // latest revision are eligible to be garbage collected.
+ properties.forEach((propName, revisionStringSortedMap) -> {
+ if (revisionStringSortedMap.keySet()
+ .stream()
+ .sorted(REVERSE)
+ .limit(1)
+ .anyMatch(revision -> revisionStringSortedMap.get(revision) == null)) {
+ // set this property for removal
+ updateOp.remove(propName);
+ }
+ });
}
- private void deleteOldRevisions(NodeDocument doc, GCPhases phases) {
+ private void deleteOldRevisions(NodeDocument doc, GCPhases phases, UpdateOp updateOp) {
// TODO Auto-generated method stub
}
@@ -813,8 +846,8 @@ public class VersionGarbageCollector {
@NotNull AtomicBoolean cancel,
@NotNull VersionGCOptions options,
@NotNull GCMonitor monitor) {
- this.headRevision = checkNotNull(headRevision);
- this.cancel = checkNotNull(cancel);
+ this.headRevision = requireNonNull(headRevision);
+ this.cancel = requireNonNull(cancel);
this.timer = Stopwatch.createUnstarted();
this.options = options;
this.monitor = monitor;
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java
index 93f22f5202..e34d8f36b0 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java
@@ -22,6 +22,9 @@ package org.apache.jackrabbit.oak.plugins.document.mongo;
import static org.apache.jackrabbit.guava.common.collect.Iterables.concat;
import static org.apache.jackrabbit.guava.common.collect.Iterables.filter;
import static org.apache.jackrabbit.guava.common.collect.Iterables.transform;
+import static com.mongodb.client.model.Filters.and;
+import static com.mongodb.client.model.Filters.gte;
+import static com.mongodb.client.model.Filters.lt;
import static java.util.Collections.emptyList;
import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
import static org.apache.jackrabbit.oak.plugins.document.Document.ID;
@@ -108,10 +111,10 @@ public class MongoVersionGCSupport extends VersionGCSupport {
@Override
public CloseableIterable<NodeDocument> getPossiblyDeletedDocs(final long fromModified, final long toModified) {
//_deletedOnce == true && _modified >= fromModified && _modified < toModified
- Bson query = Filters.and(
+ Bson query = and(
Filters.eq(DELETED_ONCE, true),
- Filters.gte(MODIFIED_IN_SECS, getModifiedInSecs(fromModified)),
- Filters.lt(MODIFIED_IN_SECS, getModifiedInSecs(toModified))
+ gte(MODIFIED_IN_SECS, getModifiedInSecs(fromModified)),
+ lt(MODIFIED_IN_SECS, getModifiedInSecs(toModified))
);
FindIterable<BasicDBObject> cursor = getNodeCollection()
.find(query).batchSize(batchSize);
@@ -120,6 +123,29 @@ public class MongoVersionGCSupport extends VersionGCSupport {
input -> store.convertFromDBObject(NODES, input)));
}
+ /**
+ * Returns documents that have a {@link NodeDocument#MODIFIED_IN_SECS} value
+ * within the given range in sorted order. The two passed modified timestamps
+ * are in milliseconds since the epoch and the implementation will convert them
+ * to seconds at the granularity of the {@link NodeDocument#MODIFIED_IN_SECS}
+ * field and then perform the comparison.
+ *
+ * @param fromModified the lower bound modified timestamp (inclusive)
+ * @param toModified the upper bound modified timestamp (exclusive)
+ * @return matching documents in sorted order of {@link NodeDocument#MODIFIED_IN_SECS}
+ */
+ @Override
+ public Iterable<NodeDocument> getModifiedDocs(final long fromModified, final long toModified, final int limit) {
+ // _modified >= fromModified && _modified < toModified
+ final Bson query = and(gte(MODIFIED_IN_SECS, getModifiedInSecs(fromModified)),
+ lt(MODIFIED_IN_SECS, getModifiedInSecs(toModified)));
+ final FindIterable<BasicDBObject> cursor = getNodeCollection()
+ .find(query)
+ .sort(new org.bson.Document(MODIFIED_IN_SECS, 1))
+ .limit(limit);
+ return CloseableIterable.wrap(transform(cursor, input -> store.convertFromDBObject(NODES, input)));
+ }
+
@Override
public long getDeletedOnceCount() {
Bson query = Filters.eq(DELETED_ONCE, Boolean.TRUE);
@@ -207,9 +233,9 @@ public class MongoVersionGCSupport extends VersionGCSupport {
}
// OAK-8351: this (last) query only contains SD_TYPE and SD_MAX_REV_TIME_IN_SECS
// so mongodb should really use that _sdType_1__sdMaxRevTime_1 index
- result.add(Filters.and(
+ result.add(and(
Filters.or(orClauses),
- Filters.lt(SD_MAX_REV_TIME_IN_SECS, getModifiedInSecs(oldestRevTimeStamp))
+ lt(SD_MAX_REV_TIME_IN_SECS, getModifiedInSecs(oldestRevTimeStamp))
));
return result;
@@ -240,16 +266,16 @@ public class MongoVersionGCSupport extends VersionGCSupport {
Bson idPathClause = Filters.or(
Filters.regex(ID, Pattern.compile(".*" + idSuffix)),
// previous documents with long paths do not have a '-' in the id
- Filters.and(
+ and(
Filters.regex(ID, Pattern.compile("[^-]*")),
Filters.regex(PATH, Pattern.compile(".*" + idSuffix))
)
);
long minMaxRevTimeInSecs = Math.min(maxRevTimeInSecs, getModifiedInSecs(r.getTimestamp()));
- result.add(Filters.and(
+ result.add(and(
Filters.eq(SD_TYPE, DEFAULT_NO_BRANCH.typeCode()),
- Filters.lt(SD_MAX_REV_TIME_IN_SECS, minMaxRevTimeInSecs),
+ lt(SD_MAX_REV_TIME_IN_SECS, minMaxRevTimeInSecs),
idPathClause
));
}
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java
index 793e3a9433..c9428429bc 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java
@@ -894,6 +894,17 @@ public class Utils {
return builder.isThrottlingEnabled() || (docStoreThrottlingFeature != null && docStoreThrottlingFeature.isEnabled());
}
+ /**
+ * Check whether detailed GC is enabled or not for document store.
+ *
+ * @param builder instance for DocumentNodeStoreBuilder
+ * @return true if detailed GC is enabled else false
+ */
+ public static boolean isDetailedGCEnabled(final DocumentNodeStoreBuilder<?> builder) {
+ final Feature docStoreDetailedGCFeature = builder.getDocStoreDetailedGCFeature();
+ return builder.isDetailedGCEnabled() || (docStoreDetailedGCFeature != null && docStoreDetailedGCFeature.isEnabled());
+ }
+
/**
* Returns true if all the revisions in the {@code a} greater or equals
* to their counterparts in {@code b}. If {@code b} contains revisions
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceConfigurationTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceConfigurationTest.java
index 29710938c0..86559c1241 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceConfigurationTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceConfigurationTest.java
@@ -33,6 +33,7 @@ import org.osgi.framework.BundleContext;
import org.osgi.service.cm.ConfigurationAdmin;
import org.osgi.service.component.ComponentContext;
+import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_DETAILED_GC_ENABLED;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_THROTTLING_ENABLED;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
@@ -84,6 +85,7 @@ public class DocumentNodeStoreServiceConfigurationTest {
assertEquals(Arrays.asList("/"), Arrays.asList(config.persistentCacheIncludes()));
assertEquals("STRICT", config.leaseCheckMode());
assertEquals(DEFAULT_THROTTLING_ENABLED, config.throttlingEnabled());
+ assertEquals(DEFAULT_DETAILED_GC_ENABLED, config.detailedGCEnabled());
}
@Test
@@ -102,6 +104,14 @@ public class DocumentNodeStoreServiceConfigurationTest {
assertEquals(throttleDocStore, config.throttlingEnabled());
}
+ @Test
+ public void detailedGCEnabled() throws Exception {
+ boolean detailedGCDocStore = true;
+ addConfigurationEntry(preset, "detailedGCEnabled", detailedGCDocStore);
+ Configuration config = createConfiguration();
+ assertEquals(detailedGCDocStore, config.detailedGCEnabled());
+ }
+
@Test
public void presetSocketKeepAlive() throws Exception {
boolean keepAlive = !DocumentNodeStoreService.DEFAULT_SO_KEEP_ALIVE;
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCQueryTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCQueryTest.java
index 73eda05759..27333b57cf 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCQueryTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCQueryTest.java
@@ -107,7 +107,7 @@ public class VersionGCQueryTest {
clock.waitUntil(clock.getTime() + TimeUnit.HOURS.toMillis(1));
VersionGarbageCollector gc = new VersionGarbageCollector(
- ns, new VersionGCSupport(store));
+ ns, new VersionGCSupport(store), false);
prevDocIds.clear();
VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
assertEquals(11, stats.deletedDocGCCount);
@@ -140,7 +140,7 @@ public class VersionGCQueryTest {
clock.waitUntil(clock.getTime() + TimeUnit.HOURS.toMillis(1));
VersionGarbageCollector gc = new VersionGarbageCollector(
- ns, new VersionGCSupport(store));
+ ns, new VersionGCSupport(store), false);
prevDocIds.clear();
VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
assertEquals(1, stats.deletedDocGCCount);
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
index 445e7c4275..48f3f362ce 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
@@ -44,6 +44,7 @@ import org.jetbrains.annotations.NotNull;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
@@ -324,7 +325,7 @@ public class VersionGCTest {
deletedOnceCountCalls.incrementAndGet();
return Iterables.size(Utils.getSelectedDocuments(store, NodeDocument.DELETED_ONCE, 1));
}
- });
+ }, false);
// run first RGC
gc.gc(1, TimeUnit.HOURS);
@@ -342,21 +343,25 @@ public class VersionGCTest {
// OAK-10199
@Test
+ @Ignore
public void testDetailGcDocumentRead_disabled() throws Exception {
DetailGCHelper.disableDetailGC(ns);
VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
assertNotNull(stats);
- assertEquals(0, stats.detailGcDocsElapsed);
+ assertEquals(0, stats.detailedGcDocsElapsed);
}
@Test
+ @Ignore
public void testDetailGcDocumentRead_enabled() throws Exception {
DetailGCHelper.enableDetailGC(ns);
VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
assertNotNull(stats);
- assertNotEquals(0, stats.detailGcDocsElapsed);
+ assertNotEquals(0, stats.detailedGcDocsElapsed);
}
+ // OAK-10199
+
private Future<VersionGCStats> gc() {
// run gc in a separate thread
return execService.submit(new Callable<VersionGCStats>() {
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
index e58741733a..d33cc8c7de 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
@@ -497,7 +497,7 @@ public class VersionGarbageCollectorIT {
});
}
};
- final VersionGarbageCollector gc = new VersionGarbageCollector(store, gcSupport);
+ final VersionGarbageCollector gc = new VersionGarbageCollector(store, gcSupport, false);
// start GC -> will try to remove /foo and /bar
Future<VersionGCStats> f = execService.submit(new Callable<VersionGCStats>() {
@Override
@@ -658,7 +658,7 @@ public class VersionGarbageCollectorIT {
return super.getPossiblyDeletedDocs(fromModified, toModified);
}
};
- gcRef.set(new VersionGarbageCollector(store, gcSupport));
+ gcRef.set(new VersionGarbageCollector(store, gcSupport, false));
VersionGCStats stats = gcRef.get().gc(30, TimeUnit.MINUTES);
assertTrue(stats.canceled);
assertEquals(0, stats.deletedDocGCCount);
@@ -710,7 +710,7 @@ public class VersionGarbageCollectorIT {
return super.getPossiblyDeletedDocs(prevLastModifiedTime, lastModifiedTime).iterator();
}
};
- gcRef.set(new VersionGarbageCollector(store, gcSupport));
+ gcRef.set(new VersionGarbageCollector(store, gcSupport, false));
VersionGCStats stats = gcRef.get().gc(30, TimeUnit.MINUTES);
assertTrue(stats.canceled);
assertEquals(0, stats.deletedDocGCCount);
@@ -739,7 +739,7 @@ public class VersionGarbageCollectorIT {
});
}
};
- final VersionGarbageCollector gc = new VersionGarbageCollector(store, nonReportingGcSupport);
+ final VersionGarbageCollector gc = new VersionGarbageCollector(store, nonReportingGcSupport, false);
final long maxAgeHours = 1;
final long clockDelta = HOURS.toMillis(maxAgeHours) + MINUTES.toMillis(5);
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentNodeStoreBuilderTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentNodeStoreBuilderTest.java
index a78aef904e..a08abe05d3 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentNodeStoreBuilderTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentNodeStoreBuilderTest.java
@@ -48,6 +48,18 @@ public class MongoDocumentNodeStoreBuilderTest {
assertNull(builder.getDocStoreThrottlingFeature());
}
+ @Test
+ public void detailedGCDisabled() {
+ MongoDocumentNodeStoreBuilder builder = new MongoDocumentNodeStoreBuilder();
+ assertFalse(builder.isDetailedGCEnabled());
+ }
+
+ @Test
+ public void detailedGCFeatureToggleDisabled() {
+ MongoDocumentNodeStoreBuilder builder = new MongoDocumentNodeStoreBuilder();
+ assertNull(builder.getDocStoreDetailedGCFeature());
+ }
+
@Test
public void collectionCompressionDisabled() {
MongoDocumentNodeStoreBuilder builder = new MongoDocumentNodeStoreBuilder();
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/util/UtilsTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/util/UtilsTest.java
index 2b01f90b34..6041a41724 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/util/UtilsTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/util/UtilsTest.java
@@ -50,13 +50,13 @@ import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.toggle.Feature;
import org.apache.jackrabbit.oak.stats.Clock;
-import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import org.mockito.Mockito;
import org.slf4j.event.Level;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder.newDocumentNodeStoreBuilder;
+import static org.apache.jackrabbit.oak.plugins.document.util.Utils.isDetailedGCEnabled;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.isThrottlingEnabled;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.is;
@@ -184,6 +184,45 @@ public class UtilsTest {
assertTrue("Throttling is enabled via Feature Toggle", throttlingEnabled);
}
+ @Test
+ public void detailedGCEnabledDefaultValue() {
+ boolean detailedGCEnabled = isDetailedGCEnabled(newDocumentNodeStoreBuilder());
+ assertFalse("Detailed GC is disabled by default", detailedGCEnabled);
+ }
+
+ @Test
+ public void detailedGCExplicitlyDisabled() {
+ DocumentNodeStoreBuilder<?> builder = newDocumentNodeStoreBuilder();
+ builder.setDetailedGCEnabled(false);
+ Feature docStoreDetailedGCFeature = mock(Feature.class);
+ when(docStoreDetailedGCFeature.isEnabled()).thenReturn(false);
+ builder.setDocStoreDetailedGCFeature(docStoreDetailedGCFeature);
+ boolean detailedGCEnabled = isDetailedGCEnabled(builder);
+ assertFalse("Detailed GC is disabled explicitly", detailedGCEnabled);
+ }
+
+ @Test
+ public void detailedGCEnabledViaConfiguration() {
+ DocumentNodeStoreBuilder<?> builder = newDocumentNodeStoreBuilder();
+ builder.setDetailedGCEnabled(true);
+ Feature docStoreDetailedGCFeature = mock(Feature.class);
+ when(docStoreDetailedGCFeature.isEnabled()).thenReturn(false);
+ builder.setDocStoreDetailedGCFeature(docStoreDetailedGCFeature);
+ boolean detailedGCEnabled = isDetailedGCEnabled(builder);
+ assertTrue("Detailed GC is enabled via configuration", detailedGCEnabled);
+ }
+
+ @Test
+ public void detailedGCEnabledViaFeatureToggle() {
+ DocumentNodeStoreBuilder<?> builder = newDocumentNodeStoreBuilder();
+ builder.setDetailedGCEnabled(false);
+ Feature docStoreDetailedGCFeature = mock(Feature.class);
+ when(docStoreDetailedGCFeature.isEnabled()).thenReturn(true);
+ builder.setDocStoreDetailedGCFeature(docStoreDetailedGCFeature);
+ boolean detailedGCEnabled = isDetailedGCEnabled(builder);
+ assertTrue("Detailed GC is enabled via Feature Toggle", detailedGCEnabled);
+ }
+
@Test
public void getDepthFromId() throws Exception{
assertEquals(1, Utils.getDepthFromId("1:/x"));
[jackrabbit-oak] 04/04: OAK-10199 : override for RDB and added unit cases for deletedProps
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
daim pushed a commit to branch OAK-10199
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 7dfef758db18aae1bcbf90fd049f792bb86e3c49
Author: Rishabh Kumar <di...@adobe.com>
AuthorDate: Wed Apr 26 21:16:44 2023 +0530
OAK-10199 : override for RDB and added unit cases for deletedProps
---
.../oak/plugins/document/NodeDocument.java | 12 +-
.../plugins/document/VersionGCRecommendations.java | 20 +-
.../plugins/document/VersionGarbageCollector.java | 374 ++++++++++++---------
.../plugins/document/mongo/MongoDocumentStore.java | 2 +-
.../plugins/document/rdb/RDBDocumentStoreJDBC.java | 1 +
.../plugins/document/rdb/RDBVersionGCSupport.java | 35 +-
.../oak/plugins/document/DetailGCHelper.java | 22 +-
.../oak/plugins/document/NodeDocumentTest.java | 32 ++
.../oak/plugins/document/VersionGCInitTest.java | 17 +
.../oak/plugins/document/VersionGCStatsTest.java | 15 +
.../oak/plugins/document/VersionGCTest.java | 13 +-
.../document/VersionGarbageCollectorIT.java | 66 ++++
12 files changed, 407 insertions(+), 202 deletions(-)
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java
index 71abba0a2e..66a1bc2eae 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java
@@ -32,7 +32,6 @@ import java.util.SortedSet;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
-import java.util.stream.Collectors;
import org.apache.jackrabbit.guava.common.base.Function;
import org.apache.jackrabbit.guava.common.base.Predicate;
@@ -59,6 +58,7 @@ import org.apache.jackrabbit.guava.common.collect.Iterables;
import org.apache.jackrabbit.guava.common.collect.Maps;
import org.apache.jackrabbit.guava.common.collect.Sets;
+import static java.util.stream.Collectors.toSet;
import static org.apache.jackrabbit.guava.common.base.Objects.equal;
import static org.apache.jackrabbit.guava.common.base.Preconditions.checkArgument;
import static org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull;
@@ -67,7 +67,6 @@ import static org.apache.jackrabbit.guava.common.collect.Iterables.filter;
import static org.apache.jackrabbit.guava.common.collect.Iterables.mergeSorted;
import static org.apache.jackrabbit.guava.common.collect.Iterables.transform;
import static java.util.Objects.requireNonNull;
-import static java.util.stream.Collectors.toMap;
import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
import static org.apache.jackrabbit.oak.plugins.document.StableRevisionComparator.REVERSE;
import static org.apache.jackrabbit.oak.plugins.document.UpdateOp.Key;
@@ -1672,17 +1671,16 @@ public final class NodeDocument extends Document {
}
/**
- * Returns all the properties on this document
- * @return Map of all properties along with their values
+ * Returns name of all the properties on this document
+ * @return Set of all property names
*/
@NotNull
- Map<String, SortedMap<Revision, String>> getProperties() {
+ Set<String> getPropertyNames() {
return data
.keySet()
.stream()
.filter(Utils::isPropertyName)
- .map(o -> Map.entry(o, getLocalMap(o)))
- .collect(toMap(Entry::getKey, Entry::getValue));
+ .collect(toSet());
}
/**
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java
index d8b091261d..d830a34653 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java
@@ -32,7 +32,7 @@ import org.slf4j.LoggerFactory;
import org.apache.jackrabbit.guava.common.collect.Maps;
-import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_DETAILED_GC_TIMESTAMP_PROP;
import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP;
/**
@@ -51,7 +51,7 @@ public class VersionGCRecommendations {
final long maxCollect;
final long deleteCandidateCount;
final long lastOldestTimestamp;
- final long fullDetailGCTimestamp;
+ final long detailedGCTimestamp;
final long originalCollectLimit;
private final long precisionMs;
@@ -108,17 +108,17 @@ public class VersionGCRecommendations {
TimeInterval scope = new TimeInterval(oldestPossible, Long.MAX_VALUE);
scope = scope.notLaterThan(keep.fromMs);
- fullDetailGCTimestamp = settings.get(SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP);
- if (fullDetailGCTimestamp == 0) {
+ detailedGCTimestamp = settings.get(SETTINGS_COLLECTION_DETAILED_GC_TIMESTAMP_PROP);
+ if (detailedGCTimestamp == 0) {
if (log.isDebugEnabled()) {
- log.debug("No fullDetailGCTimestamp found, querying for the oldest deletedOnce candidate");
+ log.debug("No detailedGCTimestamp found, querying for the oldest deletedOnce candidate");
}
oldestPossibleFullGC = vgc.getOldestModifiedTimestamp(clock) - 1;
if (log.isDebugEnabled()) {
- log.debug("fullDetailGCTimestamp found: {}", Utils.timestampToString(oldestPossibleFullGC));
+ log.debug("detailedGCTimestamp found: {}", Utils.timestampToString(oldestPossibleFullGC));
}
} else {
- oldestPossibleFullGC = fullDetailGCTimestamp - 1;
+ oldestPossibleFullGC = detailedGCTimestamp - 1;
}
TimeInterval scopeFullGC = new TimeInterval(oldestPossibleFullGC, Long.MAX_VALUE);
@@ -206,10 +206,8 @@ public class VersionGCRecommendations {
stats.needRepeat = true;
} else if (!stats.canceled && !stats.ignoredGCDueToCheckPoint) {
// success, we would not expect to encounter revisions older than this in the future
-// setLongSetting(SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP, scope.toMs);
-// setLongSetting(SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP, stats.oldestModifiedGced);
setLongSetting(ImmutableMap.of(SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP, scope.toMs,
- SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP, stats.oldestModifiedGced));
+ SETTINGS_COLLECTION_DETAILED_GC_TIMESTAMP_PROP, stats.oldestModifiedGced));
int count = stats.deletedDocGCCount - stats.deletedLeafDocGCCount;
double usedFraction;
@@ -244,7 +242,7 @@ public class VersionGCRecommendations {
// default values
settings.put(SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP, 0L);
settings.put(VersionGarbageCollector.SETTINGS_COLLECTION_REC_INTERVAL_PROP, 0L);
- settings.put(SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP, 0L);
+ settings.put(SETTINGS_COLLECTION_DETAILED_GC_TIMESTAMP_PROP, 0L);
if (versionGCDoc != null) {
for (String k : versionGCDoc.keySet()) {
Object value = versionGCDoc.get(k);
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
index 608ba02398..da06f10d18 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
@@ -21,16 +21,18 @@ package org.apache.jackrabbit.oak.plugins.document;
import java.io.Closeable;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
-import java.util.SortedMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
import org.apache.jackrabbit.guava.common.base.Function;
import org.apache.jackrabbit.guava.common.base.Joiner;
@@ -55,7 +57,11 @@ import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static java.lang.Math.round;
+import static java.util.Collections.emptySet;
import static java.util.Objects.requireNonNull;
+import static java.util.Optional.ofNullable;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.apache.jackrabbit.guava.common.base.StandardSystemProperty.LINE_SEPARATOR;
import static org.apache.jackrabbit.guava.common.collect.Iterables.all;
import static org.apache.jackrabbit.guava.common.collect.Iterators.partition;
@@ -67,14 +73,10 @@ import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MODIFIED_I
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType.COMMIT_ROOT_ONLY;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType.DEFAULT_LEAF;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType.DEFAULT_NO_BRANCH;
-import static org.apache.jackrabbit.oak.plugins.document.StableRevisionComparator.REVERSE;
import static org.slf4j.helpers.MessageFormatter.arrayFormat;
public class VersionGarbageCollector {
- /** TODO temporary global flag to enable 'detail gc' during prototyping. Should eventually become eg a system property */
- public static boolean DETAIL_GC_ENABLED = false;
-
//Kept less than MongoDocumentStore.IN_CLAUSE_BATCH_SIZE to avoid re-partitioning
private static final int DELETE_BATCH_SIZE = 450;
private static final int UPDATE_BATCH_SIZE = 450;
@@ -105,15 +107,9 @@ public class VersionGarbageCollector {
static final String SETTINGS_COLLECTION_REC_INTERVAL_PROP = "recommendedIntervalMs";
/**
- * Property name to timestamp when last full-detail-GC run happened, or -1 if not applicable/in-use.
- * <p>
- * <ul>
- * <li>-1 : full repo scan is disabled</li>
- * <li>0 : full repo scan is enabled and bound to start from zero == oldest _modified </li>
- * <li>gt 0 : full repo scan is enabled, was already done up until this value</li>
- * </ul>
+ * Property name to timestamp till when last detailed-GC run happened
*/
- static final String SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP = "fullDetailGCTimeStamp";
+ static final String SETTINGS_COLLECTION_DETAILED_GC_TIMESTAMP_PROP = "detailedGCTimeStamp";
private final DocumentNodeStore nodeStore;
private final DocumentStore ds;
@@ -170,7 +166,7 @@ public class VersionGarbageCollector {
break;
}
averageDurationMs = ((averageDurationMs * (overall.iterationCount - 1))
- + stats.active.elapsed(TimeUnit.MILLISECONDS)) / overall.iterationCount;
+ + stats.active.elapsed(MILLISECONDS)) / overall.iterationCount;
}
success = true;
return overall;
@@ -274,20 +270,27 @@ public class VersionGarbageCollector {
int deletedDocGCCount;
int deletedLeafDocGCCount;
int splitDocGCCount;
+ int updatedDetailedGCDocsCount;
+ int deletedPropsGCCount;
int intermediateSplitDocGCCount;
int updateResurrectedGCCount;
final TimeDurationFormatter df = TimeDurationFormatter.forLogging();
final Stopwatch active = Stopwatch.createUnstarted();
final Stopwatch collectDeletedDocs = Stopwatch.createUnstarted();
+ final Stopwatch collectDeletedProps = Stopwatch.createUnstarted();
+ final Stopwatch collectDeletedOldRevs = Stopwatch.createUnstarted();
+ final Stopwatch collectUnmergedBC = Stopwatch.createUnstarted();
final Stopwatch checkDeletedDocs = Stopwatch.createUnstarted();
- final Stopwatch detailedGcDocs = Stopwatch.createUnstarted();
+ final Stopwatch detailedGCDocs = Stopwatch.createUnstarted();
+ final Stopwatch deleteDetailedGCDocs = Stopwatch.createUnstarted();
final Stopwatch deleteDeletedDocs = Stopwatch.createUnstarted();
final Stopwatch collectAndDeleteSplitDocs = Stopwatch.createUnstarted();
final Stopwatch deleteSplitDocs = Stopwatch.createUnstarted();
final Stopwatch sortDocIds = Stopwatch.createUnstarted();
final Stopwatch updateResurrectedDocuments = Stopwatch.createUnstarted();
long activeElapsed, collectDeletedDocsElapsed, checkDeletedDocsElapsed, deleteDeletedDocsElapsed, collectAndDeleteSplitDocsElapsed,
- deleteSplitDocsElapsed, sortDocIdsElapsed, updateResurrectedDocumentsElapsed, detailedGcDocsElapsed;
+ deleteSplitDocsElapsed, sortDocIdsElapsed, updateResurrectedDocumentsElapsed, detailedGCDocsElapsed, collectDeletedPropsElapsed,
+ deleteDetailedGCDocsElapsed, collectDeletedOldRevsElapsed, collectUnmergedBCElapsed;
@Override
public String toString() {
@@ -302,10 +305,16 @@ public class VersionGarbageCollector {
df.format(deleteSplitDocsElapsed, MICROSECONDS));
}
timings = String.format(fmt, df.format(collectDeletedDocsElapsed, MICROSECONDS),
- df.format(checkDeletedDocsElapsed, MICROSECONDS), df.format(sortDocIdsElapsed, MICROSECONDS),
+ df.format(collectDeletedPropsElapsed, MICROSECONDS),
+ df.format(collectDeletedOldRevsElapsed, MICROSECONDS),
+ df.format(collectUnmergedBCElapsed, MICROSECONDS),
+ df.format(checkDeletedDocsElapsed, MICROSECONDS),
+ df.format(sortDocIdsElapsed, MICROSECONDS),
df.format(updateResurrectedDocumentsElapsed, MICROSECONDS),
df.format(deleteDeletedDocsElapsed, MICROSECONDS),
df.format(collectAndDeleteSplitDocsElapsed, MICROSECONDS),
+ df.format(detailedGCDocsElapsed, MICROSECONDS),
+ df.format(deleteDetailedGCDocsElapsed, MICROSECONDS),
timeDeletingSplitDocs);
} else {
String timeDeletingSplitDocs = "";
@@ -314,11 +323,16 @@ public class VersionGarbageCollector {
df.format(deleteSplitDocs.elapsed(MICROSECONDS), MICROSECONDS));
}
timings = String.format(fmt, df.format(collectDeletedDocs.elapsed(MICROSECONDS), MICROSECONDS),
+ df.format(collectDeletedProps.elapsed(MICROSECONDS), MICROSECONDS),
+ df.format(collectDeletedOldRevs.elapsed(MICROSECONDS), MICROSECONDS),
+ df.format(collectUnmergedBC.elapsed(MICROSECONDS), MICROSECONDS),
df.format(checkDeletedDocs.elapsed(MICROSECONDS), MICROSECONDS),
df.format(sortDocIds.elapsed(MICROSECONDS), MICROSECONDS),
df.format(updateResurrectedDocuments.elapsed(MICROSECONDS), MICROSECONDS),
df.format(deleteDeletedDocs.elapsed(MICROSECONDS), MICROSECONDS),
df.format(collectAndDeleteSplitDocs.elapsed(MICROSECONDS), MICROSECONDS),
+ df.format(detailedGCDocs.elapsed(MICROSECONDS), MICROSECONDS),
+ df.format(deleteDetailedGCDocs.elapsed(MICROSECONDS), MICROSECONDS),
timeDeletingSplitDocs);
}
@@ -329,6 +343,8 @@ public class VersionGarbageCollector {
", deletedDocGCCount=" + deletedDocGCCount + " (of which leaf: " + deletedLeafDocGCCount + ")" +
", updateResurrectedGCCount=" + updateResurrectedGCCount +
", splitDocGCCount=" + splitDocGCCount +
+ ", updatedDetailedGCDocsCount=" + updatedDetailedGCDocsCount +
+ ", deletedPropsGCCount=" + deletedPropsGCCount +
", intermediateSplitDocGCCount=" + intermediateSplitDocGCCount +
", iterationCount=" + iterationCount +
", timeActive=" + df.format(activeElapsed, MICROSECONDS) +
@@ -346,30 +362,40 @@ public class VersionGarbageCollector {
this.deletedDocGCCount += run.deletedDocGCCount;
this.deletedLeafDocGCCount += run.deletedLeafDocGCCount;
this.splitDocGCCount += run.splitDocGCCount;
+ this.updatedDetailedGCDocsCount += run.updatedDetailedGCDocsCount;
+ this.deletedPropsGCCount += run.deletedPropsGCCount;
this.intermediateSplitDocGCCount += run.intermediateSplitDocGCCount;
this.updateResurrectedGCCount += run.updateResurrectedGCCount;
if (run.iterationCount > 0) {
// run is cumulative with times in elapsed fields
this.activeElapsed += run.activeElapsed;
this.collectDeletedDocsElapsed += run.collectDeletedDocsElapsed;
+ this.collectDeletedPropsElapsed += run.collectDeletedPropsElapsed;
+ this.collectDeletedOldRevsElapsed += run.collectDeletedOldRevsElapsed;
+ this.collectUnmergedBCElapsed += run.collectUnmergedBCElapsed;
this.checkDeletedDocsElapsed += run.checkDeletedDocsElapsed;
this.deleteDeletedDocsElapsed += run.deleteDeletedDocsElapsed;
this.collectAndDeleteSplitDocsElapsed += run.collectAndDeleteSplitDocsElapsed;
this.deleteSplitDocsElapsed += run.deleteSplitDocsElapsed;
this.sortDocIdsElapsed += run.sortDocIdsElapsed;
this.updateResurrectedDocumentsElapsed += run.updateResurrectedDocumentsElapsed;
- this.detailedGcDocsElapsed += run.detailedGcDocsElapsed;
+ this.detailedGCDocsElapsed += run.detailedGCDocsElapsed;
+ this.deleteDetailedGCDocsElapsed += run.deleteDetailedGCDocsElapsed;
} else {
// single run -> read from stop watches
this.activeElapsed += run.active.elapsed(MICROSECONDS);
this.collectDeletedDocsElapsed += run.collectDeletedDocs.elapsed(MICROSECONDS);
+ this.collectDeletedPropsElapsed += run.collectDeletedProps.elapsed(MICROSECONDS);
+ this.collectDeletedOldRevsElapsed += run.collectDeletedOldRevs.elapsed(MICROSECONDS);
+ this.collectUnmergedBCElapsed += run.collectUnmergedBC.elapsed(MICROSECONDS);
this.checkDeletedDocsElapsed += run.checkDeletedDocs.elapsed(MICROSECONDS);
this.deleteDeletedDocsElapsed += run.deleteDeletedDocs.elapsed(MICROSECONDS);
this.collectAndDeleteSplitDocsElapsed += run.collectAndDeleteSplitDocs.elapsed(MICROSECONDS);
this.deleteSplitDocsElapsed += run.deleteSplitDocs.elapsed(MICROSECONDS);
this.sortDocIdsElapsed += run.sortDocIds.elapsed(MICROSECONDS);
this.updateResurrectedDocumentsElapsed += run.updateResurrectedDocuments.elapsed(MICROSECONDS);
- this.detailedGcDocsElapsed += run.detailedGcDocs.elapsed(MICROSECONDS);
+ this.detailedGCDocsElapsed += run.detailedGCDocs.elapsed(MICROSECONDS);
+ this.deleteDetailedGCDocsElapsed += run.deleteDetailedGCDocs.elapsed(MICROSECONDS);
}
}
}
@@ -379,9 +405,13 @@ public class VersionGarbageCollector {
COLLECTING,
CHECKING,
DETAILED_GC,
+ COLLECT_PROPS,
+ COLLECT_OLD_REVS,
+ COLLECT_UNMERGED_BC,
DELETING,
SORTING,
SPLITS_CLEANUP,
+ DETAILED_GC_CLEANUP,
UPDATING
}
@@ -406,7 +436,11 @@ public class VersionGarbageCollector {
this.watches.put(GCPhase.NONE, Stopwatch.createStarted());
this.watches.put(GCPhase.COLLECTING, stats.collectDeletedDocs);
this.watches.put(GCPhase.CHECKING, stats.checkDeletedDocs);
- this.watches.put(GCPhase.DETAILED_GC, stats.detailedGcDocs);
+ this.watches.put(GCPhase.DETAILED_GC, stats.detailedGCDocs);
+ this.watches.put(GCPhase.DETAILED_GC_CLEANUP, stats.deleteDetailedGCDocs);
+ this.watches.put(GCPhase.COLLECT_PROPS, stats.collectDeletedProps);
+ this.watches.put(GCPhase.COLLECT_OLD_REVS, stats.collectDeletedOldRevs);
+ this.watches.put(GCPhase.COLLECT_UNMERGED_BC, stats.collectUnmergedBC);
this.watches.put(GCPhase.DELETING, stats.deleteDeletedDocs);
this.watches.put(GCPhase.SORTING, stats.sortDocIds);
this.watches.put(GCPhase.SPLITS_CLEANUP, stats.collectAndDeleteSplitDocs);
@@ -534,7 +568,7 @@ public class VersionGarbageCollector {
collectDeletedDocuments(phases, headRevision, rec);
collectSplitDocuments(phases, sweepRevisions, rec);
if (detailedGCEnabled) {
- // run only if enabled
+ // run only if detailed GC enabled
collectDetailedGarbage(phases, headRevision, rec);
}
}
@@ -568,102 +602,74 @@ public class VersionGarbageCollector {
* it is okay that it takes a considerable amount of time.
*
* @param phases {@link GCPhases}
- * @param headRevision the current head revision of
- * @throws IOException
- * @throws LimitExceededException
+ * @param headRevision the current head revision of node store
*/
private void collectDetailedGarbage(final GCPhases phases, final RevisionVector headRevision, final VersionGCRecommendations rec)
throws IOException, LimitExceededException {
int docsTraversed = 0;
- long oldestModifiedGced = rec.scopeFullGC.fromMs;
+ boolean foundDoc = true;
+ long oldestModifiedGCed = rec.scopeFullGC.fromMs;
try (DetailedGC gc = new DetailedGC(headRevision, monitor, cancel)) {
final long fromModified = rec.scopeFullGC.fromMs;
final long toModified = rec.scopeFullGC.toMs;
-// if (rec.fullDetailGCTimestamp == -1) {
-// // then full detail-gc is disabled or over - use regular scope then
-// fromModified = rec.scope.fromMs;
-// toModified = rec.scope.toMs;
-// } else {
-// // then full detail-gc is enabled - use it then
-// fromModified = rec.fullDetailGCTimestamp; // TODO: once we're passed rec.scope.fromMs we should
-// // disable fullgc
-// toModified = rec.scope.toMs; // the 'to' here is the max. it will process only eg 1 batch
-// }
- // TODO : remove me
- boolean foundAnything = false; // I think this flag is redundant
- if (phases.start(GCPhase.COLLECTING)) {
- Iterable<NodeDocument> itr = versionStore.getModifiedDocs(oldestModifiedGced, toModified, 2000);
- final Stopwatch timer = Stopwatch.createUnstarted();
- timer.reset().start();
- try {
- for (NodeDocument doc : itr) {
- // continue with GC?
- if (cancel.get()) {
- break;
- }
- if (phases.start(GCPhase.DETAILED_GC)) {
- gc.detailedGC(doc, phases);
- phases.stop(GCPhase.DETAILED_GC);
- }
+ if (phases.start(GCPhase.DETAILED_GC)) {
+ while (foundDoc && oldestModifiedGCed < toModified && docsTraversed <= PROGRESS_BATCH_SIZE) {
+ // set foundDoc to false to allow exiting the while loop
+ foundDoc = false;
+ Iterable<NodeDocument> itr = versionStore.getModifiedDocs(oldestModifiedGCed, toModified, 1000);
+ try {
+ for (NodeDocument doc : itr) {
+ foundDoc = true;
+ // continue with GC?
+ if (cancel.get()) {
+ break;
+ }
+ docsTraversed++;
+ if (docsTraversed % PROGRESS_BATCH_SIZE == 0) {
+ monitor.info("Iterated through {} documents so far. {} had detail garbage",
+ docsTraversed, gc.getGarbageDocsCount());
+ }
- // TODO : remove this code, I don't think its possible to fetch these documents
- // who doesn't have _modified field
- final Long modified = doc.getModified();
- if (modified == null) {
- monitor.warn("collectDetailGarbage : document has no _modified property : {}",
- doc.getId());
- } else if (modified < oldestModifiedGced) {
- monitor.warn(
- "collectDetailGarbage : document has older _modified than query boundary : {} (from: {}, to: {})",
- modified, fromModified, toModified);
- } else {
- oldestModifiedGced = modified;
- }
- foundAnything = true;
- docsTraversed++;
- if (docsTraversed % PROGRESS_BATCH_SIZE == 0) {
- monitor.info("Iterated through {} documents so far. {} had detail garbage",
- docsTraversed, gc.getNumDocuments());
- }
- // this would never hit, since we are only fetching the oldest 2000 element in batches of 1000
- // TODO: remove this if above mentioned logic is fine
- if (rec.maxCollect > 0 && gc.getNumDocuments() > rec.maxCollect) {
- // TODO: how would we recover from this?
- // If we don't want above solution, then one of the another solution is to use lower time duration
- // as done in document deletion process or use lower limit value or
- // we should perform all the update ops in 1 go
- throw new LimitExceededException();
+ // collect the data to delete in next step
+ if (phases.start(GCPhase.COLLECTING)) {
+ gc.collectGarbage(doc, phases);
+ phases.stop(GCPhase.COLLECTING);
+ }
+
+ // TODO : remove this code, I don't think its possible to fetch these documents
+ // who doesn't have _modified field
+ final Long modified = doc.getModified();
+ if (modified == null) {
+ monitor.warn("collectDetailGarbage : document has no _modified property : {}",
+ doc.getId());
+ } else if (modified < oldestModifiedGCed) {
+ monitor.warn(
+ "collectDetailGarbage : document has older _modified than query boundary : {} (from: {}, to: {})",
+ modified, fromModified, toModified);
+ } else {
+ oldestModifiedGCed = modified;
+ }
+
+ if (gc.hasGarbage()) {
+ if (phases.start(GCPhase.DETAILED_GC_CLEANUP)) {
+ gc.removeGarbage(phases.stats);
+ phases.stop(GCPhase.DETAILED_GC_CLEANUP);
+ }
+ }
+
+ oldestModifiedGCed = modified == null ? fromModified : modified;
}
- oldestModifiedGced = modified == null ? fromModified : modified;
+ } finally {
+ Utils.closeIfCloseable(itr);
+ phases.stats.oldestModifiedGced = oldestModifiedGCed;
}
- } finally {
- Utils.closeIfCloseable(itr);
- // why do we need to stop this here, we are already stopping the original gc run.
- // can this be removed
- delayOnModifications(timer.stop().elapsed(TimeUnit.MILLISECONDS));
- phases.stats.oldestModifiedGced = oldestModifiedGced;
}
- phases.stop(GCPhase.COLLECTING);
-// if (!cancel.get() && foundAnything) {
-// // TODO: move to evaluate()
-// rec.setLongSetting(SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP, oldestModifiedGced + 1);
-// }
+ phases.stop(GCPhase.DETAILED_GC);
}
}
}
- private void delayOnModifications(long durationMs) {
- long delayMs = Math.round(durationMs * options.delayFactor);
- if (!cancel.get() && delayMs > 0) {
- try {
- Clock clock = nodeStore.getClock();
- clock.waitUntil(clock.getTime() + delayMs);
- }
- catch (InterruptedException ex) {
- /* ignore */
- }
- }
- }
+
private void collectSplitDocuments(GCPhases phases,
RevisionVector sweepRevisions,
@@ -752,77 +758,144 @@ public class VersionGarbageCollector {
}
}
- private static class DetailedGC implements Closeable {
+ private class DetailedGC implements Closeable {
private final RevisionVector headRevision;
private final GCMonitor monitor;
private final AtomicBoolean cancel;
- private int count;
+ private final Stopwatch timer;
+ private final List<UpdateOp> updateOpList;
+ private int garbageDocsCount;
public DetailedGC(@NotNull RevisionVector headRevision, @NotNull GCMonitor monitor, @NotNull AtomicBoolean cancel) {
this.headRevision = requireNonNull(headRevision);
this.monitor = monitor;
this.cancel = cancel;
+ this.updateOpList = new ArrayList<>();
+ this.timer = Stopwatch.createUnstarted();
}
- public void detailedGC(NodeDocument doc, GCPhases phases) {
-// deleteSample(doc, phases);
- UpdateOp updateOp = new UpdateOp(requireNonNull(doc.getId()), false);
- deleteDeletedProperties(doc, phases, updateOp);
- deleteUnmergedBranchCommitDocument(doc, phases, updateOp);
- deleteOldRevisions(doc, phases, updateOp);
+ public void collectGarbage(final NodeDocument doc, final GCPhases phases) {
+
+ monitor.info("Collecting Detailed Garbage for doc [{}]", doc.getId());
+
+ final UpdateOp op = new UpdateOp(requireNonNull(doc.getId()), false);
+ collectDeletedProperties(doc, phases, op);
+ collectUnmergedBranchCommitDocument(doc, phases, op);
+ collectOldRevisions(doc, phases, op);
+ // only add if there are changes for this doc
+ if (op.hasChanges()) {
+ garbageDocsCount++;
+ monitor.info("Collected [{}] garbage for doc [{}]", op.getChanges().size(), doc.getId());
+ updateOpList.add(op);
+ }
}
- /** TODO remove, this is just a skeleton sample */
-// private void deleteSample(NodeDocument doc, GCPhases phases) {
-// if (doc.getId().contains("should_delete")) {
-// if (phases.start(GCPhase.DELETING)) {
-// monitor.info("deleteSample: should do the deletion now, but this is demo only. I'm still learning");
-// System.out.println("do the actual deletion");
-// count++;
-// phases.stop(GCPhase.DELETING);
-// }
-// }
-// }
+ private boolean hasGarbage() {
+ return garbageDocsCount > 0;
+ }
- private void deleteUnmergedBranchCommitDocument(NodeDocument doc, GCPhases phases, UpdateOp updateOp) {
- // TODO Auto-generated method stub
+ private void collectUnmergedBranchCommitDocument(final NodeDocument doc, final GCPhases phases, final UpdateOp updateOp) {
+ if (phases.start(GCPhase.COLLECT_UNMERGED_BC)){
+ // TODO add umerged BC collection logic
+ phases.stop(GCPhase.COLLECT_UNMERGED_BC);
+ }
}
- private void deleteDeletedProperties(final NodeDocument doc, final GCPhases phases, final UpdateOp updateOp) {
+ private void collectDeletedProperties(final NodeDocument doc, final GCPhases phases, final UpdateOp updateOp) {
// get Map of all properties along with their values
- final Map<String, SortedMap<Revision, String>> properties = doc.getProperties();
-
- // find all the properties which can be removed from document
- // All the properties whose value is null in their respective
- // latest revision are eligible to be garbage collected.
- properties.forEach((propName, revisionStringSortedMap) -> {
- if (revisionStringSortedMap.keySet()
- .stream()
- .sorted(REVERSE)
- .limit(1)
- .anyMatch(revision -> revisionStringSortedMap.get(revision) == null)) {
- // set this property for removal
- updateOp.remove(propName);
+ if (phases.start(GCPhase.COLLECT_PROPS)) {
+ final Set<String> properties = doc.getPropertyNames();
+
+ // find all the properties which can be removed from document.
+ // All the properties whose value is null in head revision are
+ // eligible to be garbage collected.
+
+ final Set<String> retainPropSet = ofNullable(doc.getNodeAtRevision(nodeStore, headRevision, null))
+ .map(DocumentNodeState::getPropertyNames)
+ .orElse(emptySet());
+ final int deletedPropsGCCount = properties.stream()
+ .filter(p -> !retainPropSet.contains(p))
+ .mapToInt(x -> {
+ updateOp.remove(x);
+ return 1;})
+ .sum();
+
+
+ phases.stats.deletedPropsGCCount += deletedPropsGCCount;
+ if (log.isDebugEnabled()) {
+ log.debug("Collected {} deleted properties for document {}", deletedPropsGCCount, doc.getId());
}
- });
+ phases.stop(GCPhase.COLLECT_PROPS);
+ }
}
- private void deleteOldRevisions(NodeDocument doc, GCPhases phases, UpdateOp updateOp) {
- // TODO Auto-generated method stub
+ private void collectOldRevisions(NodeDocument doc, GCPhases phases, UpdateOp updateOp) {
+
+ if (phases.start(GCPhase.COLLECT_OLD_REVS)){
+ // TODO add old rev collection logic
+ phases.stop(GCPhase.COLLECT_OLD_REVS);
+ }
}
- long getNumDocuments() {
- return count;
+ int getGarbageDocsCount() {
+ return garbageDocsCount;
}
@Override
public void close() throws IOException {
}
+
+ public void removeGarbage(final VersionGCStats stats) {
+
+ if (updateOpList.isEmpty()) {
+ log.info("Skipping removal of detailed garbage, cause no garbage detected for docs [{}]", updateOpList.stream().map(UpdateOp::getId).collect(Collectors.joining(",")));
+ return;
+ }
+
+ int updatedDocs;
+
+ monitor.info("Proceeding to update [{}] documents", updateOpList.size());
+
+ if (log.isDebugEnabled()) {
+ String collect = updateOpList.stream().map(UpdateOp::getId).collect(Collectors.joining(","));
+ log.trace("Performing batch update of documents with following id's. \n" + collect);
+ }
+
+ if (cancel.get()) {
+ log.info("Aborting the removal of detailed garbage since RGC had been cancelled");
+ return;
+ }
+
+ timer.reset().start();
+ try {
+ // TODO create an api to bulk update findAndUpdate Ops
+ updatedDocs = (int) updateOpList.stream().map(op -> ds.findAndUpdate(NODES, op)).filter(Objects::nonNull).count();
+ stats.updatedDetailedGCDocsCount += updatedDocs;
+ log.info("Updated [{}] documents", updatedDocs);
+ // now reset delete metadata
+ updateOpList.clear();
+ garbageDocsCount = 0;
+ } finally {
+ delayOnModifications(timer.stop().elapsed(MILLISECONDS), cancel);
+ }
+ }
+ }
+ private void delayOnModifications(final long durationMs, final AtomicBoolean cancel) {
+ long delayMs = round(durationMs * options.delayFactor);
+ if (!cancel.get() && delayMs > 0) {
+ try {
+ Clock clock = nodeStore.getClock();
+ clock.waitUntil(clock.getTime() + delayMs);
+ }
+ catch (InterruptedException ex) {
+ /* ignore */
+ }
+ }
}
/**
@@ -959,19 +1032,6 @@ public class VersionGarbageCollector {
//------------------------------< internal >----------------------------
- private void delayOnModifications(long durationMs) {
- long delayMs = Math.round(durationMs * options.delayFactor);
- if (!cancel.get() && delayMs > 0) {
- try {
- Clock clock = nodeStore.getClock();
- clock.waitUntil(clock.getTime() + delayMs);
- }
- catch (InterruptedException ex) {
- /* ignore */
- }
- }
- }
-
private Iterator<String> previousDocIdsFor(NodeDocument doc) {
Map<Revision, Range> prevRanges = doc.getPreviousRanges(true);
if (prevRanges.isEmpty()) {
@@ -1127,7 +1187,7 @@ public class VersionGarbageCollector {
monitor.info(msg);
}
} finally {
- delayOnModifications(timer.stop().elapsed(TimeUnit.MILLISECONDS));
+ delayOnModifications(timer.stop().elapsed(MILLISECONDS), cancel);
}
}
return deletedCount;
@@ -1160,7 +1220,7 @@ public class VersionGarbageCollector {
}
}
finally {
- delayOnModifications(timer.stop().elapsed(TimeUnit.MILLISECONDS));
+ delayOnModifications(timer.stop().elapsed(MILLISECONDS), cancel);
}
return updateCount;
}
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java
index 0873cb448b..017d6cbe20 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java
@@ -1886,7 +1886,7 @@ public class MongoDocumentStore implements DocumentStore {
* @param failOnInsert whether to create an update that will fail on insert.
* @return the DBObject.
*/
- private static BasicDBObject createUpdate(UpdateOp updateOp,
+ private static BasicDBObject createUpdate(UpdateOp updateOp,
boolean failOnInsert) {
BasicDBObject setUpdates = new BasicDBObject();
BasicDBObject maxUpdates = new BasicDBObject();
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java
index 5caa65d875..26fc1311fa 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java
@@ -715,6 +715,7 @@ public class RDBDocumentStoreJDBC {
}
if (sortBy != null) {
+ // FIXME : order should be determined via sortBy field
query.append(" order by ID");
}
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java
index a463499793..66012d09ce 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java
@@ -17,6 +17,9 @@
package org.apache.jackrabbit.oak.plugins.document.rdb;
import static org.apache.jackrabbit.guava.common.collect.Iterables.filter;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MODIFIED_IN_SECS;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.getModifiedInSecs;
+import static org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStore.EMPTY_KEY_PATTERN;
import java.io.Closeable;
import java.io.IOException;
@@ -85,6 +88,30 @@ public class RDBVersionGCSupport extends VersionGCSupport {
}
}
+ /**
+ * Returns documents that have a {@link NodeDocument#MODIFIED_IN_SECS} value
+ * within the given range .The two passed modified timestamps are in milliseconds
+ * since the epoch and the implementation will convert them to seconds at
+ * the granularity of the {@link NodeDocument#MODIFIED_IN_SECS} field and
+ * then perform the comparison.
+ *
+ * @param fromModified the lower bound modified timestamp (inclusive)
+ * @param toModified the upper bound modified timestamp (exclusive)
+ * @param limit the limit of documents to return
+ * @return matching documents.
+ */
+ @Override
+ public Iterable<NodeDocument> getModifiedDocs(final long fromModified, final long toModified, final int limit) {
+ List<QueryCondition> conditions = new ArrayList<>();
+ conditions.add(new QueryCondition(MODIFIED_IN_SECS, "<", getModifiedInSecs(toModified)));
+ conditions.add(new QueryCondition(MODIFIED_IN_SECS, ">=", getModifiedInSecs(fromModified)));
+ if (MODE == 1) {
+ return getIterator(EMPTY_KEY_PATTERN, conditions);
+ } else {
+ return store.queryAsIterable(Collection.NODES, null, null, EMPTY_KEY_PATTERN, conditions, limit, MODIFIED_IN_SECS);
+ }
+ }
+
@Override
protected Iterable<NodeDocument> identifyGarbage(final Set<SplitDocType> gcTypes, final RevisionVector sweepRevs,
final long oldestRevTimeStamp) {
@@ -136,7 +163,7 @@ public class RDBVersionGCSupport extends VersionGCSupport {
List<QueryCondition> conditions1 = new ArrayList<QueryCondition>();
conditions1.add(new QueryCondition(NodeDocument.SD_TYPE, "in", gcTypeCodes));
- conditions1.add(new QueryCondition(NodeDocument.SD_MAX_REV_TIME_IN_SECS, "<=", NodeDocument.getModifiedInSecs(oldestRevTimeStamp)));
+ conditions1.add(new QueryCondition(NodeDocument.SD_MAX_REV_TIME_IN_SECS, "<=", getModifiedInSecs(oldestRevTimeStamp)));
conditions1.add(new QueryCondition(RDBDocumentStore.VERSIONPROP, ">=", 2));
name1 = "version 2 query";
it1 = store.queryAsIterable(Collection.NODES, null, null, Collections.emptyList(), conditions1,
@@ -222,8 +249,8 @@ public class RDBVersionGCSupport extends VersionGCSupport {
LOG.debug("getOldestDeletedOnceTimestamp() <- start");
try {
- long modifiedSec = store.getMinValue(Collection.NODES, NodeDocument.MODIFIED_IN_SECS, null, null,
- RDBDocumentStore.EMPTY_KEY_PATTERN,
+ long modifiedSec = store.getMinValue(Collection.NODES, MODIFIED_IN_SECS, null, null,
+ EMPTY_KEY_PATTERN,
Collections.singletonList(new QueryCondition(NodeDocument.DELETED_ONCE, "=", 1)));
modifiedMs = TimeUnit.SECONDS.toMillis(modifiedSec);
} catch (DocumentStoreException ex) {
@@ -241,7 +268,7 @@ public class RDBVersionGCSupport extends VersionGCSupport {
@Override
public long getDeletedOnceCount() {
- return store.queryCount(Collection.NODES, null, null, RDBDocumentStore.EMPTY_KEY_PATTERN,
+ return store.queryCount(Collection.NODES, null, null, EMPTY_KEY_PATTERN,
Collections.singletonList(new QueryCondition(NodeDocument.DELETED_ONCE, "=", 1)));
}
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DetailGCHelper.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DetailGCHelper.java
index 8a585c7dc0..d52c5e33ae 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DetailGCHelper.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DetailGCHelper.java
@@ -18,25 +18,19 @@
*/
package org.apache.jackrabbit.oak.plugins.document;
-public class DetailGCHelper {
+import static org.apache.commons.lang3.reflect.FieldUtils.writeField;
- public static void setLongSetting(String propName, long val, DocumentNodeStore ns) {
- UpdateOp updateOp = new UpdateOp(VersionGarbageCollector.SETTINGS_COLLECTION_ID, true);
- updateOp.set(propName, val);
- ns.getDocumentStore().createOrUpdate(Collection.SETTINGS, updateOp);
- }
+public class DetailGCHelper {
- public static void enableDetailGC(DocumentNodeStore ns) {
- VersionGarbageCollector.DETAIL_GC_ENABLED = true;
- if (ns != null) {
- setLongSetting(VersionGarbageCollector.SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP, 0, ns);
+ public static void enableDetailGC(final VersionGarbageCollector vgc) throws IllegalAccessException {
+ if (vgc != null) {
+ writeField(vgc, "detailedGCEnabled", true, true);
}
}
- public static void disableDetailGC(DocumentNodeStore ns) {
- VersionGarbageCollector.DETAIL_GC_ENABLED = false;
- if (ns != null) {
- setLongSetting(VersionGarbageCollector.SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP, -1, ns);
+ public static void disableDetailGC(final VersionGarbageCollector vgc) throws IllegalAccessException {
+ if (vgc != null) {
+ writeField(vgc, "detailedGCEnabled", false, true);
}
}
}
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentTest.java
index b99897dfb0..2adcc9f86a 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentTest.java
@@ -449,6 +449,38 @@ public class NodeDocumentTest {
assertEquals(140, uncommittedRevisions);
}
+ @Test
+ public void getPropertyNames() throws CommitFailedException {
+ DocumentStore store = new MemoryDocumentStore();
+ DocumentNodeStore ns = new DocumentMK.Builder().setDocumentStore(store).setAsyncDelay(0).getNodeStore();
+
+ // add properties
+ for (int i = 0; i < 10; i++) {
+ NodeBuilder nb = ns.getRoot().builder();
+ nb.child("x").setProperty("p"+i, i);
+ merge(ns, nb);
+ }
+
+ final NodeDocument nodeDocument = store.find(NODES, "1:/x");
+ assert nodeDocument != null;
+ assertEquals(10, nodeDocument.getPropertyNames().size());
+ }
+
+ @Test
+ public void getNoPropertyNames() throws CommitFailedException {
+ DocumentStore store = new MemoryDocumentStore();
+ DocumentNodeStore ns = new DocumentMK.Builder().setDocumentStore(store).setAsyncDelay(0).getNodeStore();
+
+ // add no property
+ NodeBuilder nb = ns.getRoot().builder();
+ nb.child("x");
+ merge(ns, nb);
+
+ final NodeDocument nodeDocument = store.find(NODES, "1:/x");
+ assert nodeDocument != null;
+ assertEquals(0, nodeDocument.getPropertyNames().size());
+ }
+
@Test
public void getNewestRevisionTooExpensive() throws Exception {
final int NUM_CHANGES = 200;
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCInitTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCInitTest.java
index 055188be46..ed39a372b2 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCInitTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCInitTest.java
@@ -22,6 +22,9 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
+import static org.apache.jackrabbit.oak.plugins.document.DetailGCHelper.enableDetailGC;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_DETAILED_GC_TIMESTAMP_PROP;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
@@ -49,6 +52,20 @@ public class VersionGCInitTest {
vgc = store.find(Collection.SETTINGS, "versionGC");
assertNotNull(vgc);
+ assertEquals(0L, vgc.get(SETTINGS_COLLECTION_DETAILED_GC_TIMESTAMP_PROP));
}
+ @Test
+ public void lazyInitializeWithDetailedGC() throws Exception {
+ DocumentStore store = ns.getDocumentStore();
+ Document vgc = store.find(Collection.SETTINGS, "versionGC");
+ assertNull(vgc);
+
+ enableDetailGC(ns.getVersionGarbageCollector());
+ ns.getVersionGarbageCollector().gc(1, TimeUnit.DAYS);
+
+ vgc = store.find(Collection.SETTINGS, "versionGC");
+ assertNotNull(vgc);
+ assertEquals(-1L, vgc.get(SETTINGS_COLLECTION_DETAILED_GC_TIMESTAMP_PROP));
+ }
}
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCStatsTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCStatsTest.java
index 13448e5403..1515ea9312 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCStatsTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCStatsTest.java
@@ -67,6 +67,11 @@ public class VersionGCStatsTest {
assertEquals(stats.collectAndDeleteSplitDocs.elapsed(MICROSECONDS), cumulative.collectAndDeleteSplitDocsElapsed);
assertEquals(stats.sortDocIds.elapsed(MICROSECONDS), cumulative.sortDocIdsElapsed);
assertEquals(stats.updateResurrectedDocuments.elapsed(MICROSECONDS), cumulative.updateResurrectedDocumentsElapsed);
+ assertEquals(stats.detailedGCDocs.elapsed(MICROSECONDS), cumulative.detailedGCDocsElapsed);
+ assertEquals(stats.deleteDetailedGCDocs.elapsed(MICROSECONDS), cumulative.deleteDetailedGCDocsElapsed);
+ assertEquals(stats.collectDeletedProps.elapsed(MICROSECONDS), cumulative.collectDeletedPropsElapsed);
+ assertEquals(stats.collectDeletedOldRevs.elapsed(MICROSECONDS), cumulative.collectDeletedOldRevsElapsed);
+ assertEquals(stats.collectUnmergedBC.elapsed(MICROSECONDS), cumulative.collectUnmergedBCElapsed);
}
@Test
@@ -83,6 +88,11 @@ public class VersionGCStatsTest {
assertEquals(stats.collectAndDeleteSplitDocs.elapsed(MICROSECONDS) * 2, cumulative.collectAndDeleteSplitDocsElapsed);
assertEquals(stats.sortDocIds.elapsed(MICROSECONDS) * 2, cumulative.sortDocIdsElapsed);
assertEquals(stats.updateResurrectedDocuments.elapsed(MICROSECONDS) * 2, cumulative.updateResurrectedDocumentsElapsed);
+ assertEquals(stats.detailedGCDocs.elapsed(MICROSECONDS) * 2, cumulative.detailedGCDocsElapsed);
+ assertEquals(stats.deleteDetailedGCDocs.elapsed(MICROSECONDS) * 2, cumulative.deleteDetailedGCDocsElapsed);
+ assertEquals(stats.collectDeletedProps.elapsed(MICROSECONDS) * 2, cumulative.collectDeletedPropsElapsed);
+ assertEquals(stats.collectDeletedOldRevs.elapsed(MICROSECONDS) * 2, cumulative.collectDeletedOldRevsElapsed);
+ assertEquals(stats.collectUnmergedBC.elapsed(MICROSECONDS) * 2, cumulative.collectUnmergedBCElapsed);
}
private void forEachStopwatch(VersionGCStats stats, Callable c) {
@@ -93,6 +103,11 @@ public class VersionGCStatsTest {
c.call(stats.collectAndDeleteSplitDocs);
c.call(stats.sortDocIds);
c.call(stats.updateResurrectedDocuments);
+ c.call(stats.detailedGCDocs);
+ c.call(stats.deleteDetailedGCDocs);
+ c.call(stats.collectDeletedProps);
+ c.call(stats.collectDeletedOldRevs);
+ c.call(stats.collectUnmergedBC);
}
private interface Callable {
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
index 48f3f362ce..f29716ca5d 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
@@ -44,7 +44,6 @@ import org.jetbrains.annotations.NotNull;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
@@ -96,7 +95,7 @@ public class VersionGCTest {
@After
public void tearDown() throws Exception {
- DetailGCHelper.disableDetailGC(ns);
+ DetailGCHelper.disableDetailGC(gc);
execService.shutdown();
execService.awaitTermination(1, MINUTES);
}
@@ -343,21 +342,19 @@ public class VersionGCTest {
// OAK-10199
@Test
- @Ignore
public void testDetailGcDocumentRead_disabled() throws Exception {
- DetailGCHelper.disableDetailGC(ns);
+ DetailGCHelper.disableDetailGC(gc);
VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
assertNotNull(stats);
- assertEquals(0, stats.detailedGcDocsElapsed);
+ assertEquals(0, stats.detailedGCDocsElapsed);
}
@Test
- @Ignore
public void testDetailGcDocumentRead_enabled() throws Exception {
- DetailGCHelper.enableDetailGC(ns);
+ DetailGCHelper.enableDetailGC(gc);
VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
assertNotNull(stats);
- assertNotEquals(0, stats.detailedGcDocsElapsed);
+ assertNotEquals(0, stats.detailedGCDocsElapsed);
}
// OAK-10199
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
index d33cc8c7de..d6ec9774dd 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
@@ -23,7 +23,9 @@ import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
+import java.util.SortedMap;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
@@ -34,10 +36,12 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import static org.apache.commons.lang3.reflect.FieldUtils.writeField;
import static org.apache.jackrabbit.guava.common.collect.Iterables.filter;
import static org.apache.jackrabbit.guava.common.collect.Iterables.size;
import static java.util.concurrent.TimeUnit.HOURS;
import static java.util.concurrent.TimeUnit.MINUTES;
+import static org.apache.jackrabbit.oak.api.Type.STRING;
import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.NUM_REVS_THRESHOLD;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.PREV_SPLIT_FACTOR;
@@ -225,6 +229,68 @@ public class VersionGarbageCollectorIT {
public void gcLongPathSplitDocs() throws Exception {
gcSplitDocsInternal(Strings.repeat("sub", 120));
}
+
+ @Test
+ public void testGCDeletedProps() throws Exception{
+ //1. Create nodes
+ NodeBuilder b1 = store.getRoot().builder();
+
+ // Add property to node & save
+ b1.child("x").setProperty("test", "t", STRING);
+ b1.child("z").setProperty("prop", "foo", STRING);
+ store.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ // update the property
+ b1 = store.getRoot().builder();
+ b1.getChildNode("z").setProperty("prop", "bar", STRING);
+ store.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ // update property again
+ b1 = store.getRoot().builder();
+ b1.getChildNode("z").setProperty("prop", "baz", STRING);
+ store.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ // enable the detailed gc flag
+ writeField(gc, "detailedGCEnabled", true, true);
+ long maxAge = 1; //hours
+ long delta = TimeUnit.MINUTES.toMillis(10);
+ //1. Go past GC age and check no GC done as nothing deleted
+ clock.waitUntil(Revision.getCurrentTimestamp() + maxAge);
+ VersionGCStats stats = gc.gc(maxAge, HOURS);
+ assertEquals(0, stats.deletedPropsGCCount);
+
+ //Remove property
+ NodeBuilder b2 = store.getRoot().builder();
+ b2.getChildNode("z").removeProperty("prop");
+ store.merge(b2, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ store.runBackgroundOperations();
+
+ //2. Check that a deleted property is not collected before maxAge
+ //Clock cannot move back (it moved forward in #1) so double the maxAge
+ clock.waitUntil(clock.getTime() + delta);
+ stats = gc.gc(maxAge*2, HOURS);
+ assertEquals(0, stats.deletedPropsGCCount);
+
+ //3. Check that deleted property does get collected post maxAge
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
+
+ stats = gc.gc(maxAge*2, HOURS);
+ assertEquals(1, stats.deletedPropsGCCount);
+
+ //4. Check that a revived property (deleted and created again) does not get gc
+ NodeBuilder b3 = store.getRoot().builder();
+ b3.child("x").removeProperty("test");
+ store.merge(b3, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ NodeBuilder b4 = store.getRoot().builder();
+ b4.child("x").setProperty("test", "t", STRING);
+ store.merge(b4, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
+ stats = gc.gc(maxAge*2, HOURS);
+ assertEquals(0, stats.deletedPropsGCCount);
+ }
private void gcSplitDocsInternal(String subNodeName) throws Exception {
long maxAge = 1; //hrs
[jackrabbit-oak] 02/04: OAK-10199 : disable the detailGc in tearDown to avoid side-effects
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
daim pushed a commit to branch OAK-10199
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 4e840d4e30230325f6634f1c62323fc5840027a3
Author: Stefan Egli <st...@apache.org>
AuthorDate: Thu Apr 20 18:38:45 2023 +0200
OAK-10199 : disable the detailGc in tearDown to avoid side-effects
---
.../java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
index 1bd81ce89c..445e7c4275 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
@@ -95,6 +95,7 @@ public class VersionGCTest {
@After
public void tearDown() throws Exception {
+ DetailGCHelper.disableDetailGC(ns);
execService.shutdown();
execService.awaitTermination(1, MINUTES);
}
[jackrabbit-oak] 01/04: OAK-10199 : initial sketch of detail gc skeleton
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
daim pushed a commit to branch OAK-10199
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit bd2a1f69a00b94b7832cdab707c4bd69b06da009
Author: Stefan Egli <st...@apache.org>
AuthorDate: Thu Apr 20 18:11:08 2023 +0200
OAK-10199 : initial sketch of detail gc skeleton
---
.../plugins/document/VersionGCRecommendations.java | 6 +-
.../oak/plugins/document/VersionGCSupport.java | 25 +++
.../plugins/document/VersionGarbageCollector.java | 183 ++++++++++++++++++++-
.../oak/plugins/document/DetailGCHelper.java | 42 +++++
.../oak/plugins/document/VersionGCTest.java | 18 ++
5 files changed, 272 insertions(+), 2 deletions(-)
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java
index 363c65789b..ac47cc69d8 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java
@@ -46,6 +46,7 @@ public class VersionGCRecommendations {
final long maxCollect;
final long deleteCandidateCount;
final long lastOldestTimestamp;
+ final long fullDetailGCTimestamp;
final long originalCollectLimit;
private final long precisionMs;
@@ -101,6 +102,8 @@ public class VersionGCRecommendations {
TimeInterval scope = new TimeInterval(oldestPossible, Long.MAX_VALUE);
scope = scope.notLaterThan(keep.fromMs);
+ fullDetailGCTimestamp = settings.get(VersionGarbageCollector.SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP);
+
suggestedIntervalMs = settings.get(VersionGarbageCollector.SETTINGS_COLLECTION_REC_INTERVAL_PROP);
if (suggestedIntervalMs > 0) {
suggestedIntervalMs = Math.max(suggestedIntervalMs, options.precisionMs);
@@ -217,6 +220,7 @@ public class VersionGCRecommendations {
// default values
settings.put(VersionGarbageCollector.SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP, 0L);
settings.put(VersionGarbageCollector.SETTINGS_COLLECTION_REC_INTERVAL_PROP, 0L);
+ settings.put(VersionGarbageCollector.SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP, -1L);
if (versionGCDoc != null) {
for (String k : versionGCDoc.keySet()) {
Object value = versionGCDoc.get(k);
@@ -228,7 +232,7 @@ public class VersionGCRecommendations {
return settings;
}
- private void setLongSetting(String propName, long val) {
+ void setLongSetting(String propName, long val) {
UpdateOp updateOp = new UpdateOp(VersionGarbageCollector.SETTINGS_COLLECTION_ID, true);
updateOp.set(propName, val);
vgc.getDocumentStore().createOrUpdate(Collection.SETTINGS, updateOp);
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupport.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupport.java
index 13171b7fd5..0e5c26c83d 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupport.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupport.java
@@ -83,6 +83,31 @@ public class VersionGCSupport {
});
}
+ /**
+ * TODO: document me!
+ */
+ public Iterable<NodeDocument> getModifiedDocs(final long fromModified, final long toModified) {
+ return filter(getSelectedDocuments(store, NodeDocument.MODIFIED_IN_SECS, fromModified), new Predicate<NodeDocument>() {
+ @Override
+ public boolean apply(NodeDocument input) {
+ return modifiedGreaterThanEquals(input, fromModified)
+ && modifiedLessThan(input, toModified);
+ }
+
+ private boolean modifiedGreaterThanEquals(NodeDocument doc,
+ long time) {
+ Long modified = doc.getModified();
+ return modified != null && modified.compareTo(getModifiedInSecs(time)) >= 0;
+ }
+
+ private boolean modifiedLessThan(NodeDocument doc,
+ long time) {
+ Long modified = doc.getModified();
+ return modified != null && modified.compareTo(getModifiedInSecs(time)) < 0;
+ }
+ });
+ }
+
/**
* Returns the underlying document store.
*
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
index ed0b333d44..e7442a7d15 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
@@ -70,6 +70,9 @@ import static org.slf4j.helpers.MessageFormatter.arrayFormat;
public class VersionGarbageCollector {
+ /** TODO temporary global flag to enable 'detail gc' during prototyping. Should eventually become eg a system property */
+ public static boolean DETAIL_GC_ENABLED = false;
+
//Kept less than MongoDocumentStore.IN_CLAUSE_BATCH_SIZE to avoid re-partitioning
private static final int DELETE_BATCH_SIZE = 450;
private static final int UPDATE_BATCH_SIZE = 450;
@@ -99,6 +102,17 @@ public class VersionGarbageCollector {
*/
static final String SETTINGS_COLLECTION_REC_INTERVAL_PROP = "recommendedIntervalMs";
+ /**
+ * Property name to timestamp when last full-detail-GC run happened, or -1 if not applicable/in-use.
+ * <p>
+ * <ul>
+ * <li>-1 : full repo scan is disabled</li>
+ * <li>0 : full repo scan is enabled and bound to start from zero == oldest _modified </li>
+ * <li>gt 0 : full repo scan is enabled, was already done up until this value</li>
+ * </ul>
+ */
+ static final String SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP = "fullDetailGCTimeStamp";
+
private final DocumentNodeStore nodeStore;
private final DocumentStore ds;
private final VersionGCSupport versionStore;
@@ -260,13 +274,14 @@ public class VersionGarbageCollector {
final Stopwatch active = Stopwatch.createUnstarted();
final Stopwatch collectDeletedDocs = Stopwatch.createUnstarted();
final Stopwatch checkDeletedDocs = Stopwatch.createUnstarted();
+ final Stopwatch detailGcDocs = Stopwatch.createUnstarted();
final Stopwatch deleteDeletedDocs = Stopwatch.createUnstarted();
final Stopwatch collectAndDeleteSplitDocs = Stopwatch.createUnstarted();
final Stopwatch deleteSplitDocs = Stopwatch.createUnstarted();
final Stopwatch sortDocIds = Stopwatch.createUnstarted();
final Stopwatch updateResurrectedDocuments = Stopwatch.createUnstarted();
long activeElapsed, collectDeletedDocsElapsed, checkDeletedDocsElapsed, deleteDeletedDocsElapsed, collectAndDeleteSplitDocsElapsed,
- deleteSplitDocsElapsed, sortDocIdsElapsed, updateResurrectedDocumentsElapsed;
+ deleteSplitDocsElapsed, sortDocIdsElapsed, updateResurrectedDocumentsElapsed, detailGcDocsElapsed;
@Override
public String toString() {
@@ -335,6 +350,7 @@ public class VersionGarbageCollector {
this.deleteSplitDocsElapsed += run.deleteSplitDocsElapsed;
this.sortDocIdsElapsed += run.sortDocIdsElapsed;
this.updateResurrectedDocumentsElapsed += run.updateResurrectedDocumentsElapsed;
+ this.detailGcDocsElapsed += run.detailGcDocsElapsed;
} else {
// single run -> read from stop watches
this.activeElapsed += run.active.elapsed(MICROSECONDS);
@@ -345,6 +361,7 @@ public class VersionGarbageCollector {
this.deleteSplitDocsElapsed += run.deleteSplitDocs.elapsed(MICROSECONDS);
this.sortDocIdsElapsed += run.sortDocIds.elapsed(MICROSECONDS);
this.updateResurrectedDocumentsElapsed += run.updateResurrectedDocuments.elapsed(MICROSECONDS);
+ this.detailGcDocsElapsed += run.detailGcDocs.elapsed(MICROSECONDS);
}
}
}
@@ -353,6 +370,7 @@ public class VersionGarbageCollector {
NONE,
COLLECTING,
CHECKING,
+ DETAILGC,
DELETING,
SORTING,
SPLITS_CLEANUP,
@@ -380,6 +398,7 @@ public class VersionGarbageCollector {
this.watches.put(GCPhase.NONE, Stopwatch.createStarted());
this.watches.put(GCPhase.COLLECTING, stats.collectDeletedDocs);
this.watches.put(GCPhase.CHECKING, stats.checkDeletedDocs);
+ this.watches.put(GCPhase.DETAILGC, stats.detailGcDocs);
this.watches.put(GCPhase.DELETING, stats.deleteDeletedDocs);
this.watches.put(GCPhase.SORTING, stats.sortDocIds);
this.watches.put(GCPhase.SPLITS_CLEANUP, stats.collectAndDeleteSplitDocs);
@@ -506,6 +525,7 @@ public class VersionGarbageCollector {
collectDeletedDocuments(phases, headRevision, rec);
collectSplitDocuments(phases, sweepRevisions, rec);
+ collectDetailGarbage(phases, headRevision, rec);
}
} catch (LimitExceededException ex) {
stats.limitExceeded = true;
@@ -521,6 +541,112 @@ public class VersionGarbageCollector {
return stats;
}
+ /**
+ * "Detail garbage" refers to additional garbage identified as part of OAK-10199
+ * et al: essentially garbage that in earlier versions of Oak were ignored. This
+ * includes: deleted properties, revision information within documents, branch
+ * commit related garbage.
+ * <p/>
+ * TODO: limit this to run only on a singleton instance, eg the cluster leader
+ * <p/>
+ * The "detail garbage" collector can be instructed to do a full repository scan
+ * - or incrementally based on where it last left off. When doing a full
+ * repository scan (but not limited to that), it executes in (small) batches
+ * followed by voluntary paused (aka throttling) to avoid excessive load on the
+ * system. The full repository scan does not have to finish particularly fast,
+ * it is okay that it takes a considerable amount of time.
+ *
+ * @param headRevision
+ * @throws IOException
+ * @throws LimitExceededException
+ */
+ private void collectDetailGarbage(GCPhases phases, RevisionVector headRevision, VersionGCRecommendations rec)
+ throws IOException, LimitExceededException {
+ if (!DETAIL_GC_ENABLED) {
+ // TODO: this toggling should be done nicer asap
+ return;
+ }
+ int docsTraversed = 0;
+ DetailGC gc = new DetailGC(headRevision, monitor);
+ try {
+ final long fromModified;
+ final long toModified;
+ if (rec.fullDetailGCTimestamp == -1) {
+ // then full detail-gc is disabled or over - use regular scope then
+ fromModified = rec.scope.fromMs;
+ toModified = rec.scope.toMs;
+ } else {
+ // then full detail-gc is enabled - use it then
+ fromModified = rec.fullDetailGCTimestamp; // TODO: once we're passed rec.scope.fromMs we should
+ // disable fullgc
+ toModified = rec.scope.toMs; // the 'to' here is the max. it will process only eg 1 batch
+ }
+ long oldestGced = fromModified;
+ boolean foundAnything = false;
+ if (phases.start(GCPhase.COLLECTING)) {
+ Iterable<NodeDocument> itr = versionStore.getModifiedDocs(fromModified, toModified);
+ final Stopwatch timer = Stopwatch.createUnstarted();
+ timer.reset().start();
+ try {
+ for (NodeDocument doc : itr) {
+ // continue with GC?
+ if (cancel.get()) {
+ break;
+ }
+ foundAnything = true;
+ if (phases.start(GCPhase.DETAILGC)) {
+ gc.detailGC(doc, phases);
+ phases.stop(GCPhase.DETAILGC);
+ }
+ final Long modified = doc.getModified();
+ if (modified == null) {
+ monitor.warn("collectDetailGarbage : document has no _modified property : {}",
+ doc.getId());
+ } else if (modified < oldestGced) {
+ monitor.warn(
+ "collectDetailGarbage : document has older _modified than query boundary : {} (from: {}, to: {})",
+ modified, fromModified, toModified);
+ } else {
+ oldestGced = modified;
+ }
+ docsTraversed++;
+ if (docsTraversed % PROGRESS_BATCH_SIZE == 0) {
+ monitor.info("Iterated through {} documents so far. {} had detail garbage",
+ docsTraversed, gc.getNumDocuments());
+ }
+ if (rec.maxCollect > 0 && gc.getNumDocuments() > rec.maxCollect) {
+ // TODO: how would we recover from this?
+ throw new LimitExceededException();
+ }
+ }
+ } finally {
+ Utils.closeIfCloseable(itr);
+ delayOnModifications(timer.stop().elapsed(TimeUnit.MILLISECONDS));
+ }
+ phases.stop(GCPhase.COLLECTING);
+ if (!cancel.get() && foundAnything) {
+ // TODO: move to evaluate()
+ rec.setLongSetting(SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP, oldestGced + 1);
+ }
+ }
+ } finally {
+ gc.close();
+ }
+ }
+
+ private void delayOnModifications(long durationMs) {
+ long delayMs = Math.round(durationMs * options.delayFactor);
+ if (!cancel.get() && delayMs > 0) {
+ try {
+ Clock clock = nodeStore.getClock();
+ clock.waitUntil(clock.getTime() + delayMs);
+ }
+ catch (InterruptedException ex) {
+ /* ignore */
+ }
+ }
+ }
+
private void collectSplitDocuments(GCPhases phases,
RevisionVector sweepRevisions,
VersionGCRecommendations rec) {
@@ -611,6 +737,61 @@ public class VersionGarbageCollector {
}
}
+ private class DetailGC implements Closeable {
+
+ private final RevisionVector headRevision;
+ private final GCMonitor monitor;
+ private int count;
+
+ public DetailGC(@NotNull RevisionVector headRevision, @NotNull GCMonitor monitor) {
+ this.headRevision = checkNotNull(headRevision);
+ this.monitor = monitor;
+ }
+
+ public void detailGC(NodeDocument doc, GCPhases phases) {
+ deleteSample(doc, phases);
+ deleteUnmergedBranchCommitDocument(doc, phases);
+ deleteDeletedProperties(doc, phases);
+ deleteOldRevisions(doc, phases);
+ }
+
+ /** TODO remove, this is just a skeleton sample */
+ private void deleteSample(NodeDocument doc, GCPhases phases) {
+ if (doc.getId().contains("should_delete")) {
+ if (phases.start(GCPhase.DELETING)) {
+ monitor.info("deleteSample: should do the deletion now, but this is demo only. I'm still learning");
+ System.out.println("do the actual deletion");
+ count++;
+ phases.stop(GCPhase.DELETING);
+ }
+ }
+ }
+
+ private void deleteUnmergedBranchCommitDocument(NodeDocument doc, GCPhases phases) {
+ // TODO Auto-generated method stub
+
+ }
+
+ private void deleteDeletedProperties(NodeDocument doc, GCPhases phases) {
+ // TODO Auto-generated method stub
+
+ }
+
+ private void deleteOldRevisions(NodeDocument doc, GCPhases phases) {
+ // TODO Auto-generated method stub
+
+ }
+
+ long getNumDocuments() {
+ return count;
+ }
+
+ @Override
+ public void close() throws IOException {
+
+ }
+ }
+
/**
* A helper class to remove document for deleted nodes.
*/
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DetailGCHelper.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DetailGCHelper.java
new file mode 100644
index 0000000000..8a585c7dc0
--- /dev/null
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DetailGCHelper.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+public class DetailGCHelper {
+
+ public static void setLongSetting(String propName, long val, DocumentNodeStore ns) {
+ UpdateOp updateOp = new UpdateOp(VersionGarbageCollector.SETTINGS_COLLECTION_ID, true);
+ updateOp.set(propName, val);
+ ns.getDocumentStore().createOrUpdate(Collection.SETTINGS, updateOp);
+ }
+
+ public static void enableDetailGC(DocumentNodeStore ns) {
+ VersionGarbageCollector.DETAIL_GC_ENABLED = true;
+ if (ns != null) {
+ setLongSetting(VersionGarbageCollector.SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP, 0, ns);
+ }
+ }
+
+ public static void disableDetailGC(DocumentNodeStore ns) {
+ VersionGarbageCollector.DETAIL_GC_ENABLED = false;
+ if (ns != null) {
+ setLongSetting(VersionGarbageCollector.SETTINGS_COLLECTION_FULL_DETAILGC_TIMESTAMP_PROP, -1, ns);
+ }
+ }
+}
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
index a3e7a5e1e3..1bd81ce89c 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
@@ -51,6 +51,7 @@ import static java.util.concurrent.TimeUnit.HOURS;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -338,6 +339,23 @@ public class VersionGCTest {
}
}
+ // OAK-10199
+ @Test
+ public void testDetailGcDocumentRead_disabled() throws Exception {
+ DetailGCHelper.disableDetailGC(ns);
+ VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
+ assertNotNull(stats);
+ assertEquals(0, stats.detailGcDocsElapsed);
+ }
+
+ @Test
+ public void testDetailGcDocumentRead_enabled() throws Exception {
+ DetailGCHelper.enableDetailGC(ns);
+ VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
+ assertNotNull(stats);
+ assertNotEquals(0, stats.detailGcDocsElapsed);
+ }
+
private Future<VersionGCStats> gc() {
// run gc in a separate thread
return execService.submit(new Callable<VersionGCStats>() {