You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by jo...@apache.org on 2024/05/28 14:30:02 UTC
(jackrabbit-oak) branch trunk updated: OAK-10739 : Provide support for FullGC in Mongo DocumentStore (#1454)
This is an automated email from the ASF dual-hosted git repository.
joscorbe pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new e5cffd353f OAK-10739 : Provide support for FullGC in Mongo DocumentStore (#1454)
e5cffd353f is described below
commit e5cffd353f83a02c0d2f51772a7e512516674893
Author: Rishabh Kumar <ri...@gmail.com>
AuthorDate: Tue May 28 19:59:57 2024 +0530
OAK-10739 : Provide support for FullGC in Mongo DocumentStore (#1454)
Co-authored-by: Stefan Egli <st...@apache.org>
Co-authored-by: Rishabh Kumar <di...@adobe.com>
Co-authored-by: Jose Cordero <co...@adobe.com>
Co-authored-by: Daniel Iancu <ia...@adobe.com>
Full details in branch: DetailedGC/OAK-10199
---
.../plugins/document/DocumentNodeStoreHelper.java | 7 +-
.../jackrabbit/oak/run/RevisionsCommand.java | 211 +-
.../java/org/apache/jackrabbit/oak/run/Utils.java | 40 +-
.../oak/plugins/document/RevisionsCommandTest.java | 392 +++
.../jackrabbit/oak/run/RevisionsCommandTest.java | 199 --
oak-store-document/pom.xml | 6 +
.../oak/plugins/document/Configuration.java | 19 +
.../oak/plugins/document/DocumentNodeStore.java | 4 +-
.../plugins/document/DocumentNodeStoreBuilder.java | 41 +
.../plugins/document/DocumentNodeStoreService.java | 31 +
.../oak/plugins/document/FullGCStatsCollector.java | 75 +
.../plugins/document/FullGCStatsCollectorImpl.java | 166 ++
.../oak/plugins/document/NodeDocument.java | 89 +-
.../document/NodeDocumentRevisionCleaner.java | 320 +++
.../plugins/document/VersionGCRecommendations.java | 234 +-
.../oak/plugins/document/VersionGCSupport.java | 157 +-
.../plugins/document/VersionGarbageCollector.java | 1535 ++++++++++-
.../document/mongo/MongoVersionGCSupport.java | 101 +
.../document/rdb/RDBDocumentNodeStoreBuilder.java | 47 +
.../oak/plugins/document/util/Utils.java | 42 +-
.../oak/plugins/document/BranchCommitGCTest.java | 742 +++++-
.../DocumentNodeStoreServiceConfigurationTest.java | 20 +
.../oak/plugins/document/FailingDocumentStore.java | 14 +
.../oak/plugins/document/FullGCHelper.java | 151 ++
.../document/FullGCStatsCollectorImplTest.java | 183 ++
.../document/NodeDocumentRevisionCleanerTest.java | 576 ++++
.../oak/plugins/document/NodeDocumentTest.java | 32 +
.../plugins/document/PausableDocumentStore.java | 4 +
.../oak/plugins/document/VersionGCInitTest.java | 114 +-
.../oak/plugins/document/VersionGCQueryTest.java | 4 +-
.../oak/plugins/document/VersionGCStatsTest.java | 40 +-
.../oak/plugins/document/VersionGCSupportTest.java | 265 +-
.../oak/plugins/document/VersionGCTest.java | 205 +-
.../document/VersionGarbageCollectorIT.java | 2790 +++++++++++++++++++-
.../document/VersionGarbageCollectorLogTest.java | 22 +
.../mongo/MongoDocumentNodeStoreBuilderTest.java | 24 +
.../rdb/RDBDocumentNodeStoreBuilderTest.java | 37 +
.../oak/plugins/document/util/UtilsTest.java | 104 +-
38 files changed, 8465 insertions(+), 578 deletions(-)
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
index cf839c88c5..4a58822ebf 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
@@ -71,9 +71,10 @@ public class DocumentNodeStoreHelper {
System.out.println("Collected in " + sw.stop());
}
- public static VersionGarbageCollector createVersionGC(
- DocumentNodeStore nodeStore, VersionGCSupport gcSupport) {
- return new VersionGarbageCollector(nodeStore, gcSupport);
+ public static VersionGarbageCollector createVersionGC(DocumentNodeStore nodeStore, VersionGCSupport gcSupport,
+ boolean fullGCEnabled, boolean isFullGCDryRun,
+ boolean embeddedVerification) {
+ return new VersionGarbageCollector(nodeStore, gcSupport, fullGCEnabled, isFullGCDryRun, embeddedVerification);
}
private static Iterable<BlobReferences> scan(DocumentNodeStore store,
diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
index e418148952..8a9688999a 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
@@ -17,12 +17,12 @@
package org.apache.jackrabbit.oak.run;
import org.apache.jackrabbit.guava.common.base.Joiner;
-import org.apache.jackrabbit.guava.common.collect.ImmutableList;
import org.apache.jackrabbit.guava.common.io.Closer;
import java.io.IOException;
import java.util.List;
import java.util.Locale;
+import java.util.Optional;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -44,25 +44,39 @@ import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder;
import org.apache.jackrabbit.oak.plugins.document.DocumentStore;
import org.apache.jackrabbit.oak.plugins.document.FormatVersion;
import org.apache.jackrabbit.oak.plugins.document.MissingLastRevSeeker;
+import org.apache.jackrabbit.oak.plugins.document.NodeDocument;
import org.apache.jackrabbit.oak.plugins.document.RevisionContextWrapper;
import org.apache.jackrabbit.oak.plugins.document.SweepHelper;
import org.apache.jackrabbit.oak.plugins.document.VersionGCSupport;
import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCInfo;
import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCStats;
+import org.apache.jackrabbit.oak.plugins.document.util.MongoConnection;
import org.apache.jackrabbit.oak.run.commons.Command;
import org.apache.jackrabbit.oak.plugins.document.VersionGCOptions;
import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector;
import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
+import org.apache.jackrabbit.oak.spi.gc.LoggingGCMonitor;
+import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static java.lang.Boolean.FALSE;
+import static java.lang.Boolean.TRUE;
+import static java.lang.System.currentTimeMillis;
+import static java.util.List.of;
import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.commons.lang3.StringUtils.isNotEmpty;
+import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreHelper.createVersionGC;
import static org.apache.jackrabbit.oak.plugins.document.FormatVersion.versionOf;
+import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getIdFromPath;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getRootDocument;
+import static org.apache.jackrabbit.oak.plugins.document.util.Utils.isFullGCEnabled;
+import static org.apache.jackrabbit.oak.plugins.document.util.Utils.isEmbeddedVerificationEnabled;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.timestampToString;
import static org.apache.jackrabbit.oak.run.Utils.asCloseable;
import static org.apache.jackrabbit.oak.run.Utils.createDocumentMKBuilder;
+import static org.apache.jackrabbit.oak.run.Utils.getMongoConnection;
/**
* Gives information about current node revisions state.
@@ -74,16 +88,22 @@ public class RevisionsCommand implements Command {
private static final String USAGE = Joiner.on(System.lineSeparator()).join(
"revisions {<jdbc-uri> | <mongodb-uri>} <sub-command> [options]",
"where sub-command is one of",
- " info give information about the revisions state without performing",
- " any modifications",
- " collect perform garbage collection",
- " reset clear all persisted metadata",
- " sweep clean up uncommitted changes"
+ " info give information about the revisions state without performing",
+ " any modifications",
+ " collect perform garbage collection",
+ " reset clear all persisted metadata",
+ " sweep clean up uncommitted changes",
+ " fullGC perform full garbage collection i.e. remove unmerged branch commits, old ",
+ " revisions, deleted properties etc. Use the --entireRepo to run it on the entire",
+ " repository, or --path argument to perform the fullGC only on the specific document"
);
- private static final ImmutableList<String> LOGGER_NAMES = ImmutableList.of(
+ private static final List<String> LOGGER_NAMES = of(
"org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector",
- "org.apache.jackrabbit.oak.plugins.document.NodeDocumentSweeper"
+ "org.apache.jackrabbit.oak.plugins.document.NodeDocumentSweeper",
+ "org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.auditDGC",
+ "org.apache.jackrabbit.oak.plugins.document.mongo.MongoVersionGCSupport",
+ "org.apache.jackrabbit.oak.plugins.document.VersionGCRecommendations"
);
private static class RevisionsOptions extends Utils.NodeStoreOptions {
@@ -92,6 +112,7 @@ public class RevisionsCommand implements Command {
static final String CMD_COLLECT = "collect";
static final String CMD_RESET = "reset";
static final String CMD_SWEEP = "sweep";
+ static final String CMD_FULL_GC = "fullGC";
final OptionSpec<?> once;
final OptionSpec<Integer> limit;
@@ -99,7 +120,14 @@ public class RevisionsCommand implements Command {
final OptionSpec<Long> olderThan;
final OptionSpec<Double> delay;
final OptionSpec<?> continuous;
+ final OptionSpec<?> fullGCOnly;
+ final OptionSpec<Boolean> resetFullGC;
final OptionSpec<?> verbose;
+ final OptionSpec<String> path;
+ final OptionSpec<?> entireRepo;
+ final OptionSpec<?> compact;
+ final OptionSpec<Boolean> dryRun;
+ final OptionSpec<Boolean> embeddedVerification;
RevisionsOptions(String usage) {
super(usage);
@@ -116,10 +144,27 @@ public class RevisionsCommand implements Command {
timeLimit = parser
.accepts("timeLimit", "cancel garbage collection after n seconds").withRequiredArg()
.ofType(Long.class).defaultsTo(-1L);
+ dryRun = parser.accepts("dryRun", "dryRun of fullGC i.e. only print what would be deleted")
+ .withRequiredArg().ofType(Boolean.class).defaultsTo(TRUE);
+ embeddedVerification = parser.accepts("verify", "enable embedded verification of fullGC " +
+ "during dryRun mode i.e. will verify the effect of fullGC operation on each document after " +
+ "applying the changes in memory and will raise flag if it can cause issues")
+ .withRequiredArg().ofType(Boolean.class).defaultsTo(TRUE);
continuous = parser
.accepts("continuous", "run continuously (collect only)");
+ fullGCOnly = parser
+ .accepts("fullGCOnly", "apply only to fullGC (reset only)");
verbose = parser
.accepts("verbose", "print INFO messages to the console");
+ path = parser
+ .accepts("path", "path to the document to be cleaned up").withRequiredArg();
+ entireRepo = parser
+ .accepts("entireRepo", "run fullGC on the entire repository");
+ compact = parser
+ .accepts("compact", "run compaction on document store (only mongo) after running fullGC");
+ resetFullGC = parser
+ .accepts("resetFullGC", "reset fullGC after running FullGC")
+ .withRequiredArg().ofType(Boolean.class).defaultsTo(FALSE);
}
public RevisionsOptions parse(String[] args) {
@@ -143,6 +188,14 @@ public class RevisionsCommand implements Command {
return limit.value(options);
}
+ boolean isDryRun() {
+ return dryRun.value(options);
+ }
+
+ boolean isEmbeddedVerificationEnabled() {
+ return embeddedVerification.value(options);
+ }
+
long getOlderThan() {
return olderThan.value(options);
}
@@ -159,9 +212,29 @@ public class RevisionsCommand implements Command {
return options.has(continuous);
}
+ String getPath() {
+ return path.value(options);
+ }
+
+ boolean isResetFullGCOnly() {
+ return options.has(fullGCOnly);
+ }
+
+ boolean resetFullGC() {
+ return resetFullGC.value(options);
+ }
+
boolean isVerbose() {
return options.has(verbose);
}
+
+ boolean isEntireRepo() {
+ return options.has(entireRepo);
+ }
+
+ boolean doCompaction() {
+ return options.has(compact);
+ }
}
@Override
@@ -175,11 +248,22 @@ public class RevisionsCommand implements Command {
if (RevisionsOptions.CMD_INFO.equals(subCmd)) {
info(options, closer);
} else if (RevisionsOptions.CMD_COLLECT.equals(subCmd)) {
- collect(options, closer);
+ collect(options, closer, false);
} else if (RevisionsOptions.CMD_RESET.equals(subCmd)) {
reset(options, closer);
} else if (RevisionsOptions.CMD_SWEEP.equals(subCmd)) {
sweep(options, closer);
+ } else if (RevisionsOptions.CMD_FULL_GC.equals(subCmd)) {
+ boolean entireRepo = options.isEntireRepo();
+ String path = options.getPath();
+ if (entireRepo) {
+ collect(options, closer, true);
+ } else if (isNotEmpty(path)) {
+ collectDocument(options, closer, path);
+ } else {
+ System.err.println("--path or --entireRepo option is required for " +
+ RevisionsOptions.CMD_FULL_GC + " command");
+ }
} else {
System.err.println("unknown revisions command: " + subCmd);
}
@@ -201,14 +285,15 @@ public class RevisionsCommand implements Command {
}
}
- private VersionGarbageCollector bootstrapVGC(RevisionsOptions options,
- Closer closer)
- throws IOException {
+ private VersionGarbageCollector bootstrapVGC(RevisionsOptions options, Closer closer, boolean fullGCEnabled) throws IOException {
DocumentNodeStoreBuilder<?> builder = createDocumentMKBuilder(options, closer);
if (builder == null) {
System.err.println("revisions mode only available for DocumentNodeStore");
System.exit(1);
}
+ // set fullGC
+ builder.setFullGCEnabled(fullGCEnabled);
+
// create a VersionGCSupport while builder is read-write
VersionGCSupport gcSupport = builder.createVersionGCSupport();
// check for matching format version
@@ -219,6 +304,9 @@ public class RevisionsCommand implements Command {
version);
System.exit(1);
}
+ if (options.isEmbeddedVerificationEnabled()) {
+ builder.setEmbeddedVerificationEnabled(true);
+ }
// set it read-only before the DocumentNodeStore is created
// this prevents the DocumentNodeStore from writing a new
// clusterId to the clusterNodes and nodes collections
@@ -226,7 +314,12 @@ public class RevisionsCommand implements Command {
useMemoryBlobStore(builder);
// create a version GC that operates on a read-only DocumentNodeStore
// and a GC support with a writable DocumentStore
- VersionGarbageCollector gc = createVersionGC(builder.build(), gcSupport);
+ System.out.println("DryRun is enabled : " + options.isDryRun());
+ System.out.println("EmbeddedVerification is enabled : " + options.isEmbeddedVerificationEnabled());
+ System.out.println("ResetFullGC is enabled : " + options.resetFullGC());
+ System.out.println("Compaction is enabled : " + options.doCompaction());
+ VersionGarbageCollector gc = createVersionGC(builder.build(), gcSupport, isFullGCEnabled(builder), options.isDryRun(),
+ isEmbeddedVerificationEnabled(builder));
VersionGCOptions gcOptions = gc.getOptions();
gcOptions = gcOptions.withDelayFactor(options.getDelay());
@@ -240,10 +333,13 @@ public class RevisionsCommand implements Command {
return gc;
}
- private void info(RevisionsOptions options, Closer closer)
- throws IOException {
- VersionGarbageCollector gc = bootstrapVGC(options, closer);
+ private void info(RevisionsOptions options, Closer closer) throws IOException {
+ VersionGarbageCollector gc = bootstrapVGC(options, closer, false);
System.out.println("retrieving gc info");
+ printInfo(gc, options);
+ }
+
+ private void printInfo(VersionGarbageCollector gc, RevisionsOptions options) throws IOException {
VersionGCInfo info = gc.getInfo(options.getOlderThan(), SECONDS);
System.out.printf(Locale.US, "%21s %s%n", "Last Successful Run:",
@@ -260,27 +356,26 @@ public class RevisionsCommand implements Command {
fmtTimestamp(info.recommendedCleanupTimestamp));
System.out.printf(Locale.US, "%21s %d%n", "Iterations Estimate:",
info.estimatedIterations);
+ System.out.printf(Locale.US, "%21s %s%n", "Oldest FullGC:",
+ fmtTimestamp(info.oldestFullGCRevisionEstimate));
}
- private void collect(final RevisionsOptions options, Closer closer)
- throws IOException {
- VersionGarbageCollector gc = bootstrapVGC(options, closer);
+ private void collect(final RevisionsOptions options, Closer closer, boolean fullGCEnabled) throws IOException {
+ VersionGarbageCollector gc = bootstrapVGC(options, closer, fullGCEnabled);
ExecutorService executor = Executors.newSingleThreadExecutor();
final Semaphore finished = new Semaphore(0);
try {
+ // collect until shutdown hook is called
+ final AtomicBoolean running = new AtomicBoolean(true);
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+ System.out.println("Detected QUIT signal.");
+ System.out.println("Stopping Revision GC...");
+ running.set(false);
+ gc.cancel();
+ finished.acquireUninterruptibly();
+ System.out.println("Stopped Revision GC.");
+ }));
if (options.isContinuous()) {
- // collect until shutdown hook is called
- final AtomicBoolean running = new AtomicBoolean(true);
- Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
- @Override
- public void run() {
- System.out.println("Detected QUIT signal.");
- System.out.println("Stopping Revision GC...");
- running.set(false);
- finished.acquireUninterruptibly();
- System.out.println("Stopped Revision GC.");
- }
- }));
while (running.get()) {
long lastRun = System.currentTimeMillis();
collectOnce(gc, options, executor);
@@ -289,8 +384,26 @@ public class RevisionsCommand implements Command {
} else {
collectOnce(gc, options, executor);
}
+ System.out.println("retrieving gc info");
+ printInfo(gc, options);
} finally {
finished.release();
+ if (options.isDryRun()) {
+ gc.resetDryRun();
+ }
+ if (options.resetFullGC()) {
+ gc.resetFullGC();
+ }
+ if (options.doCompaction()) {
+ Optional<MongoConnection> mongoClient = getMongoConnection(options, closer);
+ final long start = currentTimeMillis();
+ mongoClient.ifPresentOrElse(
+ con -> {
+ Document compact = con.getDatabase().runCommand(new Document("compact", NODES.toString()));
+ System.out.format("Compact done in %s ms, Output is %s", (currentTimeMillis() - start), compact);
+ },
+ () -> System.err.println("Database is null"));
+ }
executor.shutdownNow();
}
}
@@ -349,9 +462,13 @@ public class RevisionsCommand implements Command {
private void reset(RevisionsOptions options, Closer closer)
throws IOException {
- VersionGarbageCollector gc = bootstrapVGC(options, closer);
+ VersionGarbageCollector gc = bootstrapVGC(options, closer, false);
System.out.println("resetting recommendations and statistics");
- gc.reset();
+ if (options.isResetFullGCOnly()) {
+ gc.resetFullGC();
+ } else {
+ gc.reset();
+ }
}
private void sweep(RevisionsOptions options, Closer closer)
@@ -394,6 +511,34 @@ public class RevisionsCommand implements Command {
SweepHelper.sweep(store, new RevisionContextWrapper(ns, clusterId), seeker);
}
+ private void collectDocument(RevisionsOptions options, Closer closer, String path) throws IOException {
+ DocumentNodeStoreBuilder<?> builder = createDocumentMKBuilder(options, closer);
+ if (builder == null) {
+ System.err.println("revisions mode only available for DocumentNodeStore");
+ return;
+ }
+
+ System.out.println("running fullGC on the document: " + path);
+ DocumentStore documentStore = builder.getDocumentStore();
+ builder.setReadOnlyMode();
+ useMemoryBlobStore(builder);
+ DocumentNodeStore documentNodeStore = builder.build();
+
+ VersionGarbageCollector gc = bootstrapVGC(options, closer, true);
+ // Set a LoggingGC monitor that will output the fullGC operations
+ gc.setGCMonitor(new LoggingGCMonitor(LOG));
+ // Run fullGC on the given document
+ NodeDocument workingDocument = documentStore.find(NODES, getIdFromPath(path));
+ if (workingDocument == null) {
+ System.err.println("Document not found: " + path);
+ return;
+ }
+ gc.collectGarbageOnDocument(documentNodeStore, workingDocument, options.isVerbose());
+
+ //TODO: Probably we should output some details of fullGCStats. Could be done after OAK-10378
+ //gc.getFullGCStats();
+ }
+
private String fmtTimestamp(long ts) {
return timestampToString(ts);
}
diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
index 5cdb079f3a..f202486bc9 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
@@ -17,8 +17,13 @@
package org.apache.jackrabbit.oak.run;
+import static com.mongodb.MongoURI.MONGODB_PREFIX;
import static java.util.Arrays.asList;
+import static java.util.Objects.isNull;
+import static java.util.Optional.empty;
+import static java.util.Optional.of;
import static org.apache.jackrabbit.oak.commons.PropertiesUtil.populate;
+import static org.apache.jackrabbit.oak.plugins.document.LeaseCheckMode.DISABLED;
import static org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentNodeStoreBuilder.newMongoDocumentNodeStoreBuilder;
import static org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentNodeStoreBuilder.newRDBDocumentNodeStoreBuilder;
import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder;
@@ -33,6 +38,7 @@ import java.util.Dictionary;
import java.util.Enumeration;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Properties;
import javax.jcr.RepositoryException;
@@ -210,6 +216,15 @@ class Utils {
return createDocumentMKBuilder(new NodeStoreOptions(h).parse(args), closer);
}
+ static Optional<MongoConnection> getMongoConnection(final NodeStoreOptions options, final Closer closer) {
+ String src = options.getStoreArg();
+ if (isNull(src) || !src.startsWith(MONGODB_PREFIX)) {
+ return empty();
+ }
+
+ return of(getMongoConnection(closer, src));
+ }
+
@Nullable
static DocumentNodeStoreBuilder<?> createDocumentMKBuilder(NodeStoreOptions options,
Closer closer)
@@ -220,17 +235,9 @@ class Utils {
System.exit(1);
}
DocumentNodeStoreBuilder<?> builder;
- if (src.startsWith(MongoURI.MONGODB_PREFIX)) {
- MongoClientURI uri = new MongoClientURI(src);
- if (uri.getDatabase() == null) {
- System.err.println("Database missing in MongoDB URI: "
- + uri.getURI());
- System.exit(1);
- }
- MongoConnection mongo = new MongoConnection(uri.getURI());
- closer.register(asCloseable(mongo));
- builder = newMongoDocumentNodeStoreBuilder().setMongoDB(
- mongo.getMongoClient(), mongo.getDBName());
+ if (src.startsWith(MONGODB_PREFIX)) {
+ MongoConnection mongo = getMongoConnection(closer, src);
+ builder = newMongoDocumentNodeStoreBuilder().setMongoDB(mongo.getMongoClient(), mongo.getDBName());
} else if (src.startsWith("jdbc")) {
RDBOptions opts = new RDBOptions();
if (options.getRDBTablePrefix() != null) {
@@ -254,6 +261,17 @@ class Utils {
return builder;
}
+ private static MongoConnection getMongoConnection(Closer closer, String src) {
+ MongoClientURI uri = new MongoClientURI(src);
+ if (uri.getDatabase() == null) {
+ System.err.println("Database missing in MongoDB URI: " + uri.getURI());
+ System.exit(1);
+ }
+ MongoConnection mongo = new MongoConnection(uri.getURI());
+ closer.register(asCloseable(mongo));
+ return mongo;
+ }
+
@Nullable
public static GarbageCollectableBlobStore bootstrapDataStore(String[] args, Closer closer)
throws IOException, RepositoryException {
diff --git a/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/document/RevisionsCommandTest.java b/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/document/RevisionsCommandTest.java
new file mode 100644
index 0000000000..2a51025486
--- /dev/null
+++ b/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/document/RevisionsCommandTest.java
@@ -0,0 +1,392 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.jackrabbit.guava.common.collect.ImmutableList;
+
+import org.apache.jackrabbit.oak.plugins.document.util.MongoConnection;
+import org.apache.jackrabbit.oak.run.RevisionsCommand;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_FULL_GC_DRY_RUN_DOCUMENT_ID_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_FULL_GC_DRY_RUN_TIMESTAMP_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_ID;
+import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getIdFromPath;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
+
+public class RevisionsCommandTest {
+
+ @Rule
+ public MongoConnectionFactory connectionFactory = new MongoConnectionFactory();
+
+ @Rule
+ public DocumentMKBuilderProvider builderProvider = new DocumentMKBuilderProvider();
+
+ private DocumentNodeStore ns;
+
+ @BeforeClass
+ public static void assumeMongoDB() {
+ assumeTrue(MongoUtils.isAvailable());
+ }
+
+ @Before
+ public void before() {
+ ns = createDocumentNodeStore();
+ }
+
+ @Test
+ public void info() throws Exception {
+ ns.getVersionGarbageCollector().gc(1, TimeUnit.HOURS);
+ ns.dispose();
+
+ String output = captureSystemOut(new RevisionsCmd("info"));
+ assertTrue(output.contains("Last Successful Run"));
+ }
+
+ @Test
+ public void reset() throws Exception {
+ ns.getVersionGarbageCollector().gc(1, TimeUnit.HOURS);
+
+ Document doc = ns.getDocumentStore().find(Collection.SETTINGS, SETTINGS_COLLECTION_ID);
+ assertNotNull(doc);
+
+ ns.dispose();
+
+ String output = captureSystemOut(new RevisionsCmd("reset"));
+ assertTrue(output.contains("resetting recommendations and statistics"));
+
+ MongoConnection c = connectionFactory.getConnection();
+ assertNotNull(c);
+ ns = builderProvider.newBuilder()
+ .setMongoDB(c.getMongoClient(), c.getDBName()).getNodeStore();
+ doc = ns.getDocumentStore().find(Collection.SETTINGS, SETTINGS_COLLECTION_ID);
+ assertNull(doc);
+ }
+
+ @Test
+ public void resetFullGC() throws Exception {
+ // need to set fullGCEnabled to true, so let's bounce the default one
+ ns.dispose();
+ // and create it fresh with fullGCEnabled==true
+ ns = createDocumentNodeStore(true);
+ ns.getVersionGarbageCollector().gc(1, TimeUnit.HOURS);
+
+ Document doc = ns.getDocumentStore().find(Collection.SETTINGS, SETTINGS_COLLECTION_ID);
+ assertNotNull(doc);
+ assertNotNull(doc.get(SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP));
+ assertNotNull(doc.get(SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP));
+
+ ns.dispose();
+
+ String output = captureSystemOut(new RevisionsCmd("reset", "--fullGCOnly"));
+ assertTrue(output.contains("resetting recommendations and statistics"));
+
+ MongoConnection c = connectionFactory.getConnection();
+ assertNotNull(c);
+ ns = builderProvider.newBuilder()
+ .setMongoDB(c.getMongoClient(), c.getDBName()).getNodeStore();
+ doc = ns.getDocumentStore().find(Collection.SETTINGS, SETTINGS_COLLECTION_ID);
+ assertNotNull(doc);
+ assertNull(doc.get(SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP));
+ assertNull(doc.get(SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP));
+ }
+
+ @Test
+ public void resetFullGCWithFullGC() throws Exception {
+ // need to set fullGCEnabled to true, so let's bounce the default one
+ ns.dispose();
+ // and create it fresh with fullGCEnabled==true
+ ns = createDocumentNodeStore(true);
+ ns.getVersionGarbageCollector().gc(1, TimeUnit.HOURS);
+
+ Document doc = ns.getDocumentStore().find(Collection.SETTINGS, SETTINGS_COLLECTION_ID);
+ assertNotNull(doc);
+ assertNotNull(doc.get(SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP));
+ assertNotNull(doc.get(SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP));
+
+ ns.dispose();
+
+ String output = captureSystemOut(new RevisionsCmd("fullGC", "--resetFullGC", "true", "--entireRepo"));
+ assertTrue(output.contains("ResetFullGC is enabled : true"));
+
+ MongoConnection c = connectionFactory.getConnection();
+ assertNotNull(c);
+ ns = builderProvider.newBuilder().setMongoDB(c.getMongoClient(), c.getDBName()).getNodeStore();
+ doc = ns.getDocumentStore().find(Collection.SETTINGS, SETTINGS_COLLECTION_ID);
+ assertNotNull(doc);
+ assertNull(doc.get(SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP));
+ assertNull(doc.get(SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP));
+ }
+
+ @Test
+ public void resetDryRunFields() throws Exception {
+ // need to set fullGCEnabled to true, so let's bounce the default one
+ ns.dispose();
+ // and create it fresh with fullGCEnabled==true
+ ns = createDocumentNodeStore(true);
+ ns.getVersionGarbageCollector().gc(1, TimeUnit.HOURS);
+
+ Document doc = ns.getDocumentStore().find(Collection.SETTINGS, SETTINGS_COLLECTION_ID);
+ assertNotNull(doc);
+ assertNull(doc.get(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_TIMESTAMP_PROP));
+ assertNull(doc.get(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_DOCUMENT_ID_PROP));
+
+ ns.dispose();
+
+ String output = captureSystemOut(new RevisionsCmd("fullGC", "--entireRepo"));
+ assertTrue(output.contains("DryRun is enabled : true"));
+
+ MongoConnection c = connectionFactory.getConnection();
+ assertNotNull(c);
+ ns = builderProvider.newBuilder()
+ .setMongoDB(c.getMongoClient(), c.getDBName()).getNodeStore();
+ doc = ns.getDocumentStore().find(Collection.SETTINGS, SETTINGS_COLLECTION_ID);
+ assertNotNull(doc);
+ assertNull(doc.get(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_TIMESTAMP_PROP));
+ assertNull(doc.get(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_DOCUMENT_ID_PROP));
+ }
+
+ @Test
+ public void collect() throws Exception {
+ ns.dispose();
+
+ String output = captureSystemOut(new RevisionsCmd("collect"));
+ assertTrue(output.contains("starting gc collect"));
+ }
+
+ @Test
+ public void fullGC() {
+ ns.dispose();
+
+ String output = captureSystemOut(new RevisionsCmd("fullGC", "--entireRepo"));
+ assertTrue(output.contains("DryRun is enabled : true"));
+ assertTrue(output.contains("ResetFullGC is enabled : false"));
+ assertTrue(output.contains("Compaction is enabled : false"));
+ assertTrue(output.contains("starting gc collect"));
+ }
+
+ @Test
+ public void fullGCWithCompaction() {
+ ns.dispose();
+
+ String output = captureSystemOut(new RevisionsCmd("fullGC", "--entireRepo", "--compact"));
+ assertTrue(output.contains("DryRun is enabled : true"));
+ assertTrue(output.contains("ResetFullGC is enabled : false"));
+ assertTrue(output.contains("Compaction is enabled : true"));
+ assertTrue(output.contains("starting gc collect"));
+ }
+
+ @Test
+ public void fullGCWithoutDryRun() {
+ ns.dispose();
+
+ String output = captureSystemOut(new RevisionsCmd("fullGC", "--dryRun", "false", "--entireRepo"));
+ assertTrue(output.contains("DryRun is enabled : false"));
+ assertTrue(output.contains("starting gc collect"));
+ }
+
+ @Test
+ public void fullGCWithDryRun() {
+ ns.dispose();
+
+ String output = captureSystemOut(new RevisionsCmd("fullGC", "--dryRun", "true", "--entireRepo"));
+ assertTrue(output.contains("DryRun is enabled : true"));
+ assertTrue(output.contains("starting gc collect"));
+ }
+
+ @Test
+ public void fullGCWithoutResetFullGC() {
+ ns.dispose();
+
+ String output = captureSystemOut(new RevisionsCmd("fullGC", "--resetFullGC", "false", "--entireRepo"));
+ assertTrue(output.contains("ResetFullGC is enabled : false"));
+ assertTrue(output.contains("starting gc collect"));
+ }
+
+ @Test
+ public void fullGCWithResetFullGC() {
+ ns.dispose();
+
+ String output = captureSystemOut(new RevisionsCmd("fullGC", "--resetFullGC", "true", "--entireRepo"));
+ assertTrue(output.contains("ResetFullGC is enabled : true"));
+ assertTrue(output.contains("starting gc collect"));
+ }
+
+ @Test
+ public void embeddedVerification() {
+ ns.dispose();
+
+ String output = captureSystemOut(new RevisionsCmd("fullGC", "--entireRepo"));
+ assertTrue(output.contains("EmbeddedVerification is enabled : true"));
+ assertTrue(output.contains("starting gc collect"));
+ }
+
+ @Test
+ public void fullGCWithoutEmbeddedVerification() {
+ ns.dispose();
+
+ String output = captureSystemOut(new RevisionsCmd("fullGC", "--verify", "false", "--entireRepo"));
+ assertTrue(output.contains("EmbeddedVerification is enabled : false"));
+ assertTrue(output.contains("starting gc collect"));
+ }
+
+ @Test
+ public void fullGCWithEmbeddedVerification() {
+ ns.dispose();
+
+ String output = captureSystemOut(new RevisionsCmd("fullGC", "--verify", "true", "--entireRepo"));
+ assertTrue(output.contains("EmbeddedVerification is enabled : true"));
+ assertTrue(output.contains("starting gc collect"));
+ }
+
+ @Test
+ public void fullGCWithEmbeddedWithoutEntireRepoOrPath() {
+ ns.dispose();
+
+ String errOutput = captureSystemErr(new RevisionsCmd("fullGC", "--verify", "true"));
+ assertTrue(errOutput.contains("--path or --entireRepo option is required for fullGC command"));
+ }
+
+ @Test
+ public void fullGCWithEmbeddedWithPath() {
+ ns.dispose();
+
+ String output = captureSystemOut(new RevisionsCmd("fullGC", "--verify", "true", "--path", "/"));
+ assertTrue(output.contains("EmbeddedVerification is enabled : true"));
+ assertTrue(output.contains("running fullGC on the document: /"));
+ }
+
+ @Test
+ public void fullGCWithEmbeddedWithNonExistingPath() {
+ ns.dispose();
+
+ String errOutput = captureSystemErr(new RevisionsCmd("fullGC", "--verify", "true", "--path", "/non-existing-path"));
+ assertTrue(errOutput.contains("Document not found: /non-existing-path"));
+ }
+
+ @Test
+ public void sweep() {
+ int clusterId = ns.getClusterId();
+ String output = captureSystemErr(new Sweep(clusterId));
+ assertThat(output, containsString("cannot sweep revisions for active clusterId"));
+
+ output = captureSystemErr(new Sweep(0));
+ assertThat(output, containsString("clusterId option is required"));
+
+ output = captureSystemErr(new Sweep(99));
+ assertThat(output, containsString("store does not have changes with clusterId"));
+
+ ns.dispose();
+ output = captureSystemOut(new Sweep(clusterId));
+ assertThat(output, containsString("Revision sweep not needed for clusterId"));
+
+ // remove the sweep revision to force a sweep run
+ MongoConnection c = connectionFactory.getConnection();
+ assertNotNull(c);
+ DocumentNodeStoreBuilder<?> builder = builderProvider.newBuilder()
+ .setMongoDB(c.getMongoClient(), c.getDBName());
+ DocumentStore store = builder.getDocumentStore();
+ UpdateOp op = new UpdateOp(getIdFromPath("/"), false);
+ op.removeMapEntry("_sweepRev", new Revision(0, 0, clusterId));
+ assertNotNull(store.findAndUpdate(Collection.NODES, op));
+
+ output = captureSystemOut(new Sweep(clusterId));
+ assertThat(output, containsString("Updated sweep revision to"));
+ }
+
+ private DocumentNodeStore createDocumentNodeStore() {
+ return createDocumentNodeStore(false);
+ }
+
+ private DocumentNodeStore createDocumentNodeStore(boolean fullGCEnabled) {
+ MongoConnection c = connectionFactory.getConnection();
+ assertNotNull(c);
+ MongoUtils.dropCollections(c.getDatabase());
+ return builderProvider.newBuilder().setFullGCEnabled(fullGCEnabled)
+ .setMongoDB(c.getMongoClient(), c.getDBName()).getNodeStore();
+ }
+
+ private String captureSystemOut(Runnable r) {
+ PrintStream old = System.out;
+ try {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(baos);
+ System.setOut(ps);
+ r.run();
+ System.out.flush();
+ return baos.toString();
+ } finally {
+ System.setOut(old);
+ }
+ }
+
+ private String captureSystemErr(Runnable r) {
+ PrintStream old = System.err;
+ try {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(baos);
+ System.setErr(ps);
+ r.run();
+ System.err.flush();
+ return baos.toString();
+ } finally {
+ System.setErr(old);
+ }
+ }
+
+ private static class RevisionsCmd implements Runnable {
+
+ private final ImmutableList<String> args;
+
+ public RevisionsCmd(String... args) {
+ this.args = ImmutableList.<String>builder().add(MongoUtils.URL)
+ .add(args).build();
+ }
+
+ @Override
+ public void run() {
+ try {
+ new RevisionsCommand().execute(args.toArray(new String[0]));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private static final class Sweep extends RevisionsCmd {
+
+ Sweep(int clusterId) {
+ super("sweep", "--clusterId", String.valueOf(clusterId));
+ }
+ }
+}
diff --git a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/RevisionsCommandTest.java b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/RevisionsCommandTest.java
deleted file mode 100644
index ebd09b7a5e..0000000000
--- a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/RevisionsCommandTest.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.jackrabbit.oak.run;
-
-import java.io.ByteArrayOutputStream;
-import java.io.PrintStream;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.jackrabbit.guava.common.collect.ImmutableList;
-
-import org.apache.jackrabbit.oak.plugins.document.Collection;
-import org.apache.jackrabbit.oak.plugins.document.Document;
-import org.apache.jackrabbit.oak.plugins.document.DocumentMKBuilderProvider;
-import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
-import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder;
-import org.apache.jackrabbit.oak.plugins.document.DocumentStore;
-import org.apache.jackrabbit.oak.plugins.document.MongoConnectionFactory;
-import org.apache.jackrabbit.oak.plugins.document.MongoUtils;
-import org.apache.jackrabbit.oak.plugins.document.Revision;
-import org.apache.jackrabbit.oak.plugins.document.UpdateOp;
-import org.apache.jackrabbit.oak.plugins.document.util.MongoConnection;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-
-import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getIdFromPath;
-import static org.hamcrest.CoreMatchers.containsString;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.assumeTrue;
-
-public class RevisionsCommandTest {
-
- @Rule
- public MongoConnectionFactory connectionFactory = new MongoConnectionFactory();
-
- @Rule
- public DocumentMKBuilderProvider builderProvider = new DocumentMKBuilderProvider();
-
- private DocumentNodeStore ns;
-
- @BeforeClass
- public static void assumeMongoDB() {
- assumeTrue(MongoUtils.isAvailable());
- }
-
- @Before
- public void before() {
- ns = createDocumentNodeStore();
- }
-
- @Test
- public void info() throws Exception {
- ns.getVersionGarbageCollector().gc(1, TimeUnit.HOURS);
- ns.dispose();
-
- String output = captureSystemOut(new RevisionsCmd("info"));
- assertTrue(output.contains("Last Successful Run"));
- }
-
- @Test
- public void reset() throws Exception {
- ns.getVersionGarbageCollector().gc(1, TimeUnit.HOURS);
-
- Document doc = ns.getDocumentStore().find(Collection.SETTINGS, "versionGC");
- assertNotNull(doc);
-
- ns.dispose();
-
- String output = captureSystemOut(new RevisionsCmd("reset"));
- assertTrue(output.contains("resetting recommendations and statistics"));
-
- MongoConnection c = connectionFactory.getConnection();
- assertNotNull(c);
- ns = builderProvider.newBuilder()
- .setMongoDB(c.getMongoClient(), c.getDBName()).getNodeStore();
- doc = ns.getDocumentStore().find(Collection.SETTINGS, "versionGC");
- assertNull(doc);
- }
-
- @Test
- public void collect() throws Exception {
- ns.dispose();
-
- String output = captureSystemOut(new RevisionsCmd("collect"));
- assertTrue(output.contains("starting gc collect"));
- }
-
- @Test
- public void sweep() throws Exception {
- int clusterId = ns.getClusterId();
- String output = captureSystemErr(new Sweep(clusterId));
- assertThat(output, containsString("cannot sweep revisions for active clusterId"));
-
- output = captureSystemErr(new Sweep(0));
- assertThat(output, containsString("clusterId option is required"));
-
- output = captureSystemErr(new Sweep(99));
- assertThat(output, containsString("store does not have changes with clusterId"));
-
- ns.dispose();
- output = captureSystemOut(new Sweep(clusterId));
- assertThat(output, containsString("Revision sweep not needed for clusterId"));
-
- // remove the sweep revision to force a sweep run
- MongoConnection c = connectionFactory.getConnection();
- assertNotNull(c);
- DocumentNodeStoreBuilder<?> builder = builderProvider.newBuilder()
- .setMongoDB(c.getMongoClient(), c.getDBName());
- DocumentStore store = builder.getDocumentStore();
- UpdateOp op = new UpdateOp(getIdFromPath("/"), false);
- op.removeMapEntry("_sweepRev", new Revision(0, 0, clusterId));
- assertNotNull(store.findAndUpdate(Collection.NODES, op));
-
- output = captureSystemOut(new Sweep(clusterId));
- assertThat(output, containsString("Updated sweep revision to"));
- }
-
- private DocumentNodeStore createDocumentNodeStore() {
- MongoConnection c = connectionFactory.getConnection();
- assertNotNull(c);
- MongoUtils.dropCollections(c.getDatabase());
- return builderProvider.newBuilder()
- .setMongoDB(c.getMongoClient(), c.getDBName()).getNodeStore();
- }
-
- private String captureSystemOut(Runnable r) {
- PrintStream old = System.out;
- try {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- PrintStream ps = new PrintStream(baos);
- System.setOut(ps);
- r.run();
- System.out.flush();
- return baos.toString();
- } finally {
- System.setOut(old);
- }
- }
-
- private String captureSystemErr(Runnable r) {
- PrintStream old = System.err;
- try {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- PrintStream ps = new PrintStream(baos);
- System.setErr(ps);
- r.run();
- System.err.flush();
- return baos.toString();
- } finally {
- System.setErr(old);
- }
- }
-
- private static class RevisionsCmd implements Runnable {
-
- private final ImmutableList<String> args;
-
- public RevisionsCmd(String... args) {
- this.args = ImmutableList.<String>builder().add(MongoUtils.URL)
- .add(args).build().asList();
- }
-
- @Override
- public void run() {
- try {
- new RevisionsCommand().execute(args.toArray(new String[args.size()]));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- private static final class Sweep extends RevisionsCmd {
-
- Sweep(int clusterId) {
- super("sweep", "--clusterId", String.valueOf(clusterId));
- }
- }
-}
diff --git a/oak-store-document/pom.xml b/oak-store-document/pom.xml
index 56e5309982..e0afaa7dbc 100644
--- a/oak-store-document/pom.xml
+++ b/oak-store-document/pom.xml
@@ -349,5 +349,11 @@
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Configuration.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Configuration.java
index cee47efde0..9742a29c2d 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Configuration.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Configuration.java
@@ -34,6 +34,8 @@ import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilde
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder.DEFAULT_NODE_CACHE_PERCENTAGE;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder.DEFAULT_PREV_DOC_CACHE_PERCENTAGE;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder.DEFAULT_UPDATE_LIMIT;
+import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FULL_GC_ENABLED;
+import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_EMBEDDED_VERIFICATION_ENABLED;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_THROTTLING_ENABLED;
@ObjectClassDefinition(
@@ -305,4 +307,21 @@ import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreServic
"can be reused after a recovery, 0 or negative for no delay. Default: " + DEFAULT_REUSE_DELAY_AFTER_RECOVERY_MILLIS +
" (milliseconds).")
long clusterIdReuseDelayAfterRecoveryMillis() default DEFAULT_REUSE_DELAY_AFTER_RECOVERY_MILLIS;
+
+ @AttributeDefinition(
+ name = "Document Node Store Full GC",
+ description = "Boolean value indicating whether Full GC should be enabled for " +
+ "document node store or not. The Default value is " + DEFAULT_FULL_GC_ENABLED +
+ ". Note that this value can be overridden via framework " +
+ "property 'oak.documentstore.fullGCEnabled'")
+ boolean fullGCEnabled() default DEFAULT_FULL_GC_ENABLED;
+
+ @AttributeDefinition(
+ name = "Document Node Store Embedded Verification for Full GC",
+ description = "Boolean value indicating whether Embedded Verification (i.e. verify the document after " +
+ "applying changes in memory before any database calls) for Full GC should be enabled for " +
+ "document node store or not. The Default value is " + DEFAULT_EMBEDDED_VERIFICATION_ENABLED +
+ ". Note that this value can be overridden via framework " +
+ "property 'oak.documentstore.embeddedVerificationEnabled'")
+ boolean embeddedVerificationEnabled() default DEFAULT_EMBEDDED_VERIFICATION_ENABLED;
}
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
index 481cccbe82..af26dc9552 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
@@ -38,6 +38,8 @@ import static org.apache.jackrabbit.oak.plugins.document.Path.ROOT;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.alignWithExternalRevisions;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getIdFromPath;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getModuleVersion;
+import static org.apache.jackrabbit.oak.plugins.document.util.Utils.isFullGCEnabled;
+import static org.apache.jackrabbit.oak.plugins.document.util.Utils.isEmbeddedVerificationEnabled;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.isThrottlingEnabled;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.pathToId;
import static org.apache.jackrabbit.oak.spi.observation.ChangeSet.COMMIT_CONTEXT_OBSERVATION_CHANGESET;
@@ -659,7 +661,7 @@ public final class DocumentNodeStore
this.branches = new UnmergedBranches();
this.asyncDelay = builder.getAsyncDelay();
this.versionGarbageCollector = new VersionGarbageCollector(
- this, builder.createVersionGCSupport());
+ this, builder.createVersionGCSupport(), isFullGCEnabled(builder), false, isEmbeddedVerificationEnabled(builder));
this.versionGarbageCollector.setStatisticsProvider(builder.getStatisticsProvider());
this.versionGarbageCollector.setGCMonitor(builder.getGCMonitor());
this.journalGarbageCollector = new JournalGarbageCollector(
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java
index 8c05d8b852..3f4cfcc101 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java
@@ -129,6 +129,8 @@ public class DocumentNodeStoreBuilder<T extends DocumentNodeStoreBuilder<T>> {
private Feature docStoreThrottlingFeature;
private Feature noChildOrderCleanupFeature;
private Feature cancelInvalidationFeature;
+ private Feature docStoreFullGCFeature;
+ private Feature docStoreEmbeddedVerificationFeature;
private Weigher<CacheValue, CacheValue> weigher = new EmpiricalWeigher();
private long memoryCacheSize = DEFAULT_MEMORY_CACHE_SIZE;
private int nodeCachePercentage = DEFAULT_NODE_CACHE_PERCENTAGE;
@@ -167,6 +169,8 @@ public class DocumentNodeStoreBuilder<T extends DocumentNodeStoreBuilder<T>> {
private Predicate<Path> nodeCachePredicate = Predicates.alwaysTrue();
private boolean clusterInvisible;
private boolean throttlingEnabled;
+ private boolean fullGCEnabled;
+ private boolean embeddedVerificationEnabled = DocumentNodeStoreService.DEFAULT_EMBEDDED_VERIFICATION_ENABLED;
private long suspendTimeoutMillis = DEFAULT_SUSPEND_TIMEOUT;
/**
@@ -290,6 +294,24 @@ public class DocumentNodeStoreBuilder<T extends DocumentNodeStoreBuilder<T>> {
return this.throttlingEnabled;
}
+ public T setFullGCEnabled(boolean b) {
+ this.fullGCEnabled = b;
+ return thisBuilder();
+ }
+
+ public boolean isFullGCEnabled() {
+ return this.fullGCEnabled;
+ }
+
+ public T setEmbeddedVerificationEnabled(boolean b) {
+ this.embeddedVerificationEnabled = b;
+ return thisBuilder();
+ }
+
+ public boolean isEmbeddedVerificationEnabled() {
+ return this.embeddedVerificationEnabled;
+ }
+
public T setReadOnlyMode() {
this.isReadOnlyMode = true;
return thisBuilder();
@@ -339,6 +361,25 @@ public class DocumentNodeStoreBuilder<T extends DocumentNodeStoreBuilder<T>> {
return cancelInvalidationFeature;
}
+ public T setDocStoreFullGCFeature(@Nullable Feature docStoreFullGC) {
+ this.docStoreFullGCFeature = docStoreFullGC;
+ return thisBuilder();
+ }
+
+ public Feature getDocStoreFullGCFeature() {
+ return docStoreFullGCFeature;
+ }
+
+ public T setDocStoreEmbeddedVerificationFeature(@Nullable Feature getDocStoreEmbeddedVerification) {
+ this.docStoreEmbeddedVerificationFeature = getDocStoreEmbeddedVerification;
+ return thisBuilder();
+ }
+
+ @Nullable
+ public Feature getDocStoreEmbeddedVerificationFeature() {
+ return docStoreEmbeddedVerificationFeature;
+ }
+
public T setLeaseFailureHandler(LeaseFailureHandler leaseFailureHandler) {
this.leaseFailureHandler = leaseFailureHandler;
return thisBuilder();
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
index 556c54e882..7bbbe0d42b 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
@@ -138,6 +138,8 @@ public class DocumentNodeStoreService {
static final String DEFAULT_DB = "oak";
static final boolean DEFAULT_SO_KEEP_ALIVE = true;
static final boolean DEFAULT_THROTTLING_ENABLED = false;
+ static final boolean DEFAULT_FULL_GC_ENABLED = false;
+ static final boolean DEFAULT_EMBEDDED_VERIFICATION_ENABLED = true;
static final int DEFAULT_MONGO_LEASE_SO_TIMEOUT_MILLIS = 30000;
static final String DEFAULT_PERSISTENT_CACHE = "cache";
static final String DEFAULT_JOURNAL_CACHE = "diff-cache";
@@ -187,6 +189,15 @@ public class DocumentNodeStoreService {
* Feature toggle name to enable invalidation on cancel (due to a merge collision)
*/
private static final String FT_NAME_CANCEL_INVALIDATION = "FT_CANCELINVALIDATION_OAK-10595";
+ /**
+ * Feature toggle name to enable full GC for Mongo Document Store
+ */
+ private static final String FT_NAME_FULL_GC = "FT_FULL_GC_OAK-10199";
+
+ /**
+ * Feature toggle name to enable embedded verification for full GC mode for Mongo Document Store
+ */
+ private static final String FT_NAME_EMBEDDED_VERIFICATION = "FT_EMBEDDED_VERIFICATION_OAK-10633";
// property name constants - values can come from framework properties or OSGi config
public static final String CUSTOM_BLOB_STORE = "customBlobStore";
@@ -225,6 +236,8 @@ public class DocumentNodeStoreService {
private Feature docStoreThrottlingFeature;
private Feature noChildOrderCleanupFeature;
private Feature cancelInvalidationFeature;
+ private Feature docStoreFullGCFeature;
+ private Feature docStoreEmbeddedVerificationFeature;
private ComponentContext context;
private Whiteboard whiteboard;
private long deactivationTimestamp = 0;
@@ -261,6 +274,8 @@ public class DocumentNodeStoreService {
docStoreThrottlingFeature = Feature.newFeature(FT_NAME_DOC_STORE_THROTTLING, whiteboard);
noChildOrderCleanupFeature = Feature.newFeature(FT_NAME_DOC_STORE_NOCOCLEANUP, whiteboard);
cancelInvalidationFeature = Feature.newFeature(FT_NAME_CANCEL_INVALIDATION, whiteboard);
+ docStoreFullGCFeature = Feature.newFeature(FT_NAME_FULL_GC, whiteboard);
+ docStoreEmbeddedVerificationFeature = Feature.newFeature(FT_NAME_EMBEDDED_VERIFICATION, whiteboard);
registerNodeStoreIfPossible();
}
@@ -480,7 +495,11 @@ public class DocumentNodeStoreService {
setDocStoreThrottlingFeature(docStoreThrottlingFeature).
setNoChildOrderCleanupFeature(noChildOrderCleanupFeature).
setCancelInvalidationFeature(cancelInvalidationFeature).
+ setDocStoreFullGCFeature(docStoreFullGCFeature).
+ setDocStoreEmbeddedVerificationFeature(docStoreEmbeddedVerificationFeature).
setThrottlingEnabled(config.throttlingEnabled()).
+ setFullGCEnabled(config.fullGCEnabled()).
+ setEmbeddedVerificationEnabled(config.embeddedVerificationEnabled()).
setSuspendTimeoutMillis(config.suspendTimeoutMillis()).
setClusterIdReuseDelayAfterRecovery(config.clusterIdReuseDelayAfterRecoveryMillis()).
setRecoveryDelayMillis(config.recoveryDelayMillis()).
@@ -635,6 +654,14 @@ public class DocumentNodeStoreService {
cancelInvalidationFeature.close();
}
+ if (docStoreFullGCFeature != null) {
+ docStoreFullGCFeature.close();
+ }
+
+ if (docStoreEmbeddedVerificationFeature != null) {
+ docStoreEmbeddedVerificationFeature.close();
+ }
+
unregisterNodeStore();
}
@@ -1005,6 +1032,10 @@ public class DocumentNodeStoreService {
VersionGCStats s = gc.gc(versionGCMaxAgeInSecs, TimeUnit.SECONDS);
stats.addRun(s);
lastResult = s.toString();
+ if (s.skippedFullGCDocsCount > 0) {
+ LOGGER.warn("Version Garbage Collector's FullGC skipped {} documents due to error,"
+ + " see logs for more details", s.skippedFullGCDocsCount);
+ }
} catch (Exception e) {
lastResult = e;
LOGGER.warn("Error occurred while executing the Version Garbage Collector", e);
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/FullGCStatsCollector.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/FullGCStatsCollector.java
new file mode 100644
index 0000000000..506fa62ce6
--- /dev/null
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/FullGCStatsCollector.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCStats;
+
+/**
+ * Collector interface for {@link DocumentNodeStore} full garbage collection
+ * statistics.
+ */
+public interface FullGCStatsCollector {
+
+ /**
+ * Total No. of documents read during FullGC phase
+ */
+ void documentRead();
+
+ /**
+ * No. of orphan nodes deleted during FullGC
+ * @param numNodes no. of orphan nodes deleted in current cycle
+ */
+ void orphanNodesDeleted(long numNodes);
+
+ /**
+ * No. of properties deleted during FullGC
+ * @param numProps no. of properties deleted in current cycle
+ */
+ void propertiesDeleted(long numProps);
+
+ /**
+ * No. of unmerged (unique) branch commits deleted during FullGC
+ * @param numCommits no. of unmerged branch commits deleted in current cycle
+ */
+ void unmergedBranchCommitsDeleted(long numCommits);
+
+ /**
+ * No. of documents updated (i.e. have garbage removed) during FullGC
+ * @param numDocs no. of documents updated in current cycle
+ */
+ void documentsUpdated(long numDocs);
+
+ /**
+ * No. of documents which had skipped update (i.e. have been updated between garbage collection & removal)
+ * during FullGC
+ * @param numDocs No. of documents which had skipped update in current cycle
+ */
+ void documentsUpdateSkipped(long numDocs);
+
+ /**
+ * No. of times the FullGC has started
+ */
+ void started();
+
+ /**
+ * Timer for different phases in FullGC
+ * @param stats {@link VersionGCStats} containing FullGC phases timer
+ */
+ void finished(VersionGCStats stats);
+}
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/FullGCStatsCollectorImpl.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/FullGCStatsCollectorImpl.java
new file mode 100644
index 0000000000..64be29864c
--- /dev/null
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/FullGCStatsCollectorImpl.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCStats;
+import org.apache.jackrabbit.oak.stats.CounterStats;
+import org.apache.jackrabbit.oak.stats.MeterStats;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+import org.apache.jackrabbit.oak.stats.TimerStats;
+
+import static java.util.concurrent.TimeUnit.MICROSECONDS;
+import static org.apache.jackrabbit.oak.stats.StatsOptions.DEFAULT;
+import static org.apache.jackrabbit.oak.stats.StatsOptions.METRICS_ONLY;
+
+/**
+ * {@link DocumentNodeStore} Full garbage collection statistics.
+ */
+class FullGCStatsCollectorImpl implements FullGCStatsCollector {
+
+ static final String FULL_GC = "FullGC";
+ static final String READ_DOC = "READ_DOC";
+ static final String DELETED_ORPHAN_NODE = "DELETED_ORPHAN_NODE";
+ static final String DELETED_PROPERTY = "DELETED_PROPERTY";
+ static final String DELETED_UNMERGED_BC = "DELETED_UNMERGED_BC";
+ static final String UPDATED_DOC = "UPDATED_DOC";
+ static final String SKIPPED_DOC = "SKIPPED_DOC";
+ static final String FULL_GC_ACTIVE_TIMER = "FULL_GC_ACTIVE_TIMER";
+ static final String FULL_GC_TIMER = "FULL_GC_TIMER";
+ static final String COLLECT_FULL_GC_TIMER = "COLLECT_FULL_GC_TIMER";
+ static final String COLLECT_ORPHAN_NODES_TIMER = "COLLECT_ORPHAN_NODES_TIMER";
+ static final String COLLECT_DELETED_PROPS_TIMER = "COLLECT_DELETED_PROPS_TIMER";
+ static final String COLLECT_DELETED_OLD_REVS_TIMER = "COLLECT_DELETED_OLD_REVS_TIMER";
+ static final String COLLECT_UNMERGED_BC_TIMER = "COLLECT_UNMERGED_BC_TIMER";
+ static final String DELETE_FULL_GC_DOCS_TIMER = "DELETE_FULL_GC_DOCS_TIMER";
+
+ static final String COUNTER = "COUNTER";
+ static final String FAILURE_COUNTER = "FAILURE";
+
+ private final MeterStats readDoc;
+ private final MeterStats deletedOrphanNode;
+ private final MeterStats deletedProperty;
+ private final MeterStats deletedUnmergedBC;
+ private final MeterStats updatedDoc;
+ private final MeterStats skippedDoc;
+ private final TimerStats fullGCActiveTimer;
+ private final TimerStats fullGCTimer;
+ private final TimerStats collectFullGCTimer;
+ private final TimerStats collectOrphanNodesTimer;
+ private final TimerStats collectDeletedPropsTimer;
+ private final TimerStats collectDeletedOldRevsTimer;
+ private final TimerStats collectUnmergedBCTimer;
+ private final TimerStats deleteFullGCDocsTimer;
+
+ private final CounterStats counter;
+ private final CounterStats failureCounter;
+
+ FullGCStatsCollectorImpl(StatisticsProvider provider) {
+
+ readDoc = meter(provider, READ_DOC);
+ deletedOrphanNode = meter(provider, DELETED_ORPHAN_NODE);
+ deletedProperty = meter(provider, DELETED_PROPERTY);
+ deletedUnmergedBC = meter(provider, DELETED_UNMERGED_BC);
+ updatedDoc = meter(provider, UPDATED_DOC);
+ skippedDoc = meter(provider, SKIPPED_DOC);
+
+ fullGCActiveTimer = timer(provider, FULL_GC_ACTIVE_TIMER);
+ fullGCTimer = timer(provider, FULL_GC_TIMER);
+ collectFullGCTimer = timer(provider, COLLECT_FULL_GC_TIMER);
+ collectOrphanNodesTimer = timer(provider, COLLECT_ORPHAN_NODES_TIMER);
+ collectDeletedPropsTimer = timer(provider, COLLECT_DELETED_PROPS_TIMER);
+ collectDeletedOldRevsTimer = timer(provider, COLLECT_DELETED_OLD_REVS_TIMER);
+ collectUnmergedBCTimer = timer(provider, COLLECT_UNMERGED_BC_TIMER);
+ deleteFullGCDocsTimer = timer(provider, DELETE_FULL_GC_DOCS_TIMER);
+
+ counter = counter(provider, COUNTER);
+ failureCounter = counter(provider, FAILURE_COUNTER);
+ }
+
+ //---------------------< FullGCStatsCollector >-------------------------
+
+ @Override
+ public void documentRead() {
+ readDoc.mark();
+ }
+
+ @Override
+ public void orphanNodesDeleted(long numNodes) {
+ deletedOrphanNode.mark(numNodes);
+ }
+
+ @Override
+ public void propertiesDeleted(long numProps) {
+ deletedProperty.mark(numProps);
+ }
+
+ @Override
+ public void unmergedBranchCommitsDeleted(long numCommits) {
+ deletedUnmergedBC.mark(numCommits);
+ }
+
+ @Override
+ public void documentsUpdated(long numDocs) {
+ updatedDoc.mark(numDocs);
+ }
+
+ @Override
+ public void documentsUpdateSkipped(long numDocs) {
+ skippedDoc.mark(numDocs);
+ }
+
+ @Override
+ public void started() {
+ counter.inc();
+ }
+
+ @Override
+ public void finished(VersionGCStats stats) {
+ fullGCActiveTimer.update(stats.fullGCActiveElapsed, MICROSECONDS);
+ fullGCTimer.update(stats.fullGCDocsElapsed, MICROSECONDS);
+ collectFullGCTimer.update(stats.collectFullGCElapsed, MICROSECONDS);
+ collectOrphanNodesTimer.update(stats.collectOrphanNodesElapsed, MICROSECONDS);
+ collectDeletedPropsTimer.update(stats.collectDeletedPropsElapsed, MICROSECONDS);
+ collectDeletedOldRevsTimer.update(stats.collectDeletedOldRevsElapsed, MICROSECONDS);
+ collectUnmergedBCTimer.update(stats.collectUnmergedBCElapsed, MICROSECONDS);
+ deleteFullGCDocsTimer.update(stats.deleteFullGCDocsElapsed, MICROSECONDS);
+ if (!stats.success) {
+ failureCounter.inc();
+ }
+ }
+
+
+ //----------------------------< internal >----------------------------------
+
+ private static MeterStats meter(StatisticsProvider provider, String name) {
+ return provider.getMeter(qualifiedName(name), DEFAULT);
+ }
+
+ private static TimerStats timer(StatisticsProvider provider, String name) {
+ return provider.getTimer(qualifiedName(name), METRICS_ONLY);
+ }
+
+ private static CounterStats counter(StatisticsProvider provider, String name) {
+ return provider.getCounterStats(qualifiedName(name), METRICS_ONLY);
+ }
+
+ private static String qualifiedName(String metricName) {
+ return FULL_GC + "." + metricName;
+ }
+
+}
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java
index 6e7afeaf62..d823247fbb 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java
@@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.plugins.document;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@@ -58,6 +59,7 @@ import org.apache.jackrabbit.guava.common.collect.Iterables;
import org.apache.jackrabbit.guava.common.collect.Maps;
import org.apache.jackrabbit.guava.common.collect.Sets;
+import static java.util.stream.Collectors.toSet;
import static org.apache.jackrabbit.guava.common.base.Objects.equal;
import static org.apache.jackrabbit.guava.common.base.Preconditions.checkArgument;
import static org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull;
@@ -168,7 +170,7 @@ public final class NodeDocument extends Document {
/**
* Whether this node is deleted. Key: revision, value: true/false.
*/
- private static final String DELETED = "_deleted";
+ static final String DELETED = "_deleted";
/**
* Flag indicating that whether this node was ever deleted. Its just used as
@@ -227,7 +229,7 @@ public final class NodeDocument extends Document {
/**
* Contains revision entries for changes done by branch commits.
*/
- private static final String BRANCH_COMMITS = "_bc";
+ static final String BRANCH_COMMITS = "_bc";
/**
* The revision set by the background document sweeper. The revision
@@ -928,6 +930,53 @@ public final class NodeDocument extends Document {
return false;
}
+ /**
+ * Resolve the commit revision that holds the current value of a property based
+ * on provided readRevision if the current value is in the local
+ * map - null if the current value might be in a split doc or the node or property
+ * does not exist at all.
+ *
+ * @param nodeStore the node store.
+ * @param readRevision the read revision.
+ * @param key the key of the property to resolve
+ * @return a Revision if the value of the property resolves to a value based
+ * on what's in the local document, null if the node or property does
+ * not exist at all or the value is in a split document.
+ */
+ Revision localCommitRevisionOfProperty(@NotNull DocumentNodeStore nodeStore,
+ @NotNull RevisionVector readRevision,
+ @NotNull String key) {
+ Map<Revision, String> validRevisions = new HashMap<>();
+ Branch branch = nodeStore.getBranches().getBranch(readRevision);
+ LastRevs lastRevs = createLastRevs(readRevision,
+ nodeStore, branch, null);
+
+ Revision min = getLiveRevision(nodeStore, readRevision, validRevisions, lastRevs);
+ if (min == null) {
+ // node is deleted
+ return null;
+ }
+
+ // ignore when local map is empty (OAK-2442)
+ SortedMap<Revision, String> local = getLocalMap(key);
+ if (local.isEmpty()) {
+ return null;
+ }
+
+ // first check local map, which contains most recent values
+ Value value = getLatestValue(nodeStore, local.entrySet(),
+ readRevision, validRevisions, lastRevs);
+ if (value == null) {
+ return null;
+ }
+ // check if there may be more recent values in a previous document
+ if (requiresCompleteMapCheck(value, local, nodeStore)) {
+ return null;
+ } else {
+ return value.valueEntry.getKey();
+ }
+ }
+
/**
* Returns a {@link DocumentNodeState} as seen at the given
* <code>readRevision</code>.
@@ -979,7 +1028,7 @@ public final class NodeDocument extends Document {
readRevision, validRevisions, lastRevs);
}
String propertyName = Utils.unescapePropertyName(key);
- String v = value != null ? value.value : null;
+ String v = value != null ? value.valueEntry.getValue() : null;
if (v != null){
props.add(nodeStore.createPropertyState(propertyName, v));
}
@@ -1064,7 +1113,7 @@ public final class NodeDocument extends Document {
value = getLatestValue(context, getDeleted().entrySet(), readRevision, validRevisions, lastRevs);
}
- return value != null && "false".equals(value.value) ? value.revision : null;
+ return value != null && "false".equals(value.valueEntry.getValue()) ? value.revision : null;
}
/**
@@ -1659,6 +1708,24 @@ public final class NodeDocument extends Document {
return map;
}
+ /**
+ * Returns name of all the properties on this document
+ * <p>
+ * Note: property names returned are escaped
+ *
+ * @return Set of all property names (escaped)
+ * @see Utils#unescapePropertyName(String)
+ * @see Utils#escapePropertyName(String)
+ */
+ @NotNull
+ Set<String> getPropertyNames() {
+ return data
+ .keySet()
+ .stream()
+ .filter(Utils::isPropertyName)
+ .collect(toSet());
+ }
+
/**
* @return the {@link #REVISIONS} stored on this document.
*/
@@ -2234,7 +2301,7 @@ public final class NodeDocument extends Document {
}
if (isValidRevision(context, propRev, commitValue, readRevision, validRevisions)) {
- return new Value(commitRev, entry.getValue());
+ return new Value(commitRev, entry);
}
}
return null;
@@ -2344,14 +2411,16 @@ public final class NodeDocument extends Document {
final Revision revision;
/**
- * The value of a property at the given revision. A {@code null} value
+ * valueEntry contains both the underlying (commit) revision and
+ * the (String) value of a property. valueEntry is never null.
+ * valueEntry.getValue() being {@code null}
* indicates the property was removed.
*/
- final String value;
+ final Map.Entry<Revision, String> valueEntry;
- Value(@NotNull Revision revision, @Nullable String value) {
- this.revision = checkNotNull(revision);
- this.value = value;
+ Value(@NotNull Revision mergeRevision, @NotNull Map.Entry<Revision, String> valueEntry) {
+ this.revision = checkNotNull(mergeRevision);
+ this.valueEntry = valueEntry;
}
}
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleaner.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleaner.java
new file mode 100644
index 0000000000..8713a0aff4
--- /dev/null
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleaner.java
@@ -0,0 +1,320 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+import org.apache.jackrabbit.oak.plugins.document.UpdateOp.Key;
+import org.apache.jackrabbit.oak.plugins.document.UpdateOp.Operation;
+import org.apache.jackrabbit.oak.plugins.document.UpdateOp.Operation.Type;
+import org.apache.jackrabbit.oak.plugins.document.util.Utils;
+import org.jetbrains.annotations.NotNull;
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.Map.Entry;
+
+/**
+ * This is a prototype class of a very fine-grained revision cleaner that cleans even revisions
+ * in-between checkpoints. It is not clear if it will be used for now.
+ *
+ * A version is considered to be cleanable if it is not referenced by any checkpoint and is older than
+ * a certain threshold.
+ */
+public class NodeDocumentRevisionCleaner {
+
+ private final DocumentNodeStore documentNodeStore;
+ private final NodeDocument workingDocument;
+ private final RevisionPropertiesClassifier revisionClassifier;
+ private final RevisionCleanerUtility revisionCleaner;
+ private long toModifiedMs;
+
+ /**
+ * Constructor for NodeDocumentRevisionCleaner.
+ * @param documentNodeStore The DocumentNodeStore instance.
+ * @param workingDocument The document to clean up.
+ */
+ public NodeDocumentRevisionCleaner(DocumentNodeStore documentNodeStore, NodeDocument workingDocument) {
+ this(documentNodeStore, workingDocument, Instant.now().minus(24, ChronoUnit.HOURS).toEpochMilli());
+ }
+
+ /**
+ * Constructor for NodeDocumentRevisionCleaner.
+ * @param documentNodeStore The DocumentNodeStore instance.
+ * @param workingDocument The document to clean up.
+ */
+ public NodeDocumentRevisionCleaner(DocumentNodeStore documentNodeStore, NodeDocument workingDocument, long toModifiedMs) {
+ this.workingDocument = workingDocument;
+ this.documentNodeStore = documentNodeStore;
+ this.toModifiedMs = toModifiedMs;
+
+ revisionClassifier = new RevisionPropertiesClassifier(workingDocument);
+ revisionCleaner = new RevisionCleanerUtility(revisionClassifier);
+ }
+
+ /**
+ * Collects cleanable old revisions for the given document.
+ */
+ public void collectOldRevisions(UpdateOp op) {
+ revisionClassifier.classifyRevisionsAndProperties();
+ revisionCleaner.preserveRevisionsNewerThanThreshold(toModifiedMs);
+ revisionCleaner.preserveLastRevisionForEachProperty();
+ revisionCleaner.preserveRevisionsReferencedByCheckpoints();
+ revisionCleaner.removeCandidatesInList();
+
+ for (Map.Entry<Integer, TreeSet<Revision>> entry : revisionCleaner.getCandidateRevisionsToClean().entrySet()) {
+ for (Revision revision : entry.getValue()) {
+ TreeSet<String> properties = revisionClassifier.getPropertiesModifiedByRevision().get(revision);
+ if (properties != null) {
+ outer:for (String property : properties) {
+ Map<Key, Operation> c = op.getChanges();
+ for (Entry<Key, Operation> e : c.entrySet()) {
+ if (e.getKey().equals(new Key(property, null)) && e.getValue().type == Type.REMOVE) {
+ continue outer;
+ }
+ }
+ op.removeMapEntry(property, revision);
+ }
+ }
+ RevisionVector sweepRevisions = documentNodeStore.getSweepRevisions();
+ boolean newerThanSweep = sweepRevisions == null ? false : sweepRevisions.isRevisionNewer(revision);
+ boolean isBC = workingDocument.getLocalBranchCommits().contains(revision);
+ if (!newerThanSweep && !isBC) {
+ op.removeMapEntry("_revisions", revision);
+ }
+ }
+ }
+ }
+
+ private class RevisionPropertiesClassifier {
+ private final NodeDocument workingDocument;
+ private SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingPropertyByCluster;
+ private SortedMap<String, TreeSet<Revision>> revisionsModifyingProperty;
+ private SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision;
+
+ private RevisionPropertiesClassifier(NodeDocument workingDocument) {
+ this.workingDocument = workingDocument;
+
+ this.revisionsModifyingPropertyByCluster = new TreeMap<>();
+ this.revisionsModifyingProperty = new TreeMap<>();
+ this.propertiesModifiedByRevision = new TreeMap<>(StableRevisionComparator.INSTANCE);
+ }
+
+ /**
+ * This method processes the revisions of the working document, creating maps to
+ * track the relationships between revisions and modified properties.
+ */
+ private void classifyRevisionsAndProperties() {
+ SortedMap<Revision, String> deletedRevisions = workingDocument.getLocalDeleted();
+ // Always keep the first "_deleted" entry, as is when the document was created
+ if (!deletedRevisions.isEmpty()) {
+ Revision createdRevision = deletedRevisions.firstKey();
+ revisionCleaner.addBlockedRevisionToKeep(createdRevision);
+ }
+
+ SortedMap<Revision, String> documentRevisions = workingDocument.getLocalRevisions();
+ for (Map.Entry<Revision, String> revisionEntry : documentRevisions.entrySet()) {
+ Revision revision = revisionEntry.getKey();
+ String revisionValue = revisionEntry.getValue();
+
+ // Only check committed revisions
+ if (Utils.isCommitted(revisionValue)) {
+ // Candidate to clean up
+ revisionCleaner.addCandidateRevisionToClean(revision);
+ // Store properties usage
+ classifyPropertiesModifiedByRevision(revision);
+ }
+ }
+ }
+
+ private void classifyPropertiesModifiedByRevision(Revision revision) {
+ for (Map.Entry<String, Object> propertyEntry : workingDocument.entrySet()) {
+ if (Utils.isPropertyName(propertyEntry.getKey()) || NodeDocument.isDeletedEntry(propertyEntry.getKey())) {
+ Map<Revision, String> valueMap = (Map) propertyEntry.getValue();
+ if (valueMap.containsKey(revision)) {
+ propertiesModifiedByRevision.computeIfAbsent(revision, key ->
+ new TreeSet<>()).add(propertyEntry.getKey()
+ );
+
+ revisionsModifyingPropertyByCluster.computeIfAbsent(propertyEntry.getKey(), key ->
+ new TreeMap<>()
+ ).computeIfAbsent(revision.getClusterId(), key ->
+ new TreeSet<>(StableRevisionComparator.INSTANCE)
+ ).add(revision);
+
+ revisionsModifyingProperty.computeIfAbsent(propertyEntry.getKey(), key ->
+ new TreeSet<>(StableRevisionComparator.INSTANCE)
+ ).add(revision);
+ }
+ }
+ }
+ }
+
+ public SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> getRevisionsModifyingPropertyByCluster() {
+ return revisionsModifyingPropertyByCluster;
+ }
+
+ public SortedMap<String, TreeSet<Revision>> getRevisionsModifyingProperty() {
+ return revisionsModifyingProperty;
+ }
+
+ public SortedMap<Revision, TreeSet<String>> getPropertiesModifiedByRevision() {
+ return propertiesModifiedByRevision;
+ }
+ }
+
+ private class RevisionCleanerUtility {
+
+ private final SortedMap<Integer, TreeSet<Revision>> blockedRevisionsToKeep;
+ private final SortedMap<Integer, TreeSet<Revision>> candidateRevisionsToClean;
+ private final RevisionPropertiesClassifier revisionClassifier;
+
+ private RevisionCleanerUtility(RevisionPropertiesClassifier revisionClassifier) {
+ this.revisionClassifier = revisionClassifier;
+ this.candidateRevisionsToClean = new TreeMap<>();
+ this.blockedRevisionsToKeep = new TreeMap<>();
+ }
+
+ private void preserveLastRevisionForEachProperty() {
+ for (SortedMap<Integer, TreeSet<Revision>> revisionsByCluster : revisionClassifier.getRevisionsModifyingPropertyByCluster().values()) {
+ for (TreeSet<Revision> revisions : revisionsByCluster.values()) {
+ Revision lastRevision = revisions.last();
+ addBlockedRevisionToKeep(lastRevision);
+ }
+ }
+ }
+
+ private void preserveRevisionsNewerThanThreshold(long amount, ChronoUnit unit) {
+ long thresholdToPreserve = Instant.now().minus(amount, unit).toEpochMilli();
+ preserveRevisionsNewerThanThreshold(thresholdToPreserve);
+ }
+
+ private void preserveRevisionsNewerThanThreshold(long thresholdToPreserve) {
+ for (TreeSet<Revision> revisionSet : candidateRevisionsToClean.values()) {
+ for (Revision revision : revisionSet) {
+ if (revision.getTimestamp() > thresholdToPreserve) {
+ addBlockedRevisionToKeep(revision);
+ }
+ }
+ }
+ }
+
+ private void preserveRevisionsReferencedByCheckpoints() {
+ SortedMap<Revision, Checkpoints.Info> checkpoints = documentNodeStore.getCheckpoints().getCheckpoints();
+ checkpoints.forEach((revision, info) -> {
+ // For each checkpoint, keep the last revision that modified a property prior to checkpoint
+ revisionClassifier.getRevisionsModifyingProperty().forEach((propertyName, revisionsSet) -> {
+ // Traverse the revisionVector of the checkpoint and find the last revision that modified the property
+ info.getCheckpoint().forEach(revisionToFind -> {
+ // If the exact revision exists, keep it. If not, find the previous one that modified that property
+ if (revisionsSet.contains(revisionToFind)) {
+ addBlockedRevisionToKeep(revisionToFind);
+ } else {
+ Revision previousRevision = revisionsSet.descendingSet().ceiling(revisionToFind);
+ if (previousRevision != null) {
+ addBlockedRevisionToKeep(previousRevision);
+ }
+ }
+ });
+ });
+ });
+ }
+
+ /**
+ * Adds a revision to the list of candidates to delete.
+ * @param revision
+ */
+ private void addCandidateRevisionToClean(Revision revision) {
+ candidateRevisionsToClean.computeIfAbsent(revision.getClusterId(), key ->
+ new TreeSet<>(StableRevisionComparator.INSTANCE)
+ ).add(revision);
+ }
+
+ /**
+ * Adds a revision to the list of revisions to keep.
+ * @param revision
+ */
+ private void addBlockedRevisionToKeep(Revision revision) {
+ blockedRevisionsToKeep.computeIfAbsent(revision.getClusterId(), key ->
+ new TreeSet<>(StableRevisionComparator.INSTANCE)
+ ).add(revision);
+ }
+
+ private void removeCandidatesInList() {
+ revisionCleaner.blockedRevisionsToKeep.forEach((key, value) -> {
+ if (revisionCleaner.getCandidateRevisionsToClean().containsKey(key)) {
+ revisionCleaner.getCandidateRevisionsToClean().get(key).removeAll(value);
+ }
+ });
+ }
+
+ public SortedMap<Integer, TreeSet<Revision>> getBlockedRevisionsToKeep() {
+ return blockedRevisionsToKeep;
+ }
+
+ public SortedMap<Integer, TreeSet<Revision>> getCandidateRevisionsToClean() {
+ return candidateRevisionsToClean;
+ }
+ }
+
+ /*
+ * The following methods are used to expose the internal state of the cleaner for testing/debugging purposes.
+ */
+ protected SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> getRevisionsModifyingPropertyByCluster() {
+ return revisionClassifier.getRevisionsModifyingPropertyByCluster();
+ }
+
+ protected SortedMap<String, TreeSet<Revision>> getRevisionsModifyingProperty() {
+ return revisionClassifier.getRevisionsModifyingProperty();
+ }
+
+ protected SortedMap<Revision, TreeSet<String>> getPropertiesModifiedByRevision() {
+ return revisionClassifier.getPropertiesModifiedByRevision();
+ }
+
+ public SortedMap<Integer, TreeSet<Revision>> getBlockedRevisionsToKeep() {
+ return revisionCleaner.getBlockedRevisionsToKeep();
+ }
+
+ public SortedMap<Integer, TreeSet<Revision>> getCandidateRevisionsToClean() {
+ return revisionCleaner.getCandidateRevisionsToClean();
+ }
+
+ protected void classifyRevisionsAndProperties() {
+ revisionClassifier.classifyRevisionsAndProperties();
+ }
+
+ protected void markLastRevisionForEachProperty() {
+ revisionCleaner.preserveLastRevisionForEachProperty();
+ }
+
+ protected void markRevisionsNewerThanThresholdToPreserve(long amount, ChronoUnit unit) {
+ revisionCleaner.preserveRevisionsNewerThanThreshold(amount, unit);
+ }
+
+ protected void markCheckpointRevisionsToPreserve() {
+ revisionCleaner.preserveRevisionsReferencedByCheckpoints();
+ }
+
+ protected void removeCandidatesInList() {
+ revisionCleaner.removeCandidatesInList();
+ }
+}
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java
index 363c65789b..6ee38b4925 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java
@@ -18,18 +18,35 @@
*/
package org.apache.jackrabbit.oak.plugins.document;
+import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCStats;
import org.apache.jackrabbit.oak.plugins.document.util.TimeInterval;
import org.apache.jackrabbit.oak.plugins.document.util.Utils;
import org.apache.jackrabbit.oak.spi.gc.GCMonitor;
import org.apache.jackrabbit.oak.stats.Clock;
+import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.jackrabbit.guava.common.collect.Maps;
+import static java.lang.Long.MAX_VALUE;
+import static java.util.Map.of;
+import static java.util.Optional.ofNullable;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MIN_ID_VALUE;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGCRecommendations.GcType.DGC;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGCRecommendations.GcType.RGC;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_FULL_GC_DRY_RUN_DOCUMENT_ID_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_FULL_GC_DRY_RUN_TIMESTAMP_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_ID;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_REC_INTERVAL_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.util.Utils.timestampToString;
/**
* Gives a recommendation about parameters for the next revision garbage collection run.
@@ -38,19 +55,33 @@ public class VersionGCRecommendations {
private static final Logger log = LoggerFactory.getLogger(VersionGCRecommendations.class);
+ private static final long IGNORED_GC_WARNING_INTERVAL_MS = TimeUnit.MINUTES.toMillis(5); // 5min
+ private static long lastIgnoreWarning = 0;
+
private final VersionGCSupport vgc;
private final GCMonitor gcmon;
final boolean ignoreDueToCheckPoint;
+ final boolean ignoreFullGCDueToCheckPoint;
final TimeInterval scope;
+ final TimeInterval scopeFullGC;
final long maxCollect;
final long deleteCandidateCount;
final long lastOldestTimestamp;
+ final long fullGCTimestamp;
+ final long fullGCDryRunTimestamp;
+ final String fullGCId;
final long originalCollectLimit;
-
private final long precisionMs;
final long suggestedIntervalMs;
private final boolean scopeIsComplete;
+ private final boolean fullGCScopeIsComplete;
+ private final boolean fullGCEnabled;
+
+ // whether fullGC is running in dryRun or not. Please note that this mode is to be run only
+ // either via command line i.e. oak-run or as management bean command.
+ // It will also run only if fullGC is not running.
+ private final boolean isFullGCDryRun;
/**
* With the given maximum age of revisions to keep (earliest time in the past to collect),
@@ -67,33 +98,43 @@ public class VersionGCRecommendations {
* In the settings collection, recommendations keeps "revisionsOlderThan" from the last successful run.
* It also updates the time interval recommended for the next run.
*
- * @param maxRevisionAgeMs the minimum age for revisions to be collected
- * @param checkpoints checkpoints from {@link DocumentNodeStore}
- * @param clock clock from {@link DocumentNodeStore}
- * @param vgc VersionGC support class
- * @param options options for running the gc
- * @param gcMonitor monitor class for messages
+ * @param maxRevisionAgeMs the minimum age for revisions to be collected
+ * @param checkpoints checkpoints from {@link DocumentNodeStore}
+ * @param clock clock from {@link DocumentNodeStore}
+ * @param vgc VersionGC support class
+ * @param options options for running the gc
+ * @param gcMonitor monitor class for messages
+ * @param fullGCEnabled whether fullGC is enabled or not
+ * @param isFullGCDryRun whether fullGC is running in dryRun mode or not
*/
- public VersionGCRecommendations(long maxRevisionAgeMs, Checkpoints checkpoints, Clock clock, VersionGCSupport vgc,
- VersionGCOptions options, GCMonitor gcMonitor) {
- boolean ignoreDueToCheckPoint = false;
+ VersionGCRecommendations(long maxRevisionAgeMs, Checkpoints checkpoints, Clock clock, VersionGCSupport vgc,
+ VersionGCOptions options, GCMonitor gcMonitor, boolean fullGCEnabled,
+ boolean isFullGCDryRun) {
+ boolean ignoreDueToCheckPoint;
+ boolean ignoreFullGCDueToCheckPoint;
long deletedOnceCount = 0;
long suggestedIntervalMs;
long oldestPossible;
+ final AtomicLong oldestModifiedDocTimeStamp = new AtomicLong();
+ final AtomicLong oldestModifiedDryRunDocTimeStamp = new AtomicLong();
+ String oldestModifiedDocId;
+ String oldestModifiedDryRunDocId;
long collectLimit = options.collectLimit;
this.vgc = vgc;
this.gcmon = gcMonitor;
this.originalCollectLimit = options.collectLimit;
+ this.fullGCEnabled = fullGCEnabled;
+ this.isFullGCDryRun = isFullGCDryRun;
TimeInterval keep = new TimeInterval(clock.getTime() - maxRevisionAgeMs, Long.MAX_VALUE);
- Map<String, Long> settings = getLongSettings();
- lastOldestTimestamp = settings.get(VersionGarbageCollector.SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP);
+ Map<String, Object> settings = getVGCSettings();
+ lastOldestTimestamp = (long) settings.get(SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP);
if (lastOldestTimestamp == 0) {
- log.debug("No lastOldestTimestamp found, querying for the oldest deletedOnce candidate");
+ log.info("No lastOldestTimestamp found, querying for the oldest deletedOnce candidate");
oldestPossible = vgc.getOldestDeletedOnceTimestamp(clock, options.precisionMs) - 1;
- log.debug("lastOldestTimestamp found: {}", Utils.timestampToString(oldestPossible));
+ log.info("lastOldestTimestamp found: {}", Utils.timestampToString(oldestPossible));
} else {
oldestPossible = lastOldestTimestamp - 1;
}
@@ -101,7 +142,53 @@ public class VersionGCRecommendations {
TimeInterval scope = new TimeInterval(oldestPossible, Long.MAX_VALUE);
scope = scope.notLaterThan(keep.fromMs);
- suggestedIntervalMs = settings.get(VersionGarbageCollector.SETTINGS_COLLECTION_REC_INTERVAL_PROP);
+ fullGCTimestamp = (long) settings.get(SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP);
+ oldestModifiedDocId = (String) settings.get(SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP);
+
+ fullGCDryRunTimestamp = (long) settings.get(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_TIMESTAMP_PROP);
+ oldestModifiedDryRunDocId = (String) settings.get(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_DOCUMENT_ID_PROP);
+
+ if (log.isDebugEnabled()) {
+ if (isFullGCDryRun) {
+ log.debug("lastOldestTimestamp: {}, fullGCDryRunTimestamp: {}, oldestModifiedDryRunDocId: {}",
+ timestampToString(lastOldestTimestamp), timestampToString(fullGCDryRunTimestamp), oldestModifiedDryRunDocId);
+ } else {
+ log.debug("lastOldestTimestamp: {}, fullGCTimestamp: {}, oldestModifiedDocId: {}",
+ timestampToString(lastOldestTimestamp), timestampToString(fullGCTimestamp), oldestModifiedDocId);
+ }
+ }
+
+ if (isFullGCDryRun) {
+ if (fullGCDryRunTimestamp == 0) {
+ // it will only happen for the very first time, we run this fullGC in dry run mode
+ log.info("No fullGCDryRunTimestamp found, querying for the oldest modified candidate");
+ vgc.getOldestModifiedDoc(clock).ifPresentOrElse(
+ d -> oldestModifiedDryRunDocTimeStamp.set(SECONDS.toMillis(ofNullable(d.getModified()).orElse(0L))),
+ () -> oldestModifiedDryRunDocTimeStamp.set(0L));
+ oldestModifiedDryRunDocId = MIN_ID_VALUE;
+ log.info("fullGCDryRunTimestamp found: {}", timestampToString(oldestModifiedDryRunDocTimeStamp.get()));
+ } else {
+ oldestModifiedDryRunDocTimeStamp.set(fullGCDryRunTimestamp);
+ }
+ } else {
+ if (fullGCTimestamp == 0) {
+ // it will only happen for the very first time, we run this fullGC
+ log.info("No fullGCTimestamp found, querying for the oldest modified candidate");
+ vgc.getOldestModifiedDoc(clock).ifPresentOrElse(
+ d -> oldestModifiedDocTimeStamp.set(SECONDS.toMillis(ofNullable(d.getModified()).orElse(0L))),
+ () -> oldestModifiedDocTimeStamp.set(0L));
+ oldestModifiedDocId = MIN_ID_VALUE;
+ log.info("fullGCTimestamp found: {}", timestampToString(oldestModifiedDocTimeStamp.get()));
+ } else {
+ oldestModifiedDocTimeStamp.set(fullGCTimestamp);
+ }
+ }
+
+ TimeInterval scopeFullGC = new TimeInterval(isFullGCDryRun ? oldestModifiedDryRunDocTimeStamp.get() :
+ oldestModifiedDocTimeStamp.get(), MAX_VALUE);
+ scopeFullGC = scopeFullGC.notLaterThan(keep.fromMs);
+
+ suggestedIntervalMs = (long) settings.get(SETTINGS_COLLECTION_REC_INTERVAL_PROP);
if (suggestedIntervalMs > 0) {
suggestedIntervalMs = Math.max(suggestedIntervalMs, options.precisionMs);
if (suggestedIntervalMs < scope.getDurationMs()) {
@@ -137,17 +224,14 @@ public class VersionGCRecommendations {
//Check for any registered checkpoint which prevent the GC from running
Revision checkpoint = checkpoints.getOldestRevisionToKeep();
- if (checkpoint != null && scope.endsAfter(checkpoint.getTimestamp())) {
- TimeInterval minimalScope = scope.startAndDuration(options.precisionMs);
- if (minimalScope.endsAfter(checkpoint.getTimestamp())) {
- log.warn("Ignoring RGC run because a valid checkpoint [{}] exists inside minimal scope {}.",
- checkpoint.toReadableString(), minimalScope);
- ignoreDueToCheckPoint = true;
- } else {
- scope = scope.notLaterThan(checkpoint.getTimestamp() - 1);
- log.debug("checkpoint at [{}] found, scope now {}", Utils.timestampToString(checkpoint.getTimestamp()), scope);
- }
- }
+
+ final GCResult gcResult = getResult(options, checkpoint, clock, RGC, scope);
+ scope = gcResult.gcScope;
+ ignoreDueToCheckPoint = gcResult.ignoreGC;
+
+ final GCResult fullGCResult = getResult(options, checkpoint, clock, DGC, scopeFullGC);
+ scopeFullGC = fullGCResult.gcScope;
+ ignoreFullGCDueToCheckPoint = fullGCResult.ignoreGC;
if (scope.getDurationMs() <= options.precisionMs) {
// If we have narrowed the collect time interval down as much as we can, no
@@ -159,7 +243,11 @@ public class VersionGCRecommendations {
this.precisionMs = options.precisionMs;
this.ignoreDueToCheckPoint = ignoreDueToCheckPoint;
this.scope = scope;
+ this.ignoreFullGCDueToCheckPoint = ignoreFullGCDueToCheckPoint;
+ this.scopeFullGC = scopeFullGC;
+ this.fullGCId = isFullGCDryRun ? oldestModifiedDryRunDocId : oldestModifiedDocId;
this.scopeIsComplete = scope.toMs >= keep.fromMs;
+ this.fullGCScopeIsComplete = scopeFullGC.toMs >= keep.fromMs;
this.maxCollect = collectLimit;
this.suggestedIntervalMs = suggestedIntervalMs;
this.deleteCandidateCount = deletedOnceCount;
@@ -173,16 +261,18 @@ public class VersionGCRecommendations {
* @param stats the statistics from the last run
*/
public void evaluate(VersionGCStats stats) {
- if (stats.limitExceeded) {
+ if (stats.limitExceeded && !isFullGCDryRun) {
// if the limit was exceeded, slash the recommended interval in half.
long nextDuration = Math.max(precisionMs, scope.getDurationMs() / 2);
gcmon.info("Limit {} documents exceeded, reducing next collection interval to {} seconds",
this.maxCollect, TimeUnit.MILLISECONDS.toSeconds(nextDuration));
setLongSetting(VersionGarbageCollector.SETTINGS_COLLECTION_REC_INTERVAL_PROP, nextDuration);
stats.needRepeat = true;
- } else if (!stats.canceled && !stats.ignoredGCDueToCheckPoint) {
+ } else if (!stats.canceled && !stats.ignoredGCDueToCheckPoint && !isFullGCDryRun) {
// success, we would not expect to encounter revisions older than this in the future
- setLongSetting(VersionGarbageCollector.SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP, scope.toMs);
+ setLongSetting(of(SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP, scope.toMs,
+ SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP, stats.oldestModifiedDocTimeStamp));
+ setStringSetting(SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP, stats.oldestModifiedDocId);
int count = stats.deletedDocGCCount - stats.deletedLeafDocGCCount;
double usedFraction;
@@ -209,28 +299,100 @@ public class VersionGCRecommendations {
}
stats.needRepeat = !scopeIsComplete;
}
+
+ // save data for full GC
+ if (fullGCEnabled && !stats.canceled && !stats.ignoredFullGCDueToCheckPoint) {
+ // success, we would not expect to encounter revisions older than this in the future
+ if (isFullGCDryRun) {
+ setLongSetting(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_TIMESTAMP_PROP, stats.oldestModifiedDocTimeStamp);
+ setStringSetting(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_DOCUMENT_ID_PROP, stats.oldestModifiedDocId);
+ } else {
+ setLongSetting(SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP, stats.oldestModifiedDocTimeStamp);
+ setStringSetting(SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP, stats.oldestModifiedDocId);
+ }
+
+ final long scopeEnd = scopeFullGC.toMs;
+ final long actualEnd = stats.oldestModifiedDocTimeStamp;
+ if (actualEnd < scopeEnd) {
+ stats.needRepeat = true;
+ } else {
+ stats.needRepeat |= !fullGCScopeIsComplete;
+ }
+ }
}
- private Map<String, Long> getLongSettings() {
- Document versionGCDoc = vgc.getDocumentStore().find(Collection.SETTINGS, VersionGarbageCollector.SETTINGS_COLLECTION_ID, 0);
- Map<String, Long> settings = Maps.newHashMap();
+ private Map<String, Object> getVGCSettings() {
+ Document versionGCDoc = vgc.getDocumentStore().find(Collection.SETTINGS, SETTINGS_COLLECTION_ID, 0);
+ Map<String, Object> settings = new HashMap<>();
// default values
- settings.put(VersionGarbageCollector.SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP, 0L);
- settings.put(VersionGarbageCollector.SETTINGS_COLLECTION_REC_INTERVAL_PROP, 0L);
+ settings.put(SETTINGS_COLLECTION_OLDEST_TIMESTAMP_PROP, 0L);
+ settings.put(SETTINGS_COLLECTION_REC_INTERVAL_PROP, 0L);
+ settings.put(SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP, 0L);
+ settings.put(SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP, MIN_ID_VALUE);
+ settings.put(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_TIMESTAMP_PROP, 0L);
+ settings.put(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_DOCUMENT_ID_PROP, MIN_ID_VALUE);
if (versionGCDoc != null) {
for (String k : versionGCDoc.keySet()) {
Object value = versionGCDoc.get(k);
if (value instanceof Number) {
settings.put(k, ((Number) value).longValue());
}
+ if (value instanceof String) {
+ settings.put(k, value);
+ }
}
}
return settings;
}
private void setLongSetting(String propName, long val) {
- UpdateOp updateOp = new UpdateOp(VersionGarbageCollector.SETTINGS_COLLECTION_ID, true);
+ setLongSetting(of(propName, val));
+ }
+
+ private void setStringSetting(String propName, String val) {
+ UpdateOp updateOp = new UpdateOp(SETTINGS_COLLECTION_ID, true);
updateOp.set(propName, val);
vgc.getDocumentStore().createOrUpdate(Collection.SETTINGS, updateOp);
}
+
+ private void setLongSetting(final Map<String, Long> propValMap) {
+ UpdateOp updateOp = new UpdateOp(SETTINGS_COLLECTION_ID, true);
+ propValMap.forEach(updateOp::set);
+ vgc.getDocumentStore().createOrUpdate(Collection.SETTINGS, updateOp);
+ }
+
+ @NotNull
+ private static GCResult getResult(final VersionGCOptions options, final Revision checkpoint, final Clock clock, final GcType gcType,
+ TimeInterval gcScope) {
+ boolean ignoreGC = false;
+ if (checkpoint != null && gcScope.endsAfter(checkpoint.getTimestamp())) {
+ TimeInterval minimalScope = gcScope.startAndDuration(options.precisionMs);
+ if (minimalScope.endsAfter(checkpoint.getTimestamp())) {
+ final long now = clock.getTime();
+ if (now - lastIgnoreWarning > IGNORED_GC_WARNING_INTERVAL_MS) {
+ log.warn("Ignoring [{}] run because a valid checkpoint [{}] exists inside minimal scope {}.", gcType, checkpoint.toReadableString(), minimalScope);
+ lastIgnoreWarning = now;
+ }
+ ignoreGC = true;
+ } else {
+ gcScope = gcScope.notLaterThan(checkpoint.getTimestamp() - 1);
+ log.debug("checkpoint at [{}] found, [{}] Scope now {}", timestampToString(checkpoint.getTimestamp()), gcType, gcScope);
+ }
+ }
+ return new GCResult(ignoreGC, gcScope);
+ }
+
+ private static class GCResult {
+ public final boolean ignoreGC;
+ public final TimeInterval gcScope;
+
+ public GCResult(boolean ignoreGC, TimeInterval gcScope) {
+ this.ignoreGC = ignoreGC;
+ this.gcScope = gcScope;
+ }
+ }
+
+ enum GcType {
+ RGC, DGC
+ }
}
\ No newline at end of file
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupport.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupport.java
index 13171b7fd5..b938da536a 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupport.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupport.java
@@ -19,12 +19,27 @@
package org.apache.jackrabbit.oak.plugins.document;
+import static java.util.Comparator.comparing;
+import static java.util.Optional.empty;
+import static java.util.Optional.of;
+import static java.util.Optional.ofNullable;
+import static java.util.stream.Stream.concat;
+import static java.util.stream.StreamSupport.stream;
import static org.apache.jackrabbit.guava.common.collect.Iterables.filter;
+import static java.util.stream.Collectors.toList;
+import static org.apache.jackrabbit.oak.plugins.document.Document.ID;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MIN_ID_VALUE;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MODIFIED_IN_SECS;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.getModifiedInSecs;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getAllDocuments;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getSelectedDocuments;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
+import java.util.stream.Stream;
import org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType;
import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCStats;
@@ -59,28 +74,66 @@ public class VersionGCSupport {
* @param toModified the upper bound modified timestamp (exclusive)
* @return matching documents.
*/
- public Iterable<NodeDocument> getPossiblyDeletedDocs(final long fromModified,
- final long toModified) {
- return filter(getSelectedDocuments(store, NodeDocument.DELETED_ONCE, 1), new Predicate<NodeDocument>() {
- @Override
- public boolean apply(NodeDocument input) {
- return input.wasDeletedOnce()
- && modifiedGreaterThanEquals(input, fromModified)
- && modifiedLessThan(input, toModified);
- }
+ public Iterable<NodeDocument> getPossiblyDeletedDocs(final long fromModified, final long toModified) {
+ return stream(getSelectedDocuments(store, NodeDocument.DELETED_ONCE, 1).spliterator(), false)
+ .filter(input -> input.wasDeletedOnce() && modifiedGreaterThanEquals(input, fromModified) && modifiedLessThan(input, toModified))
+ .collect(toList());
+ }
- private boolean modifiedGreaterThanEquals(NodeDocument doc,
- long time) {
- Long modified = doc.getModified();
- return modified != null && modified.compareTo(getModifiedInSecs(time)) >= 0;
- }
+ /**
+ * Returns documents that have a {@link NodeDocument#MODIFIED_IN_SECS} value
+ * within the given range and are greater than given @{@link NodeDocument#ID}.
+ * <p>
+ * The two passed modified timestamps are in milliseconds
+ * since the epoch and the implementation will convert them to seconds at
+ * the granularity of the {@link NodeDocument#MODIFIED_IN_SECS} field and
+ * then perform the comparison.
+ * <p/>
+ *
+ *
+ * @param fromModified the lower bound modified timestamp (inclusive)
+ * @param toModified the upper bound modified timestamp (exclusive)
+ * @param limit the limit of documents to return
+ * @param fromId the lower bound {@link NodeDocument#ID}
+ * @return matching documents.
+ */
+ public Iterable<NodeDocument> getModifiedDocs(final long fromModified, final long toModified, final int limit,
+ @NotNull final String fromId) {
+ // (_modified = fromModified && _id > fromId || _modified > fromModified && _modified < toModified)
+ final Stream<NodeDocument> s1 = stream(getSelectedDocuments(store, MODIFIED_IN_SECS, 1, fromId).spliterator(), false)
+ .filter(input -> modifiedEqualsTo(input, fromModified));
- private boolean modifiedLessThan(NodeDocument doc,
- long time) {
- Long modified = doc.getModified();
- return modified != null && modified.compareTo(getModifiedInSecs(time)) < 0;
- }
- });
+ final Stream<NodeDocument> s2 = stream(getSelectedDocuments(store, MODIFIED_IN_SECS, 1).spliterator(), false)
+ .filter(input -> modifiedGreaterThan(input, fromModified) && modifiedLessThan(input, toModified));
+
+ return concat(s1, s2)
+ .sorted((o1, o2) -> comparing(NodeDocument::getModified).thenComparing(Document::getId).compare(o1, o2))
+ .limit(limit)
+ .collect(toList());
+ }
+
+ private boolean modifiedGreaterThanEquals(final NodeDocument doc, final long time) {
+ Long modified = doc.getModified();
+ return modified != null && modified.compareTo(getModifiedInSecs(time)) >= 0;
+ }
+
+ private boolean modifiedGreaterThan(final NodeDocument doc, final long time) {
+ Long modified = doc.getModified();
+ return modified != null && modified.compareTo(getModifiedInSecs(time)) > 0;
+ }
+
+ private boolean modifiedEqualsTo(final NodeDocument doc, final long time) {
+ Long modified = doc.getModified();
+ return modified != null && modified.compareTo(getModifiedInSecs(time)) == 0;
+ }
+
+ private boolean modifiedLessThan(final NodeDocument doc, final long time) {
+ Long modified = doc.getModified();
+ return modified != null && modified.compareTo(getModifiedInSecs(time)) < 0;
+ }
+
+ private boolean idEquals(final NodeDocument doc, final String id) {
+ return Objects.equals(doc.getId(), id);
}
/**
@@ -160,6 +213,70 @@ public class VersionGCSupport {
return ts;
}
+ /**
+ * Retrieve the oldest modified document.
+ *
+ * @return the oldest modified document.
+ */
+ public Optional<NodeDocument> getOldestModifiedDoc(final Clock clock) {
+ long now = clock.getTime();
+ Iterable<NodeDocument> docs = null;
+ try {
+ docs = getModifiedDocs(0, now, 1, MIN_ID_VALUE);
+ if (docs.iterator().hasNext()) {
+ final NodeDocument oldestModifiedDoc = docs.iterator().next();
+ LOG.info("Oldest modified document is {}", oldestModifiedDoc);
+ return ofNullable(oldestModifiedDoc);
+ }
+ } finally {
+ Utils.closeIfCloseable(docs);
+ }
+ LOG.info("No Modified Doc has been found, retuning empty");
+ return empty();
+ }
+
+ /**
+ * Retrieves a document with the given id from the DocumentStore.
+ * If a list of fields is provided, only these fields are included in the returned document.
+ *
+ * @param id the id of the document to retrieve
+ * @param fields the list of fields to include in the returned document. If null or empty, all fields are returned.
+ * @return an Optional that contains the requested NodeDocument if it exists, or an empty Optional if it does not.
+ */
+ public Optional<NodeDocument> getDocument(final String id, final List<String> fields) {
+
+ Iterable<NodeDocument> docs = null;
+ try {
+ docs = stream(getSelectedDocuments(store, null, 0, MIN_ID_VALUE).spliterator(), false)
+ .filter(input -> idEquals(input, id)).limit(1).collect(toList());
+ if (docs.iterator().hasNext()) {
+ final NodeDocument doc = docs.iterator().next();
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Found Document with id {}", id);
+ }
+ if (fields == null || fields.isEmpty()) {
+ return ofNullable(doc);
+ }
+
+ final Set<String> projectedSet = new HashSet<>(fields);
+ projectedSet.add(ID);
+
+ final NodeDocument newDoc = Collection.NODES.newDocument(store);
+ doc.deepCopy(newDoc);
+ newDoc.keySet().retainAll(projectedSet);
+ return of(newDoc);
+ }
+
+ } finally {
+ Utils.closeIfCloseable(docs);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("No Doc has been found with id [{}]", id);
+ }
+ return empty();
+ }
+
public long getDeletedOnceCount() throws UnsupportedOperationException {
throw new UnsupportedOperationException("getDeletedOnceCount()");
}
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
index ed0b333d44..c26e433c0e 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
@@ -21,15 +21,24 @@ package org.apache.jackrabbit.oak.plugins.document;
import java.io.Closeable;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
import java.util.Set;
+import java.util.SortedMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
import org.apache.jackrabbit.guava.common.base.Function;
import org.apache.jackrabbit.guava.common.base.Joiner;
@@ -42,30 +51,64 @@ import org.apache.jackrabbit.guava.common.collect.Maps;
import org.apache.jackrabbit.guava.common.collect.Sets;
import org.apache.jackrabbit.oak.commons.sort.StringSort;
+import org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType;
+import org.apache.jackrabbit.oak.plugins.document.UpdateOp.Key;
+import org.apache.jackrabbit.oak.plugins.document.UpdateOp.Operation;
+import org.apache.jackrabbit.oak.plugins.document.UpdateOp.Operation.Type;
import org.apache.jackrabbit.oak.plugins.document.util.TimeInterval;
import org.apache.jackrabbit.oak.plugins.document.util.Utils;
import org.apache.jackrabbit.oak.spi.gc.DelegatingGCMonitor;
import org.apache.jackrabbit.oak.spi.gc.GCMonitor;
+import org.apache.jackrabbit.oak.spi.state.AbstractNodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.stats.Clock;
+import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.commons.TimeDurationFormatter;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import static org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull;
+import static java.lang.Math.round;
+import static java.lang.String.join;
+import static java.util.Collections.emptySet;
+import static java.util.List.of;
+import static java.util.Objects.nonNull;
+import static java.util.Objects.requireNonNull;
+import static java.util.Optional.ofNullable;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static java.util.function.Function.identity;
+import static java.util.stream.Collectors.joining;
+import static java.util.stream.Collectors.toMap;
+import static java.util.stream.Collectors.toSet;
+import static java.util.stream.StreamSupport.stream;
import static org.apache.jackrabbit.guava.common.base.StandardSystemProperty.LINE_SEPARATOR;
+import static org.apache.jackrabbit.guava.common.base.Stopwatch.createUnstarted;
import static org.apache.jackrabbit.guava.common.collect.Iterables.all;
import static org.apache.jackrabbit.guava.common.collect.Iterators.partition;
import static org.apache.jackrabbit.guava.common.util.concurrent.Atomics.newReference;
import static java.util.concurrent.TimeUnit.MICROSECONDS;
import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
import static org.apache.jackrabbit.oak.plugins.document.Collection.SETTINGS;
+import static org.apache.jackrabbit.oak.plugins.document.Document.ID;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.BRANCH_COMMITS;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.COLLISIONS;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.COMMIT_ROOT;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MIN_ID_VALUE;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MODIFIED_IN_SECS;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.REVISIONS;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType.COMMIT_ROOT_ONLY;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType.DEFAULT_LEAF;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType.DEFAULT_NO_BRANCH;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.FullGCMode.GAP_ORPHANS;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.FullGCMode.GAP_ORPHANS_EMPTYPROPS;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.FullGCMode.NONE;
+import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getIdFromPath;
+import static org.apache.jackrabbit.oak.plugins.document.util.Utils.timestampToString;
+import static org.apache.jackrabbit.oak.stats.StatisticsProvider.NOOP;
+import static org.apache.jackrabbit.oak.plugins.document.util.Utils.isCommitted;
+import static org.slf4j.LoggerFactory.getLogger;
import static org.slf4j.helpers.MessageFormatter.arrayFormat;
public class VersionGarbageCollector {
@@ -74,9 +117,12 @@ public class VersionGarbageCollector {
private static final int DELETE_BATCH_SIZE = 450;
private static final int UPDATE_BATCH_SIZE = 450;
private static final int PROGRESS_BATCH_SIZE = 10000;
+ private static final int FULL_GC_BATCH_SIZE = 1000;
+ private static final int FULL_GC_MISSING_DOCS_TYPE_CACHE_SIZE = 64;
private static final String STATUS_IDLE = "IDLE";
private static final String STATUS_INITIALIZING = "INITIALIZING";
- private static final Logger log = LoggerFactory.getLogger(VersionGarbageCollector.class);
+ private static final Logger log = getLogger(VersionGarbageCollector.class);
+ private static final Logger AUDIT_LOG = getLogger(VersionGarbageCollector.class.getName() + ".auditDGC");
/**
* Split document types which can be safely garbage collected
@@ -99,24 +145,102 @@ public class VersionGarbageCollector {
*/
static final String SETTINGS_COLLECTION_REC_INTERVAL_PROP = "recommendedIntervalMs";
+ /**
+ * Property name to timestamp till when last full-GC run happened
+ */
+ static final String SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP = "fullGCTimeStamp";
+
+ /**
+ * Property name to _id till when last full-GC run happened
+ */
+ static final String SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP = "fullGCId";
+
+ /**
+ * Property name to timestamp till when last full-GC run happened in dryRun mode only
+ */
+ static final String SETTINGS_COLLECTION_FULL_GC_DRY_RUN_TIMESTAMP_PROP = "fullGCDryRunTimeStamp";
+
+ /**
+ * Property name to _id till when last full-GC run happened in dryRun mode only
+ */
+ static final String SETTINGS_COLLECTION_FULL_GC_DRY_RUN_DOCUMENT_ID_PROP = "fullGCDryRunId";
+
+ /**
+ * During hardening of FullGC one can choose level type of garbage should be cleaned up.
+ * Ultimately the goal is to clean up all possible garbage. After hardening these modes
+ * might no longer be supported.
+ */
+ enum FullGCMode {
+ /** no full GC is done at all */
+ NONE,
+ /** GC only orphaned nodes with gaps in ancestor docs */
+ GAP_ORPHANS,
+ /** GC orphaned nodes with gaps in ancestor docs, plus empty properties */
+ GAP_ORPHANS_EMPTYPROPS,
+ /** GC any kind of orphaned nodes, plus empty properties */
+ ALL_ORPHANS_EMPTYPROPS,
+ /**
+ * GC any kind of orphaned nodes, empty properties plus keep 1 (== keep
+ * traversed) revision, applied to user properties only
+ */
+ ORPHANS_EMPTYPROPS_KEEP_ONE_USER_PROPS,
+ /**
+ * GC any kind of orphaned nodes, empty properties plus keep 1 (== keep
+ * traversed) revision, applied to all properties
+ */
+ ORPHANS_EMPTYPROPS_KEEP_ONE_ALL_PROPS,
+ /**
+ * GC any kind of orphaned nodes, empty properties plus cleanup unmerged BCs
+ */
+ ORPHANS_EMPTYPROPS_UNMERGED_BC,
+ /**
+ * GC any kind of orphaned nodes, empty properties plus cleanup revisions, also
+ * between checkpoints
+ */
+ ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_NO_UNMERGED_BC,
+ /**
+ * GC any kind of orphaned nodes, empty properties, cleanup revisions, also
+ * between checkpoints, plus cleanup unmerged BCs
+ */
+ ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_WITH_UNMERGED_BC
+ }
+
+ private static FullGCMode fullGcMode = GAP_ORPHANS_EMPTYPROPS;
+
+ static FullGCMode getFullGcMode() {
+ return fullGcMode;
+ }
+
private final DocumentNodeStore nodeStore;
private final DocumentStore ds;
+ private final boolean fullGCEnabled;
+ private final boolean isFullGCDryRun;
+ private final boolean embeddedVerification;
private final VersionGCSupport versionStore;
private final AtomicReference<GCJob> collector = newReference();
private VersionGCOptions options;
private GCMonitor gcMonitor = GCMonitor.EMPTY;
- private RevisionGCStats gcStats = new RevisionGCStats(StatisticsProvider.NOOP);
+ private RevisionGCStats gcStats = new RevisionGCStats(NOOP);
+ private FullGCStatsCollector fullGCStats = new FullGCStatsCollectorImpl(NOOP);
VersionGarbageCollector(DocumentNodeStore nodeStore,
- VersionGCSupport gcSupport) {
+ VersionGCSupport gcSupport,
+ final boolean fullGCEnabled,
+ final boolean isFullGCDryRun,
+ final boolean embeddedVerification) {
this.nodeStore = nodeStore;
this.versionStore = gcSupport;
this.ds = gcSupport.getDocumentStore();
+ this.fullGCEnabled = fullGCEnabled;
+ this.isFullGCDryRun = isFullGCDryRun;
+ this.embeddedVerification = embeddedVerification;
this.options = new VersionGCOptions();
+ AUDIT_LOG.info("<init> VersionGarbageCollector created with fullGcMode = {}", fullGcMode);
}
void setStatisticsProvider(StatisticsProvider provider) {
this.gcStats = new RevisionGCStats(provider);
+ this.fullGCStats = new FullGCStatsCollectorImpl(provider);
}
@NotNull
@@ -135,6 +259,9 @@ public class VersionGarbageCollector {
VersionGCStats overall = new VersionGCStats();
overall.active.start();
gcStats.started();
+ if (fullGCEnabled) {
+ fullGCStats.started();
+ }
boolean success = false;
try {
long averageDurationMs = 0;
@@ -160,6 +287,9 @@ public class VersionGarbageCollector {
collector.set(null);
overall.success = success;
gcStats.finished(overall);
+ if (fullGCEnabled) {
+ fullGCStats.finished(overall);
+ }
if (overall.iterationCount > 1) {
gcMonitor.info("Revision garbage collection finished after {} iterations - aggregate statistics: {}",
overall.iterationCount, overall);
@@ -187,7 +317,7 @@ public class VersionGarbageCollector {
}
public void setGCMonitor(@NotNull GCMonitor gcMonitor) {
- this.gcMonitor = checkNotNull(gcMonitor);
+ this.gcMonitor = requireNonNull(gcMonitor);
}
public VersionGCOptions getOptions() {
@@ -202,20 +332,34 @@ public class VersionGarbageCollector {
ds.remove(SETTINGS, SETTINGS_COLLECTION_ID);
}
+ public void resetFullGC() {
+ UpdateOp op = new UpdateOp(SETTINGS_COLLECTION_ID, false);
+ op.remove(SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP);
+ op.remove(SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP);
+ ds.findAndUpdate(SETTINGS, op);
+ }
+
+ public void resetDryRun() {
+ UpdateOp op = new UpdateOp(SETTINGS_COLLECTION_ID, false);
+ op.remove(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_TIMESTAMP_PROP);
+ op.remove(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_DOCUMENT_ID_PROP);
+ ds.findAndUpdate(SETTINGS, op);
+ }
+
public VersionGCInfo getInfo(long maxRevisionAge, TimeUnit unit)
throws IOException {
long maxRevisionAgeInMillis = unit.toMillis(maxRevisionAge);
long now = nodeStore.getClock().getTime();
VersionGCRecommendations rec = new VersionGCRecommendations(maxRevisionAgeInMillis, nodeStore.getCheckpoints(),
- nodeStore.getClock(), versionStore, options, gcMonitor);
+ nodeStore.getClock(), versionStore, options, gcMonitor, fullGCEnabled, isFullGCDryRun);
int estimatedIterations = -1;
if (rec.suggestedIntervalMs > 0) {
- estimatedIterations = (int)Math.ceil(
- (now - rec.scope.toMs) / rec.suggestedIntervalMs);
+ estimatedIterations = (int)Math.ceil((double) (now - rec.scope.toMs) / rec.suggestedIntervalMs);
}
return new VersionGCInfo(rec.lastOldestTimestamp, rec.scope.fromMs,
rec.deleteCandidateCount, rec.maxCollect,
- rec.suggestedIntervalMs, rec.scope.toMs, estimatedIterations);
+ rec.suggestedIntervalMs, rec.scope.toMs, estimatedIterations,
+ rec.scopeFullGC.fromMs);
}
public static class VersionGCInfo {
@@ -226,6 +370,7 @@ public class VersionGarbageCollector {
public final long recommendedCleanupInterval;
public final long recommendedCleanupTimestamp;
public final int estimatedIterations;
+ public final long oldestFullGCRevisionEstimate;
VersionGCInfo(long lastSuccess,
long oldestRevisionEstimate,
@@ -233,7 +378,8 @@ public class VersionGarbageCollector {
long collectLimit,
long recommendedCleanupInterval,
long recommendedCleanupTimestamp,
- int estimatedIterations) {
+ int estimatedIterations,
+ long oldestFullGCRevisionEstimate) {
this.lastSuccess = lastSuccess;
this.oldestRevisionEstimate = oldestRevisionEstimate;
this.revisionsCandidateCount = revisionsCandidateCount;
@@ -241,11 +387,14 @@ public class VersionGarbageCollector {
this.recommendedCleanupInterval = recommendedCleanupInterval;
this.recommendedCleanupTimestamp = recommendedCleanupTimestamp;
this.estimatedIterations = estimatedIterations;
+ this.oldestFullGCRevisionEstimate = oldestFullGCRevisionEstimate;
}
}
public static class VersionGCStats {
boolean ignoredGCDueToCheckPoint;
+ boolean fullGCDryRunMode;
+ boolean ignoredFullGCDueToCheckPoint;
boolean canceled;
boolean success = true;
boolean limitExceeded;
@@ -256,8 +405,19 @@ public class VersionGarbageCollector {
int splitDocGCCount;
int intermediateSplitDocGCCount;
int updateResurrectedGCCount;
+ long oldestModifiedDocTimeStamp;
+ String oldestModifiedDocId;
+ int updatedFullGCDocsCount;
+ int skippedFullGCDocsCount;
+ int deletedPropsCount;
+ int deletedInternalPropsCount;
+ int deletedPropRevsCount;
+ int deletedInternalPropRevsCount;
+ int deletedUnmergedBCCount;
+ int deletedOrphanNodesCount;
final TimeDurationFormatter df = TimeDurationFormatter.forLogging();
final Stopwatch active = Stopwatch.createUnstarted();
+ final Stopwatch fullGCActive = Stopwatch.createUnstarted();
final Stopwatch collectDeletedDocs = Stopwatch.createUnstarted();
final Stopwatch checkDeletedDocs = Stopwatch.createUnstarted();
final Stopwatch deleteDeletedDocs = Stopwatch.createUnstarted();
@@ -265,13 +425,23 @@ public class VersionGarbageCollector {
final Stopwatch deleteSplitDocs = Stopwatch.createUnstarted();
final Stopwatch sortDocIds = Stopwatch.createUnstarted();
final Stopwatch updateResurrectedDocuments = Stopwatch.createUnstarted();
- long activeElapsed, collectDeletedDocsElapsed, checkDeletedDocsElapsed, deleteDeletedDocsElapsed, collectAndDeleteSplitDocsElapsed,
- deleteSplitDocsElapsed, sortDocIdsElapsed, updateResurrectedDocumentsElapsed;
+ final Stopwatch fullGCDocs = Stopwatch.createUnstarted();
+ final Stopwatch deleteFullGCDocs = Stopwatch.createUnstarted();
+ final Stopwatch collectFullGC = Stopwatch.createUnstarted();
+ final Stopwatch collectOrphanNodes = Stopwatch.createUnstarted();
+ final Stopwatch collectDeletedProps = Stopwatch.createUnstarted();
+ final Stopwatch collectDeletedOldRevs = Stopwatch.createUnstarted();
+ final Stopwatch collectUnmergedBC = Stopwatch.createUnstarted();
+ long activeElapsed, fullGCActiveElapsed, collectDeletedDocsElapsed, checkDeletedDocsElapsed, deleteDeletedDocsElapsed,
+ collectAndDeleteSplitDocsElapsed, deleteSplitDocsElapsed, sortDocIdsElapsed, updateResurrectedDocumentsElapsed,
+ fullGCDocsElapsed, collectFullGCElapsed, collectOrphanNodesElapsed, collectDeletedPropsElapsed,
+ deleteFullGCDocsElapsed, collectDeletedOldRevsElapsed, collectUnmergedBCElapsed;
@Override
public String toString() {
String timings;
- String fmt = "timeToCollectDeletedDocs=%s, timeToCheckDeletedDocs=%s, timeToSortDocIds=%s, timeTakenToUpdateResurrectedDocs=%s, timeTakenToDeleteDeletedDocs=%s, timeTakenToCollectAndDeleteSplitDocs=%s%s";
+ String fmt = "timeToCollectDeletedDocs=%s, timeToCheckDeletedDocs=%s, timeToSortDocIds=%s, timeTakenToUpdateResurrectedDocs=%s, timeTakenToDeleteDeletedDocs=%s, timeTakenToCollectAndDeleteSplitDocs=%s%s, " +
+ "timeToRunFullGC=%s, which includes [timeToDeleteFullGC=%s and timeToCollectFullGC=%s, (of which timeToCollectOrphanNodes=%s timeToCollectDeletedProps=%s, timeToCollectOldRevs=%s, timeToCollectUnmergedBranchCommits=%s)]";
// aggregated timings?
if (iterationCount > 0) {
@@ -285,7 +455,14 @@ public class VersionGarbageCollector {
df.format(updateResurrectedDocumentsElapsed, MICROSECONDS),
df.format(deleteDeletedDocsElapsed, MICROSECONDS),
df.format(collectAndDeleteSplitDocsElapsed, MICROSECONDS),
- timeDeletingSplitDocs);
+ timeDeletingSplitDocs,
+ df.format(fullGCDocsElapsed, MICROSECONDS),
+ df.format(deleteFullGCDocsElapsed, MICROSECONDS),
+ df.format(collectFullGCElapsed, MICROSECONDS),
+ df.format(collectOrphanNodesElapsed, MICROSECONDS),
+ df.format(collectDeletedPropsElapsed, MICROSECONDS),
+ df.format(collectDeletedOldRevsElapsed, MICROSECONDS),
+ df.format(collectUnmergedBCElapsed, MICROSECONDS));
} else {
String timeDeletingSplitDocs = "";
if (deleteSplitDocs.elapsed(MICROSECONDS) > 0) {
@@ -298,17 +475,37 @@ public class VersionGarbageCollector {
df.format(updateResurrectedDocuments.elapsed(MICROSECONDS), MICROSECONDS),
df.format(deleteDeletedDocs.elapsed(MICROSECONDS), MICROSECONDS),
df.format(collectAndDeleteSplitDocs.elapsed(MICROSECONDS), MICROSECONDS),
- timeDeletingSplitDocs);
+ timeDeletingSplitDocs,
+ df.format(fullGCDocs.elapsed(MICROSECONDS), MICROSECONDS),
+ df.format(deleteFullGCDocs.elapsed(MICROSECONDS), MICROSECONDS),
+ df.format(collectFullGC.elapsed(MICROSECONDS), MICROSECONDS),
+ df.format(collectOrphanNodes.elapsed(MICROSECONDS), MICROSECONDS),
+ df.format(collectDeletedProps.elapsed(MICROSECONDS), MICROSECONDS),
+ df.format(collectDeletedOldRevs.elapsed(MICROSECONDS), MICROSECONDS),
+ df.format(collectUnmergedBC.elapsed(MICROSECONDS), MICROSECONDS));
}
return "VersionGCStats{" +
"ignoredGCDueToCheckPoint=" + ignoredGCDueToCheckPoint +
+ ", fullGCDryRunMode=" + fullGCDryRunMode +
+ ", ignoredFullGCDueToCheckPoint=" + ignoredFullGCDueToCheckPoint +
", canceled=" + canceled +
", deletedDocGCCount=" + deletedDocGCCount + " (of which leaf: " + deletedLeafDocGCCount + ")" +
", updateResurrectedGCCount=" + updateResurrectedGCCount +
", splitDocGCCount=" + splitDocGCCount +
", intermediateSplitDocGCCount=" + intermediateSplitDocGCCount +
+ ", oldestModifiedDocId=" + oldestModifiedDocId +
+ ", oldestModifiedDocTimeStamp=" + timestampToString(oldestModifiedDocTimeStamp) +
+ ", updatedFullGCDocsCount=" + updatedFullGCDocsCount +
+ ", skippedFullGCDocsCount=" + skippedFullGCDocsCount +
+ ", deletedPropsCount=" + deletedPropsCount +
+ ", deletedInternalPropsCount=" + deletedInternalPropsCount +
+ ", deletedPropRevsCount=" + deletedPropRevsCount +
+ ", deletedInternalPropRevsCount=" + deletedInternalPropRevsCount +
+ ", deletedUnmergedBCCount=" + deletedUnmergedBCCount +
+ ", deletedOrphanNodesCount=" + deletedOrphanNodesCount +
", iterationCount=" + iterationCount +
+ ", timeFullGCActive=" + df.format(fullGCActiveElapsed, MICROSECONDS) +
", timeActive=" + df.format(activeElapsed, MICROSECONDS) +
", " + timings + "}";
}
@@ -316,6 +513,8 @@ public class VersionGarbageCollector {
void addRun(VersionGCStats run) {
++iterationCount;
this.ignoredGCDueToCheckPoint = run.ignoredGCDueToCheckPoint;
+ this.fullGCDryRunMode = run.fullGCDryRunMode;
+ this.ignoredFullGCDueToCheckPoint = run.ignoredFullGCDueToCheckPoint;
this.canceled = run.canceled;
this.success = run.success;
this.limitExceeded = run.limitExceeded;
@@ -325,9 +524,20 @@ public class VersionGarbageCollector {
this.splitDocGCCount += run.splitDocGCCount;
this.intermediateSplitDocGCCount += run.intermediateSplitDocGCCount;
this.updateResurrectedGCCount += run.updateResurrectedGCCount;
+ this.oldestModifiedDocTimeStamp = run.oldestModifiedDocTimeStamp;
+ this.oldestModifiedDocId = run.oldestModifiedDocId;
+ this.updatedFullGCDocsCount += run.updatedFullGCDocsCount;
+ this.skippedFullGCDocsCount += run.skippedFullGCDocsCount;
+ this.deletedPropsCount += run.deletedPropsCount;
+ this.deletedInternalPropsCount += run.deletedInternalPropsCount;
+ this.deletedPropRevsCount += run.deletedPropRevsCount;
+ this.deletedInternalPropRevsCount += run.deletedInternalPropRevsCount;
+ this.deletedUnmergedBCCount += run.deletedUnmergedBCCount;
+ this.deletedOrphanNodesCount += run.deletedOrphanNodesCount;
if (run.iterationCount > 0) {
// run is cumulative with times in elapsed fields
this.activeElapsed += run.activeElapsed;
+ this.fullGCActiveElapsed += run.fullGCActiveElapsed;
this.collectDeletedDocsElapsed += run.collectDeletedDocsElapsed;
this.checkDeletedDocsElapsed += run.checkDeletedDocsElapsed;
this.deleteDeletedDocsElapsed += run.deleteDeletedDocsElapsed;
@@ -335,9 +545,17 @@ public class VersionGarbageCollector {
this.deleteSplitDocsElapsed += run.deleteSplitDocsElapsed;
this.sortDocIdsElapsed += run.sortDocIdsElapsed;
this.updateResurrectedDocumentsElapsed += run.updateResurrectedDocumentsElapsed;
+ this.fullGCDocsElapsed += run.fullGCDocsElapsed;
+ this.deleteFullGCDocsElapsed += run.deleteFullGCDocsElapsed;
+ this.collectFullGCElapsed += run.collectFullGCElapsed;
+ this.collectOrphanNodesElapsed += run.collectOrphanNodesElapsed;
+ this.collectDeletedPropsElapsed += run.collectDeletedPropsElapsed;
+ this.collectDeletedOldRevsElapsed += run.collectDeletedOldRevsElapsed;
+ this.collectUnmergedBCElapsed += run.collectUnmergedBCElapsed;
} else {
// single run -> read from stop watches
this.activeElapsed += run.active.elapsed(MICROSECONDS);
+ this.fullGCActiveElapsed += run.fullGCActive.elapsed(MICROSECONDS);
this.collectDeletedDocsElapsed += run.collectDeletedDocs.elapsed(MICROSECONDS);
this.checkDeletedDocsElapsed += run.checkDeletedDocs.elapsed(MICROSECONDS);
this.deleteDeletedDocsElapsed += run.deleteDeletedDocs.elapsed(MICROSECONDS);
@@ -345,6 +563,13 @@ public class VersionGarbageCollector {
this.deleteSplitDocsElapsed += run.deleteSplitDocs.elapsed(MICROSECONDS);
this.sortDocIdsElapsed += run.sortDocIds.elapsed(MICROSECONDS);
this.updateResurrectedDocumentsElapsed += run.updateResurrectedDocuments.elapsed(MICROSECONDS);
+ this.fullGCDocsElapsed += run.fullGCDocs.elapsed(MICROSECONDS);
+ this.deleteFullGCDocsElapsed += run.deleteFullGCDocs.elapsed(MICROSECONDS);
+ this.collectFullGCElapsed += run.collectFullGC.elapsed(MICROSECONDS);
+ this.collectOrphanNodesElapsed += run.collectOrphanNodes.elapsed(MICROSECONDS);
+ this.collectDeletedPropsElapsed += run.collectDeletedProps.elapsed(MICROSECONDS);
+ this.collectDeletedOldRevsElapsed += run.collectDeletedOldRevs.elapsed(MICROSECONDS);
+ this.collectUnmergedBCElapsed += run.collectUnmergedBC.elapsed(MICROSECONDS);
}
}
}
@@ -356,6 +581,13 @@ public class VersionGarbageCollector {
DELETING,
SORTING,
SPLITS_CLEANUP,
+ FULL_GC,
+ FULL_GC_COLLECT_GARBAGE,
+ FULL_GC_COLLECT_ORPHAN_NODES,
+ FULL_GC_COLLECT_PROPS,
+ FULL_GC_COLLECT_OLD_REVS,
+ FULL_GC_COLLECT_UNMERGED_BC,
+ FULL_GC_CLEANUP,
UPDATING
}
@@ -384,6 +616,13 @@ public class VersionGarbageCollector {
this.watches.put(GCPhase.SORTING, stats.sortDocIds);
this.watches.put(GCPhase.SPLITS_CLEANUP, stats.collectAndDeleteSplitDocs);
this.watches.put(GCPhase.UPDATING, stats.updateResurrectedDocuments);
+ this.watches.put(GCPhase.FULL_GC, stats.fullGCDocs);
+ this.watches.put(GCPhase.FULL_GC_COLLECT_GARBAGE, stats.collectFullGC);
+ this.watches.put(GCPhase.FULL_GC_COLLECT_ORPHAN_NODES, stats.collectOrphanNodes);
+ this.watches.put(GCPhase.FULL_GC_COLLECT_PROPS, stats.collectDeletedProps);
+ this.watches.put(GCPhase.FULL_GC_COLLECT_OLD_REVS, stats.collectDeletedOldRevs);
+ this.watches.put(GCPhase.FULL_GC_COLLECT_UNMERGED_BC, stats.collectUnmergedBC);
+ this.watches.put(GCPhase.FULL_GC_CLEANUP, stats.deleteFullGCDocs);
this.canceled = canceled;
}
@@ -492,21 +731,45 @@ public class VersionGarbageCollector {
VersionGCStats stats = new VersionGCStats();
stats.active.start();
VersionGCRecommendations rec = new VersionGCRecommendations(maxRevisionAgeInMillis, nodeStore.getCheckpoints(),
- nodeStore.getClock(), versionStore, options, gcMonitor);
+ nodeStore.getClock(), versionStore, options, gcMonitor, fullGCEnabled, isFullGCDryRun);
GCPhases phases = new GCPhases(cancel, stats, gcMonitor);
try {
- if (rec.ignoreDueToCheckPoint) {
- phases.stats.ignoredGCDueToCheckPoint = true;
- monitor.skipped("Checkpoint prevented revision garbage collection");
- cancel.set(true);
+ if (!isFullGCDryRun) {
+ // only run if not fullGC dryRun mode
+ if (rec.ignoreDueToCheckPoint) {
+ phases.stats.ignoredGCDueToCheckPoint = true;
+ monitor.skipped("Checkpoint prevented revision garbage collection");
+ } else {
+ final RevisionVector headRevision = nodeStore.getHeadRevision();
+ final RevisionVector sweepRevisions = nodeStore.getSweepRevisions();
+ monitor.info("Looking at revisions in {}", rec.scope);
+
+ collectDeletedDocuments(phases, headRevision, rec);
+ collectSplitDocuments(phases, sweepRevisions, rec);
+ }
} else {
- final RevisionVector headRevision = nodeStore.getHeadRevision();
- final RevisionVector sweepRevisions = nodeStore.getSweepRevisions();
- monitor.info("Looking at revisions in {}", rec.scope);
+ phases.stats.fullGCDryRunMode = true;
+ }
- collectDeletedDocuments(phases, headRevision, rec);
- collectSplitDocuments(phases, sweepRevisions, rec);
+ // now run full GC if enabled
+ if (fullGCEnabled) {
+ stats.fullGCActive.start();
+ if (rec.ignoreFullGCDueToCheckPoint) {
+ phases.stats.ignoredFullGCDueToCheckPoint = true;
+ monitor.skipped("Checkpoint prevented Full garbage collection");
+ } else {
+ final RevisionVector headRevision = nodeStore.getHeadRevision();
+ monitor.info("Looking at revisions in {} for full GC", rec.scopeFullGC);
+ collectFullGC(phases, headRevision, rec);
+ }
}
+
+ if ((!fullGCEnabled || rec.ignoreFullGCDueToCheckPoint) && rec.ignoreDueToCheckPoint) {
+ // cancel if both are stopped by checkpoint
+ // otherwise we need to continue
+ cancel.set(true);
+ }
+
} catch (LimitExceededException ex) {
stats.limitExceeded = true;
} finally {
@@ -517,10 +780,130 @@ public class VersionGarbageCollector {
rec.evaluate(stats);
monitor.info("Revision garbage collection finished in {}. {}",
TimeDurationFormatter.forLogging().format(phases.elapsed.elapsed(MICROSECONDS), MICROSECONDS), stats);
+ if (fullGCEnabled && stats.fullGCActive.isRunning()) {
+ // probably limitExceeded == true case
+ stats.fullGCActive.stop();
+ }
stats.active.stop();
return stats;
}
+ /**
+ * "Full garbage" refers to additional garbage identified as part of OAK-10199
+ * et al: essentially garbage that in earlier versions of Oak were ignored. This
+ * includes: deleted properties, revision information within documents, branch
+ * commit related garbage.
+ * <p>
+ * The "Full garbage" collector can be instructed to do a full repository scan
+ * - or incrementally based on where it last left off. When doing a full
+ * repository scan (but not limited to that), it executes in (small) batches
+ * followed by voluntary paused (aka throttling) to avoid excessive load on the
+ * system. The full repository scan does not have to finish particularly fast,
+ * it is okay that it takes a considerable amount of time.
+ *
+ * @param phases {@link GCPhases}
+ * @param headRevision the current head revision of node store
+ * @param rec {@link VersionGCRecommendations} to recommend GC operation
+ */
+ private void collectFullGC(final GCPhases phases, final RevisionVector headRevision, final VersionGCRecommendations rec) {
+
+ final long oldestModifiedMs = rec.scopeFullGC.fromMs;
+ final long toModifiedMs = rec.scopeFullGC.toMs;
+ final String oldestModifiedDocId = rec.fullGCId;
+
+ int docsTraversed = 0;
+ boolean foundDoc = true;
+ long oldModifiedMs = oldestModifiedMs;
+ final LinkedHashMap<Path, Boolean> missingDocsTypes = new LinkedHashMap<>() {
+ @Override
+ protected boolean removeEldestEntry(Entry<Path, Boolean> eldest) {
+ return size() > FULL_GC_MISSING_DOCS_TYPE_CACHE_SIZE;
+ }
+ };
+
+ try (FullGC gc = new FullGC(headRevision, toModifiedMs, missingDocsTypes, monitor, cancel)) {
+ long fromModifiedMs = oldestModifiedMs;
+ String fromId = ofNullable(oldestModifiedDocId).orElse(MIN_ID_VALUE);
+ NodeDocument lastDoc;
+ if (phases.start(GCPhase.FULL_GC)) {
+ while (foundDoc && fromModifiedMs < toModifiedMs && docsTraversed < PROGRESS_BATCH_SIZE) {
+ // set foundDoc to false to allow exiting the while loop
+ foundDoc = false;
+ lastDoc = null;
+ if (log.isDebugEnabled()) {
+ log.debug("Fetching docs from [{}] to [{}] with Id starting from [{}]", timestampToString(fromModifiedMs), timestampToString(toModifiedMs), fromId);
+ }
+ Iterable<NodeDocument> itr = versionStore.getModifiedDocs(fromModifiedMs, toModifiedMs, FULL_GC_BATCH_SIZE, fromId);
+ try {
+ for (NodeDocument doc : itr) {
+ foundDoc = true;
+ // continue with GC?
+ if (cancel.get()) {
+ foundDoc = false; // to exit while loop as well
+ AUDIT_LOG.info("<Terminating> Received GC cancel call");
+ break;
+ }
+ docsTraversed++;
+ if (docsTraversed % 100 == 0) {
+ monitor.info("Iterated through {} documents so far. {} had Full garbage", docsTraversed, gc.getGarbageCount());
+ }
+
+ lastDoc = doc;
+ // collect the data to delete in next step
+ if (phases.start(GCPhase.FULL_GC_COLLECT_GARBAGE)) {
+ gc.collectGarbage(doc, phases);
+ phases.stop(GCPhase.FULL_GC_COLLECT_GARBAGE);
+ }
+
+ final Long modified = lastDoc.getModified();
+ if (modified == null) {
+ monitor.warn("collectFullGC : document has no _modified property : {}", doc.getId());
+ } else if (SECONDS.toMillis(modified) < fromModifiedMs) {
+ monitor.warn("collectFullGC : document has older _modified than query boundary : {} (from: {}, to: {})",
+ modified, timestampToString(fromModifiedMs), timestampToString(toModifiedMs));
+ }
+ }
+ // now remove the garbage in one go, if any
+ if (gc.hasGarbage() && phases.start(GCPhase.FULL_GC_CLEANUP)) {
+ gc.removeGarbage(phases.stats);
+ phases.stop(GCPhase.FULL_GC_CLEANUP);
+ }
+ if (lastDoc != null) {
+ fromModifiedMs = lastDoc.getModified() == null ? oldModifiedMs : SECONDS.toMillis(lastDoc.getModified());
+ fromId = lastDoc.getId();
+ }
+ } finally {
+ Utils.closeIfCloseable(itr);
+ phases.stats.oldestModifiedDocTimeStamp = fromModifiedMs;
+ phases.stats.oldestModifiedDocId = fromId;
+ oldModifiedMs = fromModifiedMs;
+ if (log.isDebugEnabled()) {
+ log.debug("Fetched docs till [{}] with Id [{}]", timestampToString(fromModifiedMs), fromId);
+ }
+ }
+ // if we didn't find any document i.e. either we are already at last document
+ // of current timeStamp or there is no document for this timeStamp
+ // we need to reset fromId & increment fromModifiedMs and check again
+ if (!foundDoc && !Objects.equals(fromId, MIN_ID_VALUE)) {
+ fromId = MIN_ID_VALUE;
+ fromModifiedMs = fromModifiedMs + SECONDS.toMillis(5);
+ foundDoc = true; // to run while loop again
+ }
+ }
+ phases.stop(GCPhase.FULL_GC);
+ }
+ } finally {
+ if (docsTraversed < PROGRESS_BATCH_SIZE) {
+ // we have traversed all the docs within given time range and nothing is left
+ // lets set oldModifiedDocTimeStamp to upper limit of this cycle
+ phases.stats.oldestModifiedDocTimeStamp = toModifiedMs;
+ phases.stats.oldestModifiedDocId = MIN_ID_VALUE;
+ }
+ }
+ }
+
+
+
private void collectSplitDocuments(GCPhases phases,
RevisionVector sweepRevisions,
VersionGCRecommendations rec) {
@@ -539,8 +922,7 @@ public class VersionGarbageCollector {
VersionGCRecommendations rec)
throws IOException, LimitExceededException {
int docsTraversed = 0;
- DeletedDocsGC gc = new DeletedDocsGC(headRevision, cancel, options, monitor);
- try {
+ try (DeletedDocsGC gc = new DeletedDocsGC(headRevision, cancel, options, monitor)) {
if (phases.start(GCPhase.COLLECTING)) {
Iterable<NodeDocument> itr = versionStore.getPossiblyDeletedDocs(rec.scope.fromMs, rec.scope.toMs);
try {
@@ -605,8 +987,1081 @@ public class VersionGarbageCollector {
gc.updateResurrectedDocuments(phases.stats);
phases.stop(GCPhase.UPDATING);
}
+ }
+ }
+ }
+
+ private class FullGC implements Closeable {
+
+ private final long toModifiedMs;
+ private final GCMonitor monitor;
+ private final AtomicBoolean cancel;
+ private final Stopwatch timer;
+ private final List<UpdateOp> updateOpList;
+
+ /** contains the list of _ids of orphan or deleted documents to be removed in the current batch **/
+ private final Map<String, Long> orphanOrDeletedRemovalMap;
+
+ /** contains the list of _ids of orphan or deleted documents to be removed, mapped to their path */
+ private final Map<String, Path> orphanOrDeletedRemovalPathMap;
+
+ /**
+ * Map of documentId => total no. of deleted properties.
+ * <p>
+ *
+ * The document can be updated or property might get recreated between collecting and deletion phases.
+ * This would lead to document not getting deleted (since now modified date & mod count would have changed)
+ * SO the Bulk API wouldn't update this doc.
+ * <p>
+ * In order to calculate the correct no. of updated documents & deleted properties, we save them in a map
+ */
+ private final Map<String, Integer> deletedPropsCountMap;
+ private final Map<String, Integer> deletedInternalPropsCountMap;
+
+ /**
+ * Map of documentId => total no. of deleted property revisions.
+ * <p>
+ *
+ * The document can be updated between collecting and deletion phases.
+ * This would lead to document not getting deleted (since now modified date & mod count would have changed)
+ * SO the Bulk API wouldn't update this doc.
+ * <p>
+ * In order to calculate the correct no. of updated documents & deleted property revisions, we save them in a map
+ */
+ private final Map<String, Integer> deletedPropRevsCountMap;
+ private final Map<String, Integer> deletedInternalPropRevsCountMap;
+
+ /**
+ * {@link Set} of unmergedBranchCommit Revisions to calculate the no. of unmergedBranchCommits that would be
+ * removed in this iteration of FullGC.
+ */
+ private final Set<Revision> deletedUnmergedBCSet;
+ private int garbageDocsCount;
+ private int totalGarbageDocsCount;
+ private final Revision revisionForModified;
+ private final DocumentNodeState root;
+
+ /** small cache for classification of missing nodes : documents that do not exist vs deleted nodes */
+ private final LinkedHashMap<Path, Boolean> missingDocsTypes;
+
+ public FullGC(@NotNull RevisionVector headRevision, long toModifiedMs,
+ LinkedHashMap<Path, Boolean> missingDocsTypes, @NotNull GCMonitor monitor,
+ @NotNull AtomicBoolean cancel) {
+ this.toModifiedMs = toModifiedMs;
+ this.missingDocsTypes = missingDocsTypes;
+ this.monitor = monitor;
+ this.cancel = cancel;
+ this.updateOpList = new ArrayList<>();
+ this.orphanOrDeletedRemovalMap = new HashMap<>();
+ this.orphanOrDeletedRemovalPathMap = new HashMap<>();
+ this.deletedPropsCountMap = new HashMap<>();
+ this.deletedInternalPropsCountMap = new HashMap<>();
+ this.deletedPropRevsCountMap = new HashMap<>();
+ this.deletedInternalPropRevsCountMap = new HashMap<>();
+ this.deletedUnmergedBCSet = new HashSet<>();
+ this.timer = createUnstarted();
+ // clusterId is not used
+ this.revisionForModified = Revision.newRevision(0);
+ this.root = nodeStore.getRoot(headRevision);
+ }
+
+ public void collectGarbage(final NodeDocument doc, final GCPhases phases) {
+ if (fullGcMode == NONE) {
+ monitor.warn("Skipping FullGC. No Mode has been selected.");
+ return;
+ }
+
+ fullGCStats.documentRead();
+ monitor.info("Collecting Full Garbage for doc [{}]", doc.getId());
+
+ if (AUDIT_LOG.isTraceEnabled()) {
+ AUDIT_LOG.trace("<Collecting> Garbage in doc [{}]", doc.getId());
+ }
+
+ final UpdateOp op = new UpdateOp(requireNonNull(doc.getId()), false);
+
+ // traversed state == state of node at doc.id based on head revision
+ NodeState traversedState = root;
+ Path greatestExistingAncestorOrSelf = root.getPath();
+ for (String name : doc.getPath().elements()) {
+ traversedState = traversedState.getChildNode(name);
+ if (traversedState.exists()) {
+ greatestExistingAncestorOrSelf = new Path(
+ greatestExistingAncestorOrSelf, name);
+ }
+ }
+
+ if (!isDeletedOrOrphanedNode(traversedState, greatestExistingAncestorOrSelf, phases, doc)) {
+ // here the node is not orphaned which means that we can reach the node from root
+ switch(fullGcMode) {
+ case NONE : {
+ // shouldn't be reached
+ return;
+ }
+ case GAP_ORPHANS : {
+ // this mode does neither unusedproprev, nor unmergedBC
+ break;
+ }
+ case GAP_ORPHANS_EMPTYPROPS :
+ case ALL_ORPHANS_EMPTYPROPS : {
+ collectDeletedProperties(doc, phases, op, traversedState);
+ // this mode does neither unusedproprev, nor unmergedBC
+ break;
+ }
+ case ORPHANS_EMPTYPROPS_KEEP_ONE_ALL_PROPS : {
+ collectDeletedProperties(doc, phases, op, traversedState);
+ collectUnusedPropertyRevisions(doc, phases, op, (DocumentNodeState) traversedState, false);
+ combineInternalPropRemovals(doc, op);
+ break;
+ }
+ case ORPHANS_EMPTYPROPS_KEEP_ONE_USER_PROPS : {
+ collectDeletedProperties(doc, phases, op, traversedState);
+ collectUnusedPropertyRevisions(doc, phases, op, (DocumentNodeState) traversedState, true);
+ combineInternalPropRemovals(doc, op);
+ break;
+ }
+ case ORPHANS_EMPTYPROPS_UNMERGED_BC : {
+ collectDeletedProperties(doc, phases, op, traversedState);
+ collectUnmergedBranchCommits(doc, phases, op, toModifiedMs);
+ break;
+ }
+ case ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_WITH_UNMERGED_BC : {
+ collectDeletedProperties(doc, phases, op, traversedState);
+ collectUnmergedBranchCommits(doc, phases, op, toModifiedMs);
+ collectRevisionsOlderThan24hAndBetweenCheckpoints(doc, toModifiedMs, phases, op);
+ break;
+ }
+ case ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_NO_UNMERGED_BC : {
+ collectDeletedProperties(doc, phases, op, traversedState);
+ collectRevisionsOlderThan24hAndBetweenCheckpoints(doc, toModifiedMs, phases, op);
+ break;
+ }
+ }
+ // only add if there are changes for this doc
+ if (op.hasChanges()) {
+ op.equals(MODIFIED_IN_SECS, doc.getModified());
+ garbageDocsCount++;
+ totalGarbageDocsCount++;
+ monitor.info("Collected [{}] garbage count in [{}]", op.getChanges().size(), doc.getId());
+ AUDIT_LOG.info("<Collected> [{}] garbage count in [{}]", op.getChanges().size(), doc.getId());
+ updateOpList.add(op);
+ }
+ }
+ if (log.isTraceEnabled() && op.hasChanges()) {
+ // only log in case of changes & debug level enabled
+ log.trace("UpdateOp for [{}] is [{}]", doc.getId(), op);
+ }
+ }
+
+ private void combineInternalPropRemovals(final NodeDocument doc, final UpdateOp op) {
+ // now for any of the handled system properties (the normal properties would
+ // already be cleaned up by cleanupDeletedProperties), the resulting
+ // sub document could in theory become empty after removing all unmerged branch
+ // commit revisions is done later.
+ // so we need to go through all of them and check if we'd have removed
+ // the entirety - and then, instead of individually remove revisions, just
+ // delete the entire property.
+ if (op.hasChanges()) {
+ final int deletedSystemPropsCount = getSystemRemoveMapEntryCounts(op)
+ .entrySet().stream()
+ .filter(e -> filterEmptyProps(doc, e.getKey(), e.getValue()))
+ .mapToInt(e -> {
+ final String prop = e.getKey();
+ int countBefore = op.getChanges().entrySet().size();
+ boolean removed = op.getChanges().entrySet().removeIf(opEntry -> Objects.equals(prop, opEntry.getKey().getName()));
+ int countAfter = op.getChanges().entrySet().size();
+ if (removed) {
+ if (prop.startsWith("_")) {
+ deletedInternalPropRevsCountMap.merge(doc.getId(), (countAfter - countBefore), Integer::sum);
+ } else {
+ deletedPropRevsCountMap.merge(doc.getId(), (countAfter - countBefore), Integer::sum);
+ }
+ }
+ op.remove(prop);
+ return 1;})
+ .sum();
+
+ // update the deleted properties count Map to calculate the total no. of deleted properties
+ int totalDeletedSystemPropsCount = deletedInternalPropsCountMap.merge(doc.getId(), deletedSystemPropsCount, Integer::sum);
+ if (AUDIT_LOG.isDebugEnabled()) {
+ AUDIT_LOG.debug("<Collected> [{}] internal prop revs in [{}] mode [{}]", totalDeletedSystemPropsCount, doc.getId(), fullGcMode);
+ }
+ }
+ }
+
+ /**
+ * Check if the node represented by the given doc and traversedState is
+ * <i>orphaned</i>. A node is considered orphaned if it does not have a visible
+ * parent node. But from a GC point of view this also includes regular
+ * deletion cases that have not otherwise been deleted already (eg by DeletedDocsGC).
+ *
+ * @param traversedState the state of current doc when we start traversing from root via lastRef
+ * @param greatestExistingAncestorOrSelf ancestor furthest up the tree (==greatest) that exists when traversing states
+ * @param phases GC phases
+ * @param doc current document node to check whether it is orphan or not
+ * @return true if the node is orphaned (and/or can be removed), false
+ * otherwise
+ */
+ private boolean isDeletedOrOrphanedNode(final NodeState traversedState, final Path greatestExistingAncestorOrSelf,
+ final GCPhases phases, final NodeDocument doc) {
+
+ if (!phases.start(GCPhase.FULL_GC_COLLECT_ORPHAN_NODES)) {
+ // gc has been cancelled
+ return false;
+ }
+
+ // several different cases here, but ultimately they all lead back to:
+ boolean isOrphan = !traversedState.exists();
+ // if the node when reading at current headRevision (rather than traversed)
+ // does not exist, then this is rather a regular deletion, nothing special.
+ // that is usually handled in DeletedDocsGC - but if FullGC sees this,
+ // it seems risky to not delete it right away.
+
+ // if the node when reading at current headRevision (rather than traversed)
+ // does exist, then it could be either due to the parent node having
+ // been deleted (true orphan) - or the node itself got late-write-added.
+ // in both of these cases we should now delete it. That's all with above return.
+ if (!isOrphan) {
+ // nothing to do here
+ phases.stop(GCPhase.FULL_GC_COLLECT_ORPHAN_NODES);
+ return false;
+ }
+ if (fullGcMode == GAP_ORPHANS || fullGcMode == GAP_ORPHANS_EMPTYPROPS) {
+ // check the ancestor docs for gaps
+ final Path docPath = doc.getPath();
+ final Path geaChildPath = docPath.getAncestor(docPath.getDepth() - greatestExistingAncestorOrSelf.getDepth() - 1);
+ Boolean missingType = missingDocsTypes.get(geaChildPath);
+ if (missingType == null) {
+ // we don't have it cached yet - so do the potentially expensive find
+ missingType = versionStore.getDocument(getIdFromPath(geaChildPath), of(ID)).isEmpty();
+ missingDocsTypes.put(geaChildPath, missingType);
+ }
+ if (!missingType) {
+ // then it is not a gap orphan
+ // nothing to do here then
+ // (even though somewhere along descendants
+ // there might be a gap, it is too expensive to traverse)
+ phases.stop(GCPhase.FULL_GC_COLLECT_ORPHAN_NODES);
+ // return true here as this is an orphan but we decided not to
+ // GC it - to prevent caller from attempting other GC types
+ return true;
+ }
+ }
+
+ // if this is an orphaned node, all that is needed is its removal
+ garbageDocsCount++;
+ totalGarbageDocsCount++;
+ monitor.info("Deleted orphaned or deleted doc [{}]", doc.getId());
+ orphanOrDeletedRemovalMap.put(doc.getId(), doc.getModified());
+ orphanOrDeletedRemovalPathMap.put(doc.getId(), doc.getPath());
+
+ if (AUDIT_LOG.isDebugEnabled()) {
+ AUDIT_LOG.debug("<Collected> [{}] orphaned node", doc.getId());
+ }
+
+ phases.stop(GCPhase.FULL_GC_COLLECT_ORPHAN_NODES);
+ return true;
+
+ }
+
+ private boolean hasGarbage() {
+ return garbageDocsCount > 0;
+ }
+
+ private void collectDeletedProperties(final NodeDocument doc, final GCPhases phases, final UpdateOp updateOp,
+ final NodeState traversedState) {
+
+ // get Map of all properties along with their values
+ if (phases.start(GCPhase.FULL_GC_COLLECT_PROPS)) {
+ final Set<String> properties = doc.getPropertyNames();
+
+ // find all the properties which can be removed from document.
+ // All the properties whose value is null in head revision are
+ // eligible to be garbage collected.
+
+ final Set<String> retainPropSet = ofNullable(traversedState instanceof DocumentNodeState ? (DocumentNodeState)traversedState : null)
+ .map(DocumentNodeState::getAllBundledProperties)
+ .map(Map::keySet)
+ .map(p -> p.stream().map(Utils::escapePropertyName).collect(toSet()))
+ .orElse(emptySet());
+
+ final int deletedPropsCount = properties.stream()
+ .filter(p -> !retainPropSet.contains(p))
+ .mapToInt(x -> {
+ updateOp.remove(x);
+ return 1;})
+ .sum();
+
+ deletedPropsCountMap.put(doc.getId(), deletedPropsCount);
+
+ if (AUDIT_LOG.isDebugEnabled() && deletedPropsCount > 0) {
+ AUDIT_LOG.debug("<Collected> [{}] deleted props in [{}]", deletedPropsCount, doc.getId());
+ }
+ phases.stop(GCPhase.FULL_GC_COLLECT_PROPS);
+ }
+ }
+
+ private void collectUnmergedBranchCommits(final NodeDocument doc, final GCPhases phases, final UpdateOp updateOp,
+ final long toModifiedMs) {
+ if (!phases.start(GCPhase.FULL_GC_COLLECT_UNMERGED_BC)) {
+ // GC was cancelled, stop
+ return;
+ }
+
+ // from https://jackrabbit.apache.org/oak/docs/nodestore/documentmk.html#previous-documents
+ // "branch commits are not moved to previous documents until the branch is merged."
+ // i.e. if we're looking for unmerged branch commits, they cannot be in
+ // previous documents, they have to be in the main one - hence we have to use
+ // getLocalBranchCommits here
+ final Set<Revision> olderUnmergedBranchCommits = doc.getLocalBranchCommits().stream()
+ .filter(bcRevision -> isRevisionOlderThan(bcRevision, toModifiedMs))
+ .filter(bcRevision -> !isCommitted(nodeStore.getCommitValue(bcRevision, doc)))
+ .collect(toSet());
+
+ if (olderUnmergedBranchCommits.isEmpty()) {
+ // nothing to do then
+ phases.stop(GCPhase.FULL_GC_COLLECT_UNMERGED_BC);
+ return;
+ }
+
+ // !Note, the _bc sub-document was introduced with Oak 1.8 and is not present in older versions.
+ // The branch commit revision is added to _bc whenever a change is done on a document with a
+ // branch commit. This helps the DocumentNodeStore to more easily identify branch commit changes."
+ // The current implementation of "collectUnmergedBranchCommitDocument" only
+ // supports branch commits that are created after Oak 1.8
+
+ olderUnmergedBranchCommits.forEach(bcRevision -> removeUnmergedBCRevision(bcRevision, doc, updateOp));
+ deletedUnmergedBCSet.addAll(olderUnmergedBranchCommits);
+
+ if (AUDIT_LOG.isDebugEnabled()) {
+ AUDIT_LOG.debug("<Collected> [{}] unmerged branch commits in [{}]", olderUnmergedBranchCommits.size(), doc.getId());
+ }
+
+ // now for any of the handled system properties (the normal properties would
+ // already be cleaned up by cleanupDeletedProperties), the resulting
+ // sub document could in theory become empty after removing all unmerged branch
+ // commit revisions is done later.
+ // so we need to go through all of them and check if we'd have removed
+ // the entirety - and then, instead of individually remove revisions, just
+ // delete the entire property.
+ if (updateOp.hasChanges()) {
+ final int deletedSystemPropsCount = getSystemRemoveMapEntryCounts(updateOp)
+ .entrySet().stream()
+ .filter(e -> filterEmptyProps(doc, e.getKey(), e.getValue()))
+ .mapToInt(e -> {
+ final String prop = e.getKey();
+ final int origCount = updateOp.getChanges().size();
+ updateOp.getChanges().entrySet().removeIf(opEntry -> Objects.equals(prop, opEntry.getKey().getName()));
+ final int diff = origCount - updateOp.getChanges().size();
+ //TODO: beautify once this is in FullGC branch
+ if (diff > 0) {
+ deletedInternalPropRevsCountMap.merge(doc.getId(), -diff, Integer::sum);
+ }
+ updateOp.remove(prop);
+ return 1;})
+ .sum();
+
+ // update the deleted properties count Map to calculate the total no. of deleted properties
+ int totalDeletedSystemPropsCount = deletedInternalPropsCountMap.merge(doc.getId(), deletedSystemPropsCount, Integer::sum);
+ if (AUDIT_LOG.isDebugEnabled()) {
+ AUDIT_LOG.debug("<Collected> [{}] internal prop revs in [{}] mode [{}]", totalDeletedSystemPropsCount, doc.getId(), fullGcMode);
+ }
+ }
+ phases.stop(GCPhase.FULL_GC_COLLECT_UNMERGED_BC);
+ }
+
+ /**
+ * Filter all would be empty system properties (after cleanup operation).
+ * <p>
+ * It verifies this by comparing the size of sub-document with given <code>value</code>
+ *
+ * @param doc {@link NodeDocument} on whose properties needs to be checked
+ * @param prop Name of sub-document which needs to checked whether it would be empty after cleanup or not
+ * @param value expected no. of entries
+ * @return true if sub-document would eventually be empty or not
+ */
+ private boolean filterEmptyProps(final NodeDocument doc, final String prop, final int value) {
+ final Object d = doc.data.get(prop);
+ if (d instanceof Map) {
+ @SuppressWarnings("rawtypes") final Map m = (Map) d;
+ // then we're not removing all revisions - so cannot clean up
+ return m.size() == value;
+ } else {
+ // unexpected and would likely indicate a bug, hence log.error
+ log.error("collectUnmergedBranchCommitDocument: property without sub-document as expected. " +
+ "id={}, prop={}", doc.getId(), prop);
+ return false;
+ }
+ }
+
+ /** small helper to count number of REMOVE_MAP_ENTRY per system property */
+ private Map<String, Integer> getSystemRemoveMapEntryCounts(final UpdateOp updateOp) {
+ final Map<String, Integer> propMap = new HashMap<>();
+
+ updateOp.getChanges().entrySet().stream()
+ .filter(e -> e.getValue().type == Type.REMOVE_MAP_ENTRY)
+ .map(e -> e.getKey().getName())
+ .filter(propName -> propName.startsWith("_"))
+ .forEach(propName -> propMap.merge(propName, 1, Integer::sum));
+
+ return propMap;
+ }
+
+ /** small helper to check if the revision is older than the timestamp */
+ private boolean isRevisionOlderThan(Revision revision, long toModifiedMillis) {
+ long time = revision.getTimestamp();
+ return time <= toModifiedMillis;
+ }
+
+ /**
+ * Clean up one uncommitted branch commit in the provided document. The caller
+ * establishes whether a branch commit revision is committed or not - this is no
+ * longer checked in this method. The resulting operations are added to the
+ * provided updateOp.
+ * <p/>
+ * The actions depend on the exacty property key - here's the comprehensive list
+ * of system properties and how they are handled:
+ * <ul>
+ * <li>"_id" and "_path" are not affected</li>
+ * <li>"_prev" and "stalePrev" are not affected</li>
+ * <li>"_lastRev" and "_sweepRev" are not affected</li>
+ * <li>"_bc" is cleaned up</li>
+ * <li>"_revisions" is cleaned up</li>
+ * <li>"_commitRoot" is cleaned up</li>
+ * <li>"_collisions" might be a special case but is also cleaned up</li>
+ * <li>"_modified" and "_deletedOnce" are just single values and thus not
+ * generally affected, but they are updated when "_deleted" is cleaned up</li>
+ * <li>"_deleted" is cleaned up - plus "_modified" and "_deletedOnce" are also
+ * updated for this, to have classic GC later potentially delete the
+ * document</li>
+ * <li>"_children" is only ever set to true, never back - hence no need for
+ * cleanup/modification by FullGC</li>
+ * <li>"_bin" is only ever set to 1, never back - hence no need for
+ * cleanup/modification by FullGC</li>
+ * </ul>
+ *
+ * @param unmergedBCRevision the unmerged branch commit revision - the caller
+ * makes sure this revision is indeed not merged and
+ * old enough to be removed
+ * @param doc the document from which the uncommittedBCRevision
+ * should be removed
+ * @param updateOp the resulting operations yet to be applied
+ */
+ private void removeUnmergedBCRevision(final Revision unmergedBCRevision, final NodeDocument doc,
+ final UpdateOp updateOp) {
+ int internalRevEntriesCount = 0;
+ int revEntriesCount = 0;
+ // caller ensures the provided revision is an unmerged branch commit
+ if (doc.getLocalBranchCommits().contains(unmergedBCRevision)) {
+ internalRevEntriesCount++;
+ }
+ NodeDocument.removeBranchCommit(updateOp, unmergedBCRevision);
+
+ // phase 1 : remove unmerged bc revisions from _deleted - unmerged branch
+ // commits can only be in the local set
+ final String unmergedDeleted = doc.getLocalDeleted().get(unmergedBCRevision);
+ if (unmergedDeleted != null) {
+ internalRevEntriesCount++;
+ NodeDocument.removeDeleted(updateOp, unmergedBCRevision);
+
+ // phase 2: the document could now effectively be "deleted" the actual
+ // removal from the document from the store is left to DeletedDocsGC (to
+ // avoid code duplication)
+
+ // if unmergedDeleted is "false", then it was created with this branch
+ // commit, but that was never merged. when we now remove that, it could be
+ // that it is then deleted.
+
+ // to know whether the node is actually deleted, would potentially
+ // require several commit value lookups.
+ // in order to keep the execution time of fullGC in this regard small,
+ // the code here stops with any further checks and just sets
+ // "_deletedOnce" to true and updates "_modified" - the DeletedDocsGC
+ // later would either resurrect or delete the document properly (eg
+ // including previous docs)
+ if ("false".equals(unmergedDeleted)) {
+ if (!doc.wasDeletedOnce()) {
+ NodeDocument.setDeletedOnce(updateOp);
+ }
+ NodeDocument.setModified(updateOp, revisionForModified);
+ }
+ }
+ // phase 3 : go through other system properties
+ if (doc.getLocalCommitRoot().containsKey(unmergedBCRevision)) {
+ internalRevEntriesCount++;
+ NodeDocument.removeCommitRoot(updateOp, unmergedBCRevision);
+ }
+ if (doc.getLocalRevisions().containsKey(unmergedBCRevision)) {
+ internalRevEntriesCount++;
+ NodeDocument.removeRevision(updateOp, unmergedBCRevision);
+ }
+ if (doc.getLocalMap(COLLISIONS).containsKey(unmergedBCRevision)) {
+ internalRevEntriesCount++;
+ NodeDocument.removeCollision(updateOp, unmergedBCRevision);
+ }
+ // phase 4 : go through normal properties
+ for (String propName : doc.getPropertyNames()) {
+ // first check if this property might have been (flagged to be)
+ // entirely removed by the collectDeletedProperties - in which case
+ // there is a corresponding UpdateOp
+ final Operation op = updateOp.getChanges().get(new Key(propName, null));
+ if (op != null && op.type == Type.REMOVE) {
+ // ignore this property then, it will be removed entirely
+ continue;
+ }
+ if (doc.getLocalMap(propName).containsKey(unmergedBCRevision)) {
+ updateOp.removeMapEntry(propName, unmergedBCRevision);
+ revEntriesCount++;
+ }
+ };
+ if (internalRevEntriesCount > 0) {
+ deletedInternalPropRevsCountMap.merge(doc.getId(), internalRevEntriesCount, Integer::sum);
+ }
+ if (revEntriesCount > 0) {
+ deletedPropRevsCountMap.merge(doc.getId(), revEntriesCount, Integer::sum);
+ }
+
+ if (AUDIT_LOG.isDebugEnabled()) {
+ AUDIT_LOG.debug("<Collected> [{}] prop revs, [{}] internal prop revs in [{}] mode [{}]", deletedPropRevsCountMap.get(doc.getId()), deletedInternalPropRevsCountMap.get(doc.getId()), doc.getId(), fullGcMode);
+ }
+
+ }
+
+ private void collectRevisionsOlderThan24hAndBetweenCheckpoints(final NodeDocument doc, final long toModifiedMs,
+ final GCPhases phases, final UpdateOp updateOp) {
+ if (phases.start(GCPhase.FULL_GC_COLLECT_OLD_REVS)){
+ NodeDocumentRevisionCleaner cleaner = new NodeDocumentRevisionCleaner(nodeStore, doc, toModifiedMs);
+ final int beforeRevs = countRevs(updateOp, false);
+ final int beforeIntRevs = countRevs(updateOp, true);
+ cleaner.collectOldRevisions(updateOp);
+ final int revsDiff = countRevs(updateOp, false) - beforeRevs;
+ if (revsDiff > 0) {
+ deletedPropRevsCountMap.merge(doc.getId(), revsDiff, Integer::sum);
+ }
+ final int intRevsDiff = countRevs(updateOp, true) - beforeIntRevs;
+ if (intRevsDiff > 0) {
+ deletedInternalPropRevsCountMap.merge(doc.getId(), intRevsDiff, Integer::sum);
+ }
+ if (AUDIT_LOG.isDebugEnabled()) {
+ AUDIT_LOG.debug("<Collected> [{}] prop revs, [{}] internal prop revs in [{}] mode [{}]", deletedPropRevsCountMap.get(doc.getId()), deletedInternalPropRevsCountMap.get(doc.getId()), doc.getId(), fullGcMode);
+ }
+ phases.stop(GCPhase.FULL_GC_COLLECT_OLD_REVS);
+ }
+ }
+
+ private int countRevs(UpdateOp updateOp, boolean internalProps) {
+ return updateOp.getChanges().entrySet().stream()
+ .filter(e -> (e.getValue().type == Type.REMOVE_MAP_ENTRY))
+ .filter(e -> (e.getKey().getName().startsWith("_") == internalProps))
+ .mapToInt(x -> 1).sum();
+ }
+
+ /**
+ * Remove all property revisions in the local document that are no longer used.
+ * This includes bundled properties. It also includes related entries that
+ * become obsolete as a result - i.e. _commitRoot and _bc (though the latter
+ * is never removed on root)
+ */
+ private void collectUnusedPropertyRevisions(final NodeDocument doc,
+ final GCPhases phases, final UpdateOp updateOp,
+ final DocumentNodeState traversedMainNode,
+ final boolean ignoreInternalProperties) {
+
+ if (!phases.start(GCPhase.FULL_GC_COLLECT_OLD_REVS)){
+ // cancelled
+ return;
+ }
+ final Set<Revision> allKeepRevs = new HashSet<>();
+ // phase A : collectUnusedUserPropertyRevisions
+ int deletedTotalRevsCount = collectUnusedUserPropertyRevisions(doc, updateOp, traversedMainNode, allKeepRevs);
+ int deletedUserRevsCount = deletedTotalRevsCount;
+ // phase B : collectUnusedInternalPropertyRevisions
+ if (!ignoreInternalProperties) {
+ deletedTotalRevsCount = collectUnusedInternalPropertyRevisions(doc, updateOp, allKeepRevs, deletedTotalRevsCount);
+ }
+
+ // then some accounting...
+ int deletedInternalRevsCount = deletedTotalRevsCount - deletedUserRevsCount;
+ if (deletedUserRevsCount != 0) {
+ deletedPropRevsCountMap.merge(doc.getId(), deletedUserRevsCount, Integer::sum);
+ }
+ if (deletedInternalRevsCount != 0) {
+ deletedInternalPropRevsCountMap.merge(doc.getId(), deletedInternalRevsCount, Integer::sum);
+ }
+
+ if (AUDIT_LOG.isDebugEnabled()) {
+ AUDIT_LOG.debug("<Collected> [{}] prop revs, [{}] internal prop revs in [{}] mode [{}]", deletedPropRevsCountMap.get(doc.getId()), deletedInternalPropRevsCountMap.get(doc.getId()), doc.getId(), fullGcMode);
+ }
+
+ phases.stop(GCPhase.FULL_GC_COLLECT_OLD_REVS);
+ }
+
+ private int collectUnusedUserPropertyRevisions(final NodeDocument doc, final UpdateOp updateOp,
+ final DocumentNodeState traversedMainNode, final Set<Revision> allKeepRevs) {
+ // phase 1 : non bundled nodes only
+ int deletedRevsCount = stream(
+ traversedMainNode.getProperties().spliterator(), false)
+ .map(p -> Utils.escapePropertyName(p.getName()))
+ .mapToInt(p -> removeUnusedPropertyEntries(doc,
+ traversedMainNode, updateOp, p,
+ r -> updateOp.removeMapEntry(p, r),
+ allKeepRevs))
+ .sum();
+
+ // phase 2 : bundled nodes only
+ final Map<Path, DocumentNodeState> bundledNodeStates = stream(
+ traversedMainNode.getAllBundledNodesStates().spliterator(), false)
+ .collect(toMap(DocumentNodeState::getPath, identity()));
+ // remember that getAllBundledProperties returns unescaped keys
+ for (String propName : traversedMainNode.getAllBundledProperties().keySet()) {
+ final int lastSlash = propName.lastIndexOf("/");
+ if (lastSlash == -1) {
+ // then it is an unbundled property which was already handled in phase 1
+ continue;
+ }
+ final String escapedPropName = Utils.escapePropertyName(propName);
+ // bundled values are of format sub/tree/propertyKey
+ // to extract this we need the last index of "/"
+ final String unbundledSubtreeName = propName.substring(0, lastSlash);
+ final String unbundledPropName = propName.substring(lastSlash + 1);
+ final String unbundledPath = traversedMainNode.getPath().toString() + "/" + unbundledSubtreeName;
+ final DocumentNodeState traversedNode = bundledNodeStates.get(Path.fromString(unbundledPath));
+ if (traversedNode == null) {
+ log.error("collectUnusedPropertyRevisions : could not find traversed node for bundled key [{}] unbundledPath [{}] in doc [{}]",
+ propName, unbundledPath, doc.getId());
+ continue;
+ }
+ final PropertyState traversedProperty = traversedNode.getProperty(unbundledPropName);
+ if (traversedProperty == null) {
+ log.error("collectUnusedPropertyRevisions : could not get property [{}] from traversed node [{}]",
+ unbundledPropName, traversedNode.getPath());
+ continue;
+ }
+ deletedRevsCount += removeUnusedPropertyEntries(doc, traversedNode, updateOp, escapedPropName,
+ (r) -> updateOp.removeMapEntry(escapedPropName, r),
+ allKeepRevs);
+ }
+
+ // phase 3 : "_deleted"
+ int numDeleted = removeUnusedPropertyEntries(doc, traversedMainNode,
+ updateOp, NodeDocument.DELETED,
+ (r) -> NodeDocument.removeDeleted(updateOp, r),
+ allKeepRevs);
+ deletedRevsCount += numDeleted;
+ return deletedRevsCount;
+ }
+
+ private int collectUnusedInternalPropertyRevisions(final NodeDocument doc, final UpdateOp updateOp,
+ final Set<Revision> toKeepUserPropRevs, int deletedRevsCount) {
+ boolean hasUnmergedBranchCommits = doc.getLocalBranchCommits().stream()
+ .anyMatch(r -> !isCommitted(nodeStore.getCommitValue(r, doc)));
+ if (deletedRevsCount == 0 && !hasUnmergedBranchCommits) {
+ return deletedRevsCount;
+ }
+ // if we did some rev deletion OR there are unmerged BCs, then let's deep-dive
+ Set<Revision> allRequiredRevs = new HashSet<>(toKeepUserPropRevs);
+ // "_collisions"
+ deletedRevsCount += getDeletedRevsCount(
+ doc.getLocalMap(COLLISIONS).keySet(), updateOp,
+ allRequiredRevs, COLLISIONS,
+ NodeDocument::removeCollision);
+ // "_revisions"
+ for (Entry<Revision, String> e : doc.getLocalRevisions().entrySet()) {
+ Revision revision = e.getKey();
+ if (allRequiredRevs.contains(revision)) {
+ // if it is still referenced locally, keep it
+ continue;
+ }
+ final boolean isRoot = doc.getId().equals(getIdFromPath(Path.ROOT));
+ // local bcs only considered for removal
+ final boolean isBC = doc.getLocalBranchCommits().contains(revision);
+ final boolean newerThanSweep = nodeStore.getSweepRevisions().isRevisionNewer(revision);
+ if (newerThanSweep) {
+ if (isBC) {
+ // analysed further down, we can remove them if unmerged
+ } else {
+ // note that ideally we wouldn't ever get to handle a revision
+ // that is younger than sweep, as sweep is usually only a few
+ // seconds behind and GC usually 24h.
+ // in edge cases however, it could happen.
+ // hence we could try to avoid even that it happens at all
+ // by taking sweep into account in the original query.
+ // Except, that's not feasible with the current state of the document,
+ // where "_modified" is cross-cluster, but sweep is per-clusterId.
+ // So which ever value we'd choose in the original query, it wouldn't
+ // precisely match the sweep revision vector in any case.
+ // thus we should rather accept that we could in theory
+ // get a revision that is newer than sweep, and act accordingly.
+ // one way would be to try to distinguish branch commit vs normal commits.
+ // another, simpler, way however is to simply skip these.
+ // that, yes, opens the possibility of some garbage not being
+ // cleaned up, should the document remain unchanged ever after.
+ // but that seems low probability and justifyable.
+ // so:
+ // for normal commits, we need to keep them -> also add to allRequiredRevs
+ allRequiredRevs.add(revision);
+ continue;
+ }
+ }
+ // committed revisions are hard to delete, as determining
+ // whether or not they are orphaned is difficult.
+ // child nodes could be having _commitRoots pointing to this doc
+ // and this revision - in which case it wouldn't be orphaned.
+ // without scanning the children (or having knowledge about
+ // which revisions are referenced by children) it is unsafe
+ // to delete committed revisions.
+ // if they are uncommitted, then they are garbage, as then they
+ // are either normal commits (thus a not finished commit)
+ // or an unmerged branch commit. In both cases they are garbage.
+ boolean isCommitted = isCommitted(nodeStore.getCommitValue(revision, doc));
+ if (isCommitted) {
+ if (isRoot) {
+ // root : cannot remove since it could be referenced and required by a child
+ // also add to allRequiredRevs
+ allRequiredRevs.add(revision);
+ continue;
+ } else if (!isBC) {
+ // non root and normal : cannot remove, same as above,
+ // it could be referenced and required by a child
+ // also add to allRequiredRevs
+ allRequiredRevs.add(revision);
+ continue;
+ } else {
+ // non root and bc : can remove, as non root bc cannot be referenced by child
+ }
+ }
+ Operation has = updateOp.getChanges().get(new Key(REVISIONS, revision));
+ if (has != null) {
+ // then skip
+ continue;
+ }
+ NodeDocument.removeRevision(updateOp, revision);
+ deletedRevsCount++;
+ }
+ // "_commitRoot"
+ deletedRevsCount += getDeletedRevsCount(doc.getLocalCommitRoot().keySet(),
+ updateOp, allRequiredRevs, COMMIT_ROOT, NodeDocument::removeCommitRoot);
+ // "_bc"
+ deletedRevsCount += getDeletedRevsCount(doc.getLocalBranchCommits(), updateOp,
+ allRequiredRevs, BRANCH_COMMITS, NodeDocument::removeBranchCommit);
+ return deletedRevsCount;
+ }
+
+ private int getDeletedRevsCount(Set<Revision> revisionSet, UpdateOp updateOp, Set<Revision> allRequiredRevs, String updateOpKey, BiConsumer<UpdateOp, Revision> op) {
+ return revisionSet.stream().filter(r -> !allRequiredRevs.contains(r))
+ .mapToInt(r -> {
+ Operation has = updateOp.getChanges().get(new Key(updateOpKey, r));
+ if (has != null) {
+ // then skip
+ return 0;
+ }
+
+ op.accept(updateOp, r);
+ return 1;
+ }).sum();
+ }
+
+ private int removeUnusedPropertyEntries(NodeDocument doc,
+ DocumentNodeState traversedMainNode, UpdateOp updateOp,
+ String propertyKey, Consumer<Revision> removeRevision,
+ Set<Revision> allKeepRevs) {
+ // we need to use the traversedNode.getLastRevision() as the readRevision,
+ // as that is what was originally used in getNodeAtRevision when traversing
+ final Revision keepCommitRev = doc.localCommitRevisionOfProperty(nodeStore,
+ traversedMainNode.getLastRevision(), propertyKey);
+ if (keepCommitRev == null) {
+ // could be due to node not existing or current value being in a split
+ // doc - while the former is unexpected, the latter might happen.
+ // in both cases let's skip this property
+ if (AUDIT_LOG.isTraceEnabled()) {
+ AUDIT_LOG.trace("<remove> : [0] visible revision for property [{}] in [{}]", propertyKey, doc.getId());
+ }
+ return 0;
+ }
+ // if we get a revision it is from the local map.
+ // paranoia check that
+ final SortedMap<Revision, String> localMap = doc.getLocalMap(propertyKey);
+ if (!localMap.containsKey(keepCommitRev)) {
+ // this is unexpected - log and skip this property
+ log.error("removeUnusedPropertyEntries : revision [{}] for property [{}] not found in doc [{}]", keepCommitRev, propertyKey, doc.getId());
+ return 0;
+ }
+ int count = 0;
+ // in this case we are good to delete all but the keepRevision
+ for (Revision localRev : localMap.keySet()) {
+ if (!keepCommitRev.equals(localRev)) {
+ // if the localRev is a branch commit, it might be unmerged,
+ // in which case it might already have been marked for removal
+ // via collectUnmergedBranchCommits. Checking for that next.
+ Operation c = updateOp.getChanges().get(new Key(propertyKey, localRev));
+ if (c == null) {
+ if (AUDIT_LOG.isTraceEnabled()) {
+ AUDIT_LOG.trace("<remove> : property key [{}] revision [{}] in [{}]", propertyKey, localRev, doc.getId());
+ }
+ removeRevision.accept(localRev);
+ count++;
+ }
+ }
+ }
+ allKeepRevs.add(keepCommitRev);
+ return count;
+ }
+
+
+ int getGarbageCount() {
+ return totalGarbageDocsCount;
+ }
+
+ @Override
+ public void close() {
+ totalGarbageDocsCount = 0;
+ }
+
+
+ public void removeGarbage(final VersionGCStats stats) {
+
+ if (updateOpList.isEmpty() && orphanOrDeletedRemovalMap.isEmpty()) {
+ if (log.isDebugEnabled() || isFullGCDryRun) {
+ log.debug("Skipping removal of Full garbage, cause no garbage detected");
+ }
+ return;
+ }
+
+ monitor.info("Proceeding to update [{}] documents", updateOpList.size());
+
+ if (AUDIT_LOG.isDebugEnabled() || isFullGCDryRun) {
+ String updateIds = updateOpList.stream().map(UpdateOp::getId).collect(joining(", "));
+ String orphanIds = join(", ", orphanOrDeletedRemovalMap.keySet());
+ log.debug("Performing batch update of ids [{}] and removal of orphan ids [{}]", updateIds, orphanIds);
+ }
+
+ if (cancel.get()) {
+ log.info("Aborting the removal of Full garbage since RGC had been cancelled");
+ return;
+ }
+
+ timer.reset().start();
+ try {
+ if (embeddedVerification) {
+ // embedded verification is done completely independent of DocumentStore
+ // also, it's done irrespective of dry-run or not
+ final Iterator<UpdateOp> it = updateOpList.iterator();
+ while(it.hasNext()) {
+ final UpdateOp update = it.next();
+ NodeDocument oldDoc = ds.find(Collection.NODES, update.getId());
+ if (oldDoc == null) {
+ log.error("removeGarbage.verify : no document found for update with id [{}]", update.getId());
+ continue;
+ }
+ NodeState traversedParent = null;
+ NodeState traversedState = root;
+ for (String name : oldDoc.getPath().elements()) {
+ traversedParent = traversedState;
+ traversedState = traversedState.getChildNode(name);
+ }
+
+ NodeDocument newDoc = Collection.NODES.newDocument(ds);
+ oldDoc.deepCopy(newDoc);
+ UpdateUtils.applyChanges(newDoc, update);
+ // for the time being, verify both with classic and traversed
+ if (!verifyViaTraversedState(traversedState, traversedParent, newDoc)) {
+ // verification failure
+ // let's skip this document
+ if (log.isDebugEnabled()) {
+ log.debug("removeGarbage.verify : verifyViaTraversedState failed for [{}]", newDoc.getId());
+ }
+ it.remove();
+ stats.skippedFullGCDocsCount++;
+ }
+ }
+ for (Entry<String, Long> e : orphanOrDeletedRemovalMap.entrySet()) {
+ NodeState traversedState = root;
+ final String id = e.getKey();
+ final Path path = orphanOrDeletedRemovalPathMap.get(id);
+ if (path == null) {
+ // rather a bug, so let's skip it
+ log.error("removeGarbage.verify : no path available for id : [{}]", id);
+ it.remove();
+ stats.skippedFullGCDocsCount++;
+ continue;
+ }
+ for (String name : path.elements()) {
+ traversedState = traversedState.getChildNode(name);
+ }
+ if (!verifyDeletion(traversedState)) {
+ // verification failure
+ // let's skip this document
+ if (log.isDebugEnabled()) {
+ log.debug("removeGarbage.verify : verifyDeletion failed for [{}]", e.getKey());
+ }
+ it.remove();
+ stats.skippedFullGCDocsCount++;
+ }
+ }
+ }
+ if (!isFullGCDryRun) {
+ // only delete these in case it is not a dryRun
+
+ if (!orphanOrDeletedRemovalMap.isEmpty()) {
+ // use remove() with the modified check to rule
+ // out any further race-condition where this removal
+ // races with a un-orphan/re-creation as a result of which
+ // the node should now not be removed. The modified check
+ // ensures a node would then not be removed
+ // (and as a result the removedSize != map.size())
+ final int removedSize = ds.remove(NODES, orphanOrDeletedRemovalMap);
+ stats.updatedFullGCDocsCount += removedSize;
+ stats.deletedDocGCCount += removedSize;
+ stats.deletedOrphanNodesCount += removedSize;
+
+ if (AUDIT_LOG.isDebugEnabled()) {
+ AUDIT_LOG.debug("<delete> [{}] documents (from intended {})", removedSize, orphanOrDeletedRemovalMap.size());
+ }
+
+ // save stats
+ fullGCStats.documentsUpdated(removedSize);
+ fullGCStats.orphanNodesDeleted(removedSize);
+ gcStats.documentsDeleted(removedSize);
+ // fix for sonar : converted to long before operation
+ fullGCStats.documentsUpdateSkipped((long)orphanOrDeletedRemovalMap.size() - removedSize);
+ }
+
+ if (!updateOpList.isEmpty()) {
+ List<NodeDocument> oldDocs = ds.findAndUpdate(NODES, updateOpList);
+
+
+ int deletedProps = oldDocs.stream().filter(Objects::nonNull).mapToInt(d -> deletedPropsCountMap.getOrDefault(d.getId(), 0)).sum();
+ int deletedInternalProps = oldDocs.stream().filter(Objects::nonNull).mapToInt(d -> deletedInternalPropsCountMap.getOrDefault(d.getId(), 0)).sum();
+ int deletedRevEntriesCount = oldDocs.stream().filter(Objects::nonNull).mapToInt(d -> deletedPropRevsCountMap.getOrDefault(d.getId(), 0)).sum();
+ int deletedInternalRevEntriesCount = oldDocs.stream().filter(Objects::nonNull).mapToInt(d -> deletedInternalPropRevsCountMap.getOrDefault(d.getId(), 0)).sum();
+ int updatedDocs = (int) oldDocs.stream().filter(Objects::nonNull).count();
+ stats.updatedFullGCDocsCount += updatedDocs;
+ stats.deletedPropsCount += deletedProps;
+ stats.deletedInternalPropsCount += deletedInternalProps;
+ stats.deletedPropRevsCount += deletedRevEntriesCount;
+ stats.deletedInternalPropRevsCount += deletedInternalRevEntriesCount;
+ stats.deletedUnmergedBCCount += deletedUnmergedBCSet.size();
+
+ if (log.isDebugEnabled()) {
+ log.debug("Updated [{}] docs, deleted [{}] props, deleted [{}] unmergedBCs, deleted [{}] internal Props, deleted [{}] prop revs, deleted [{}] internal prop revs",
+ updatedDocs, deletedProps, deletedUnmergedBCSet.size(), deletedInternalProps, deletedRevEntriesCount, deletedInternalRevEntriesCount);
+ }
+
+ // save stats
+ fullGCStats.propertiesDeleted(deletedProps);
+ fullGCStats.unmergedBranchCommitsDeleted(deletedUnmergedBCSet.size());
+ fullGCStats.documentsUpdated(updatedDocs);
+ // fix for sonar : converted to long before operation
+ fullGCStats.documentsUpdateSkipped((long)oldDocs.size() - updatedDocs);
+ }
+ } else {
+ // collect approx stats only in case of dryRun by assuming everything would succeed
+
+ // for deleted properties, oldRevs & unmergedBC
+ stats.updatedFullGCDocsCount += updateOpList.size();
+ stats.deletedPropsCount += deletedPropsCountMap.values().stream().reduce(0, Integer::sum);
+ stats.deletedInternalPropsCount += deletedInternalPropsCountMap.values().stream().reduce(0, Integer::sum);
+ stats.deletedPropRevsCount += deletedPropRevsCountMap.values().stream().reduce(0, Integer::sum);
+ stats.deletedInternalPropRevsCount += deletedInternalPropRevsCountMap.values().stream().reduce(0, Integer::sum);
+ stats.deletedUnmergedBCCount += deletedUnmergedBCSet.size();
+
+ // for orphan nodes.
+ stats.updatedFullGCDocsCount += orphanOrDeletedRemovalMap.size();
+ stats.deletedDocGCCount += orphanOrDeletedRemovalMap.size();
+ stats.deletedOrphanNodesCount += orphanOrDeletedRemovalMap.size();
+ }
} finally {
- gc.close();
+ // now reset delete metadata
+ updateOpList.clear();
+ orphanOrDeletedRemovalMap.clear();
+ orphanOrDeletedRemovalPathMap.clear();
+ deletedPropsCountMap.clear();
+ deletedInternalPropsCountMap.clear();
+ deletedPropRevsCountMap.clear();
+ deletedInternalPropRevsCountMap.clear();
+ deletedUnmergedBCSet.clear();
+ garbageDocsCount = 0;
+ delayOnModifications(timer.stop().elapsed(MILLISECONDS), cancel);
+ }
+ }
+
+ private boolean verifyViaTraversedState(NodeState traversedState, NodeState traversedParent, NodeDocument newDoc) {
+ final Path path = newDoc.getPath();
+ final Revision lastRevision = nodeStore.getPendingModifications().get(path);
+ if (traversedParent == null && !newDoc.getPath().isRoot()) {
+ log.error("verify : no parent but not root for path : [{}]", newDoc.getPath());
+ return false;
+ }
+ final RevisionVector lastRev;
+ if (traversedParent == null && newDoc.getPath().isRoot()) {
+ if (!(traversedState instanceof DocumentNodeState)) {
+ log.error("verify : traversedState not a DocumentNodeState : [{}]", traversedState.getClass());
+ return false;
+ }
+ lastRev = ((DocumentNodeState) traversedState).getLastRevision();
+ } else {
+ // in case we are reaching here, it means that node is none of parents are null
+ if (!traversedParent.exists()) {
+ // if the parent doesn't exist we shouldn't reach this point at all
+ log.error("verify : no parent but not marked for removal for path : {}", newDoc.getPath());
+ return false;
+ }
+ if (!(traversedParent instanceof DocumentNodeState)) {
+ log.error("verify : traversedParent not a DocumentNodeState : {}", traversedParent.getClass());
+ return false;
+ }
+ lastRev = ((DocumentNodeState) traversedParent).getLastRevision();
+ }
+ final NodeState actual = newDoc.getNodeAtRevision(nodeStore, lastRev, lastRevision);
+ // use more thorough version of equals to ensure properties are checked
+ // (the faster state.equals() would stop if lastRev matches,
+ // but as we're fiddling with immutability rule of a document,
+ // we need to do a full check)
+ return nonNull(actual) && AbstractNodeState.equals(traversedState, actual);
+ }
+
+ private boolean verifyDeletion(NodeState traversedState) {
+ return !traversedState.exists();
+ }
+ }
+ private void delayOnModifications(final long durationMs, final AtomicBoolean cancel) {
+ long delayMs = round(durationMs * options.delayFactor);
+ if (!cancel.get() && delayMs > 0) {
+ try {
+ Clock clock = nodeStore.getClock();
+ clock.waitUntil(clock.getTime() + delayMs);
+ }
+ catch (InterruptedException ex) {
+ /* ignore */
+ }
+ }
+ }
+
+ public void collectGarbageOnDocument(DocumentNodeStore store, NodeDocument doc, boolean verbose) {
+ VersionGCStats stats = new VersionGCStats();
+ stats.active.start();
+ AtomicBoolean cancel = new AtomicBoolean();
+ GCPhases phases = new GCPhases(cancel, stats, gcMonitor);
+
+ final LinkedHashMap<Path, Boolean> missingDocsTypes = new LinkedHashMap<>();
+ final RevisionVector headRevision = store.getHeadRevision();
+ try (FullGC gc = new FullGC(headRevision, 0, missingDocsTypes, gcMonitor, cancel)) {
+
+ if (phases.start(GCPhase.FULL_GC_COLLECT_GARBAGE)) {
+ gc.collectGarbage(doc, phases);
+ phases.stop(GCPhase.FULL_GC_COLLECT_GARBAGE);
+ }
+
+ if (verbose) {
+ gcMonitor.info("GarbageCollector will run [{}] operations", gc.updateOpList.size());
+ for (UpdateOp update : gc.updateOpList) {
+ gcMonitor.info(update.toString());
+ }
+ }
+
+ if (gc.hasGarbage() && phases.start(GCPhase.FULL_GC_CLEANUP)) {
+ gc.removeGarbage(phases.stats);
+ phases.stop(GCPhase.FULL_GC_CLEANUP);
}
}
}
@@ -625,6 +2080,7 @@ public class VersionGarbageCollector {
private final Set<String> exclude = Sets.newHashSet();
private boolean sorted = false;
private final Stopwatch timer;
+ @SuppressWarnings("unused")
private final VersionGCOptions options;
private final GCMonitor monitor;
@@ -632,8 +2088,8 @@ public class VersionGarbageCollector {
@NotNull AtomicBoolean cancel,
@NotNull VersionGCOptions options,
@NotNull GCMonitor monitor) {
- this.headRevision = checkNotNull(headRevision);
- this.cancel = checkNotNull(cancel);
+ this.headRevision = requireNonNull(headRevision);
+ this.cancel = requireNonNull(cancel);
this.timer = Stopwatch.createUnstarted();
this.options = options;
this.monitor = monitor;
@@ -745,19 +2201,6 @@ public class VersionGarbageCollector {
//------------------------------< internal >----------------------------
- private void delayOnModifications(long durationMs) {
- long delayMs = Math.round(durationMs * options.delayFactor);
- if (!cancel.get() && delayMs > 0) {
- try {
- Clock clock = nodeStore.getClock();
- clock.waitUntil(clock.getTime() + delayMs);
- }
- catch (InterruptedException ex) {
- /* ignore */
- }
- }
- }
-
private Iterator<String> previousDocIdsFor(NodeDocument doc) {
Map<Revision, Range> prevRanges = doc.getPreviousRanges(true);
if (prevRanges.isEmpty()) {
@@ -913,7 +2356,7 @@ public class VersionGarbageCollector {
monitor.info(msg);
}
} finally {
- delayOnModifications(timer.stop().elapsed(TimeUnit.MILLISECONDS));
+ delayOnModifications(timer.stop().elapsed(TimeUnit.MILLISECONDS), cancel);
}
}
return deletedCount;
@@ -946,7 +2389,7 @@ public class VersionGarbageCollector {
}
}
finally {
- delayOnModifications(timer.stop().elapsed(TimeUnit.MILLISECONDS));
+ delayOnModifications(timer.stop().elapsed(TimeUnit.MILLISECONDS), cancel);
}
return updateCount;
}
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java
index 93f22f5202..67751d912a 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java
@@ -19,9 +19,18 @@
package org.apache.jackrabbit.oak.plugins.document.mongo;
+import static com.mongodb.client.model.Filters.eq;
+import static com.mongodb.client.model.Filters.exists;
+import static com.mongodb.client.model.Filters.gt;
+import static com.mongodb.client.model.Filters.or;
+import static com.mongodb.client.model.Projections.include;
+import static java.util.Optional.empty;
+import static java.util.Optional.ofNullable;
import static org.apache.jackrabbit.guava.common.collect.Iterables.concat;
import static org.apache.jackrabbit.guava.common.collect.Iterables.filter;
import static org.apache.jackrabbit.guava.common.collect.Iterables.transform;
+import static com.mongodb.client.model.Filters.and;
+import static com.mongodb.client.model.Filters.lt;
import static java.util.Collections.emptyList;
import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
import static org.apache.jackrabbit.oak.plugins.document.Document.ID;
@@ -33,14 +42,17 @@ import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.SD_TYPE;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.getModifiedInSecs;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType.DEFAULT_NO_BRANCH;
import static org.apache.jackrabbit.oak.plugins.document.mongo.MongoUtils.hasIndex;
+import static org.apache.jackrabbit.oak.plugins.document.util.CloseableIterable.wrap;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
+import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
+import com.mongodb.client.MongoCursor;
import org.apache.jackrabbit.oak.plugins.document.Document;
import org.apache.jackrabbit.oak.plugins.document.NodeDocument;
import org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType;
@@ -120,6 +132,71 @@ public class MongoVersionGCSupport extends VersionGCSupport {
input -> store.convertFromDBObject(NODES, input)));
}
+ /**
+ * Returns documents that have a {@link NodeDocument#MODIFIED_IN_SECS} value
+ * within the given range and are greater than given @{@link NodeDocument#ID}.
+ * <p>
+ * The two passed modified timestamps are in milliseconds
+ * since the epoch and the implementation will convert them to seconds at
+ * the granularity of the {@link NodeDocument#MODIFIED_IN_SECS} field and
+ * then perform the comparison.
+ * <p/>
+ *
+ * @param fromModified the lower bound modified timestamp in millis (inclusive)
+ * @param toModified the upper bound modified timestamp in millis (exclusive)
+ * @param limit the limit of documents to return
+ * @param fromId the lower bound {@link NodeDocument#ID}
+ * @return matching documents.
+ */
+ @Override
+ public Iterable<NodeDocument> getModifiedDocs(final long fromModified, final long toModified, final int limit,
+ @NotNull final String fromId) {
+ // (_modified = fromModified && _id > fromId || _modified > fromModified && _modified < toModified)
+ final Bson query = or(
+ and(eq(MODIFIED_IN_SECS, getModifiedInSecs(fromModified)), gt(ID, fromId)),
+ and(gt(MODIFIED_IN_SECS, getModifiedInSecs(fromModified)), lt(MODIFIED_IN_SECS, getModifiedInSecs(toModified))));
+
+ // first sort by _modified and then by _id
+ final Bson sort = and(eq(MODIFIED_IN_SECS, 1), eq(ID, 1));
+
+ final FindIterable<BasicDBObject> cursor = getNodeCollection()
+ .find(query)
+ .sort(sort)
+ .limit(limit);
+ return wrap(transform(cursor, input -> store.convertFromDBObject(NODES, input)));
+ }
+
+ /**
+ * Retrieves a document with the given id from the MongoDB collection.
+ * If a list of fields is provided, only these fields are included in the returned document.
+ *
+ * @param id the id of the document to retrieve
+ * @param fields the list of fields to include in the returned document. If null or empty, all fields are returned.
+ * @return an Optional that contains the requested NodeDocument if it exists, or an empty Optional if it does not.
+ */
+ @Override
+ public Optional<NodeDocument> getDocument(final String id, final List<String> fields) {
+
+ final Bson query = eq(ID, id);
+
+ final FindIterable<BasicDBObject> result = getNodeCollection().find(query);
+
+ if (fields != null && !fields.isEmpty()) {
+ result.projection(include(fields));
+ }
+
+ try(MongoCursor<BasicDBObject> cur = result.iterator()) {
+ return cur.hasNext() ? ofNullable(store.convertFromDBObject(NODES, cur.next())) : empty();
+ } catch (Exception ex) {
+ LOG.error("getDocument() <- error while fetching data from Mongo", ex);
+ }
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("No Doc has been found with id [{}], retuning empty", id);
+ }
+ return empty();
+
+ }
+
@Override
public long getDeletedOnceCount() {
Bson query = Filters.eq(DELETED_ONCE, Boolean.TRUE);
@@ -193,6 +270,30 @@ public class MongoVersionGCSupport extends VersionGCSupport {
return result.get(0);
}
+ /**
+ * Retrieve the time of the oldest modified document.
+ *
+ * @param clock System Clock to measure time in accuracy of millis
+ * @return the timestamp of the oldest modified document.
+ */
+ @Override
+ public Optional<NodeDocument> getOldestModifiedDoc(final Clock clock) {
+ final Bson sort = and(eq(MODIFIED_IN_SECS, 1), eq(ID, 1));
+
+ // we need to add query condition to ignore `previous` documents which doesn't have this field
+ final Bson query = exists(MODIFIED_IN_SECS);
+
+ FindIterable<BasicDBObject> limit = getNodeCollection().find(query).sort(sort).limit(1);
+
+ try(MongoCursor<BasicDBObject> cur = limit.iterator()) {
+ return cur.hasNext() ? ofNullable(store.convertFromDBObject(NODES, cur.next())) : empty();
+ } catch (Exception ex) {
+ LOG.error("getOldestModifiedDoc() <- error while fetching data from Mongo", ex);
+ }
+ LOG.info("No Modified Doc has been found, retuning empty");
+ return empty();
+ }
+
private List<Bson> createQueries(Set<SplitDocType> gcTypes,
RevisionVector sweepRevs,
long oldestRevTimeStamp) {
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilder.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilder.java
index f997e4bf9a..ca7bf9aa0b 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilder.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilder.java
@@ -26,6 +26,8 @@ import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder;
import org.apache.jackrabbit.oak.plugins.document.DocumentStore;
import org.apache.jackrabbit.oak.plugins.document.MissingLastRevSeeker;
import org.apache.jackrabbit.oak.plugins.document.VersionGCSupport;
+import org.apache.jackrabbit.oak.spi.toggle.Feature;
+import org.jetbrains.annotations.Nullable;
/**
* A builder for a {@link DocumentNodeStore} backed by a relational database.
@@ -113,4 +115,49 @@ public class RDBDocumentNodeStoreBuilder
return super.createMissingLastRevSeeker();
}
}
+
+ @Override
+ public boolean isFullGCEnabled() {
+ // fullGC is non supported for RDB
+ return false;
+ }
+
+ @Override
+ public RDBDocumentNodeStoreBuilder setFullGCEnabled(boolean b) {
+ // fullGC is non supported for RDB
+ return thisBuilder();
+ }
+
+ @Override
+ public RDBDocumentNodeStoreBuilder setDocStoreFullGCFeature(@Nullable Feature docStoreFullGC) {
+ return thisBuilder();
+ }
+
+ @Override
+ @Nullable
+ public Feature getDocStoreFullGCFeature() {
+ return null;
+ }
+
+ @Override
+ public boolean isEmbeddedVerificationEnabled() {
+ return false;
+ }
+
+ @Override
+ public RDBDocumentNodeStoreBuilder setEmbeddedVerificationEnabled(boolean b) {
+ // embeddedVerification is non supported for RDB since fullGC is not.
+ return thisBuilder();
+ }
+
+ @Override
+ public RDBDocumentNodeStoreBuilder setDocStoreEmbeddedVerificationFeature(@Nullable Feature getDocStoreEmbeddedVerification) {
+ return thisBuilder();
+ }
+
+ @Override
+ @Nullable
+ public Feature getDocStoreEmbeddedVerificationFeature() {
+ return null;
+ }
}
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java
index 0920eb056b..f4ba7c245c 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java
@@ -63,6 +63,7 @@ import org.slf4j.LoggerFactory;
import static org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull;
import static org.apache.jackrabbit.guava.common.collect.Iterables.transform;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MIN_ID_VALUE;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.isDeletedEntry;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.isCommitRootEntry;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.isRevisionsEntry;
@@ -751,7 +752,7 @@ public class Utils {
* @return an {@link Iterable} over all documents in the store.
*/
public static Iterable<NodeDocument> getAllDocuments(final DocumentStore store) {
- return internalGetSelectedDocuments(store, null, 0, DEFAULT_BATCH_SIZE);
+ return internalGetSelectedDocuments(store, null, 0, MIN_ID_VALUE, DEFAULT_BATCH_SIZE);
}
/**
@@ -793,7 +794,7 @@ public class Utils {
*/
public static Iterable<NodeDocument> getSelectedDocuments(
DocumentStore store, String indexedProperty, long startValue, int batchSize) {
- return internalGetSelectedDocuments(store, indexedProperty, startValue, batchSize);
+ return internalGetSelectedDocuments(store, indexedProperty, startValue, MIN_ID_VALUE, batchSize);
}
/**
@@ -802,12 +803,21 @@ public class Utils {
*/
public static Iterable<NodeDocument> getSelectedDocuments(
DocumentStore store, String indexedProperty, long startValue) {
- return internalGetSelectedDocuments(store, indexedProperty, startValue, DEFAULT_BATCH_SIZE);
+ return internalGetSelectedDocuments(store, indexedProperty, startValue, MIN_ID_VALUE, DEFAULT_BATCH_SIZE);
+ }
+
+ /**
+ * Like {@link #getSelectedDocuments(DocumentStore, String, long, int)} with
+ * a default {@code batchSize}.
+ */
+ public static Iterable<NodeDocument> getSelectedDocuments(
+ DocumentStore store, String indexedProperty, long startValue, String fromId) {
+ return internalGetSelectedDocuments(store, indexedProperty, startValue, fromId, DEFAULT_BATCH_SIZE);
}
private static Iterable<NodeDocument> internalGetSelectedDocuments(
final DocumentStore store, final String indexedProperty,
- final long startValue, final int batchSize) {
+ final long startValue, String fromId, final int batchSize) {
if (batchSize < 2) {
throw new IllegalArgumentException("batchSize must be > 1");
}
@@ -816,7 +826,7 @@ public class Utils {
public Iterator<NodeDocument> iterator() {
return new AbstractIterator<NodeDocument>() {
- private String startId = NodeDocument.MIN_ID_VALUE;
+ private String startId = fromId;
private Iterator<NodeDocument> batch = nextBatch();
@@ -977,6 +987,28 @@ public class Utils {
return builder.isThrottlingEnabled() || (docStoreThrottlingFeature != null && docStoreThrottlingFeature.isEnabled());
}
+ /**
+ * Check whether full GC is enabled or not for document store.
+ *
+ * @param builder instance for DocumentNodeStoreBuilder
+ * @return true if full GC is enabled else false
+ */
+ public static boolean isFullGCEnabled(final DocumentNodeStoreBuilder<?> builder) {
+ final Feature docStoreFullGCFeature = builder.getDocStoreFullGCFeature();
+ return builder.isFullGCEnabled() || (docStoreFullGCFeature != null && docStoreFullGCFeature.isEnabled());
+ }
+
+ /**
+ * Check whether embedded verification for full GC mode is enabled or not for document store.
+ *
+ * @param builder instance for DocumentNodeStoreBuilder
+ * @return true if embedded verification is enabled else false
+ */
+ public static boolean isEmbeddedVerificationEnabled(final DocumentNodeStoreBuilder<?> builder) {
+ final Feature docStoreEmbeddedVerificationFeature = builder.getDocStoreEmbeddedVerificationFeature();
+ return builder.isEmbeddedVerificationEnabled() || (docStoreEmbeddedVerificationFeature != null && docStoreEmbeddedVerificationFeature.isEnabled());
+ }
+
/**
* Returns true if all the revisions in the {@code a} greater or equals
* to their counterparts in {@code b}. If {@code b} contains revisions
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BranchCommitGCTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BranchCommitGCTest.java
index 2aca59d7af..8f9a338bf7 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BranchCommitGCTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BranchCommitGCTest.java
@@ -16,21 +16,54 @@
*/
package org.apache.jackrabbit.oak.plugins.document;
+import static java.util.concurrent.TimeUnit.HOURS;
+import static org.apache.commons.lang3.reflect.FieldUtils.writeField;
+import static org.apache.commons.lang3.reflect.FieldUtils.writeStaticField;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.assertBranchRevisionRemovedFromAllDocuments;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.build;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.allOrphProp;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.assertStatsCountsEqual;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.assertStatsCountsZero;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.betweenChkp;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.btwnChkpUBC;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.gapOrphOnly;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.gapOrphProp;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.isModeOneOf;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.keepOneFull;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.keepOneUser;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.unmergedBcs;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.SortedMap;
+import java.util.function.Consumer;
+
+import org.apache.jackrabbit.oak.plugins.document.DocumentMK.Builder;
+import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.FullGCMode;
+import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.GCCounts;
+import org.apache.jackrabbit.oak.plugins.document.util.Utils;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.stats.Clock;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.Rule;
-import org.junit.Before;
import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertEquals;
-import static java.util.concurrent.TimeUnit.HOURS;
-import static org.apache.jackrabbit.oak.plugins.document.TestUtils.persistToBranch;
-
+@RunWith(Parameterized.class)
public class BranchCommitGCTest {
@Rule
@@ -38,85 +71,696 @@ public class BranchCommitGCTest {
private Clock clock;
private DocumentNodeStore store;
private VersionGarbageCollector gc;
+ private VersionGarbageCollector.VersionGCStats stats;
+
+ @Parameterized.Parameters(name="{index}: {0} with {1}")
+ public static java.util.Collection<Object[]> params() throws IOException {
+ java.util.Collection<Object[]> params = new LinkedList<>();
+ for (Object[] fixture : AbstractDocumentStoreTest.fixtures()) {
+ DocumentStoreFixture f = (DocumentStoreFixture)fixture[0];
+ for (FullGCMode gcType : FullGCMode.values()) {
+ params.add(new Object[] {f, gcType});
+ }
+ }
+ return params;
+ }
+
+ @Parameter(0)
+ public DocumentStoreFixture fixture;
+
+ @Parameter(1)
+ public FullGCMode fullGcMode;
+
+ private FullGCMode originalFullGcMode;
@Before
- public void setUp() throws InterruptedException {
+ public void setUp() throws Exception {
clock = new Clock.Virtual();
clock.waitUntil(System.currentTimeMillis());
Revision.setClock(clock);
- store = builderProvider.newBuilder()
- .clock(clock)
- .setLeaseCheckMode(LeaseCheckMode.DISABLED)
- .setAsyncDelay(0)
- .getNodeStore();
+ store = builderProvider.newBuilder().clock(clock)
+ .setLeaseCheckMode(LeaseCheckMode.DISABLED).setFullGCEnabled(true)
+ .setAsyncDelay(0).getNodeStore();
gc = store.getVersionGarbageCollector();
+ originalFullGcMode = VersionGarbageCollector.getFullGcMode();
+ writeStaticField(VersionGarbageCollector.class, "fullGcMode", fullGcMode, true);
}
@After
public void tearDown() throws Exception {
+ writeStaticField(VersionGarbageCollector.class, "fullGcMode", originalFullGcMode, true);
+ assertNoEmptyProperties();
if (store != null) {
store.dispose();
}
Revision.resetClockToDefault();
}
- @Ignore
@Test
- public void orphanedBranchCommitDetect() throws Exception {
+ public void unmergedAddChildren() throws Exception {
+ RevisionVector br = unmergedBranchCommit(b -> {
+ b.child("a");
+ b.child("b");
+ });
+ assertExists("1:/a");
+ assertExists("1:/b");
+ assertFalse(store.getDocumentStore().find(Collection.NODES, "1:/a").wasDeletedOnce());
+ assertFalse(store.getDocumentStore().find(Collection.NODES, "1:/b").wasDeletedOnce());
+
+ store.runBackgroundOperations();
+
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ // clean everything older than one hour
+ VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS);
+
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(2, 0, 0, 0, 0, 0, 2),
+ keepOneFull(2, 0, 1, 0, 1, 0, 3),
+ keepOneUser(2, 0, 0, 0, 0, 0, 2),
+ betweenChkp(2, 0, 0, 0, 0, 0, 2),
+ unmergedBcs(2, 0, 1, 0, 1, 1, 3),
+ btwnChkpUBC(2, 0, 1, 0, 1, 1, 3));
+
+ // now do another gc - should not have anything left to clean up though
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ stats = gc.gc(1, HOURS);
+ assertStatsCountsZero(stats);
+ if (!isModeOneOf(FullGCMode.NONE, FullGCMode.GAP_ORPHANS, FullGCMode.GAP_ORPHANS_EMPTYPROPS)) {
+ assertNotExists("1:/a");
+ assertNotExists("1:/b");
+ assertBranchRevisionRemovedFromAllDocuments(store, br);
+ }
+ }
+
+ @Test
+ public void unmergedAddThenMergedAddAndRemoveChildren() throws Exception {
+ RevisionVector br = unmergedBranchCommit(b -> {
+ b.child("a");
+ b.child("b");
+ });
+ assertExists("1:/a");
+ assertExists("1:/b");
+ assertFalse(store.getDocumentStore().find(Collection.NODES, "1:/a").wasDeletedOnce());
+ assertFalse(store.getDocumentStore().find(Collection.NODES, "1:/b").wasDeletedOnce());
+
+ store.runBackgroundOperations();
+
+ mergedBranchCommit(b -> {
+ b.child("a");
+ b.child("b");
+ });
+
+ assertEquals(1, countCollisionsOnRoot());
+
+ store.runBackgroundOperations();
+
+ mergedBranchCommit(b -> {
+ b.child("a").remove();
+ b.child("b").remove();
+ });
+
+ store.runBackgroundOperations();
+
+ int collisionsBeforeGC = countCollisionsOnRoot();
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ // clean everything older than one hour
+ VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS);
+
+ if (collisionsBeforeGC == 1) {
+ // expects a collision to have happened - which was cleaned up - hence a _bc (but not the _revision I guess)
+ // the collisions cleaned up - with the 1 collision and a _bc
+ assertStatsCountsEqual(stats,
+ new GCCounts(FullGCMode.NONE,
+ 2, 0, 0, 0, 0, 0, 0),
+ gapOrphOnly(2, 0, 0, 0, 0, 0, 0),
+ gapOrphProp(2, 0, 0, 0, 0, 0, 0),
+ allOrphProp(2, 0, 0, 0, 0, 0, 0),
+ keepOneFull(2, 0, 1, 0, 2, 0, 1),
+ keepOneUser(2, 0, 0, 0, 0, 0, 0),
+ betweenChkp(2, 0, 0, 0, 0, 0, 0),
+ unmergedBcs(2, 0, 1, 0, 2, 1, 1),
+ btwnChkpUBC(2, 0, 1, 0, 2, 1, 1));
+ } else {
+ // in this case classic collision cleanup already took care of everything, nothing left
+ assertStatsCountsEqual(stats,
+ new GCCounts(FullGCMode.NONE,
+ 2, 0, 0, 0, 0, 0, 0),
+ gapOrphOnly(2, 0, 0, 0, 0, 0, 0),
+ gapOrphProp(2, 0, 0, 0, 0, 0, 0),
+ allOrphProp(2, 0, 0, 0, 0, 0, 0),
+ keepOneFull(2, 1, 0, 0, 0, 0, 0),
+ keepOneUser(2, 1, 0, 0, 0, 0, 0),
+ betweenChkp(2, 0, 0, 0, 0, 0, 0),
+ unmergedBcs(2, 0, 0, 0, 0, 0, 0),
+ btwnChkpUBC(2, 0, 0, 0, 0, 0, 0));
+ }
+
+ assertNotExists("1:/a");
+ assertNotExists("1:/b");
+
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ stats = gc.gc(1, HOURS);
+ assertEquals(0, stats.updatedFullGCDocsCount);
+ assertEquals(0, stats.deletedDocGCCount);
+ assertEquals(0, stats.deletedUnmergedBCCount);
+ assertBranchRevisionRemovedFromAllDocuments(store, br);
+ }
+
+ private int countCollisionsOnRoot() {
+ NodeDocument r = store.getDocumentStore().find(Collection.NODES, "0:/");
+ SortedMap<Revision, String> colls = r.getLocalMap(NodeDocument.COLLISIONS);
+ int countCollisions = colls.size();
+ return countCollisions;
+ }
+
+ @Test
+ public void testDeletedPropsAndUnmergedBC() throws Exception {
+ // create a node with property.
+ NodeBuilder nb = store.getRoot().builder();
+ nb.child("bar").setProperty("prop", "value");
+ nb.child("bar").setProperty("x", "y");
+ store.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store.runBackgroundOperations();
+
+ // remove the property
+ nb = store.getRoot().builder();
+ nb.child("bar").removeProperty("prop");
+ store.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store.runBackgroundOperations();
+
+ // create branch commits
+ mergedBranchCommit(b -> b.child("foo").setProperty("p", "prop"));
+ RevisionVector br1 = unmergedBranchCommit(b -> b.child("foo").setProperty("a", "b"));
+ RevisionVector br2 = unmergedBranchCommit(b -> b.child("foo").setProperty("a", "c"));
+ RevisionVector br3 = unmergedBranchCommit(b -> b.child("foo").setProperty("a", "d"));
+ RevisionVector br4 = unmergedBranchCommit(b -> b.child("bar").setProperty("x", "z"));
+ mergedBranchCommit(b -> b.child("foo").removeProperty("p"));
+ store.runBackgroundOperations();
+
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ // clean everything older than one hour
+ VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS);
+
+ // 6 deleted props: 0:/[_collisions], 1:/foo[p, a], 1:/bar[_bc,prop,_revisions]
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 3, 0, 0, 0, 0, 2),
+ allOrphProp(0, 3, 0, 0, 0, 0, 2),
+ keepOneFull(0, 3, 2, 1,17, 0, 3),
+ keepOneUser(0, 3, 0, 1, 0, 0, 2),
+ betweenChkp(0, 3, 0, 0, 1, 0, 3),
+ unmergedBcs(0, 3, 2, 1,15, 4, 3),
+ btwnChkpUBC(0, 3, 2, 1,16, 4, 3));
+ if (!isModeOneOf(FullGCMode.NONE, FullGCMode.GAP_ORPHANS)) {
+ assertBranchRevisionRemovedFromAllDocuments(store, br1);
+ assertBranchRevisionRemovedFromAllDocuments(store, br2);
+ assertBranchRevisionRemovedFromAllDocuments(store, br3);
+ assertBranchRevisionRemovedFromAllDocuments(store, br4);
+ }
+ }
+
+ @Test
+ public void unmergedAddTwoChildren() throws Exception {
+ RevisionVector br1 = unmergedBranchCommit(b -> {
+ b.child("a");
+ b.child("b");
+ });
+ RevisionVector br2 = unmergedBranchCommit(b -> {
+ b.child("a");
+ b.child("b");
+ });
+ assertExists("1:/a");
+ assertExists("1:/b");
+ assertFalse(store.getDocumentStore().find(Collection.NODES, "1:/a").wasDeletedOnce());
+ assertFalse(store.getDocumentStore().find(Collection.NODES, "1:/b").wasDeletedOnce());
+
+ store.runBackgroundOperations();
+
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ // clean everything older than one hour
+ VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS);
+
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(2, 0, 0, 0, 0, 0, 2),
+ keepOneFull(2, 0, 2, 0, 2, 0, 3),
+ keepOneUser(2, 0, 0, 0, 0, 0, 2),
+ betweenChkp(2, 0, 0, 0, 0, 0, 2),
+ unmergedBcs(2, 0, 2, 0, 2, 2, 3),
+ btwnChkpUBC(2, 0, 2, 0, 2, 2, 3));
+
+ if (!isModeOneOf(FullGCMode.NONE, FullGCMode.GAP_ORPHANS, FullGCMode.GAP_ORPHANS_EMPTYPROPS)) {
+ assertNotExists("1:/a");
+ assertNotExists("1:/b");
+ }
+
+ // now do another gc - should not have anything left to clean up though
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ stats = gc.gc(1, HOURS);
+ assertStatsCountsZero(stats);
+ if (!isModeOneOf(FullGCMode.NONE)) {
+ assertBranchRevisionRemovedFromAllDocuments(store, br1);
+ assertBranchRevisionRemovedFromAllDocuments(store, br2);
+ }
+ }
+
+ @Test
+ public void unmergedAddsThenMergedAddsChildren() throws Exception {
+ RevisionVector br1 = unmergedBranchCommit(b -> {
+ b.child("a");
+ b.child("b");
+ });
+ RevisionVector br2 = unmergedBranchCommit(b -> {
+ b.child("a");
+ b.child("b");
+ });
+ assertExists("1:/a");
+ assertExists("1:/b");
+ assertFalse(store.getDocumentStore().find(Collection.NODES, "1:/a").wasDeletedOnce());
+ assertFalse(store.getDocumentStore().find(Collection.NODES, "1:/b").wasDeletedOnce());
+
+ store.runBackgroundOperations();
+
+ mergedBranchCommit(b -> {
+ b.child("a");
+ b.child("b");
+ });
+
+ store.runBackgroundOperations();
+
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ // clean everything older than one hour
+ VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS);
+
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
+ keepOneFull(0, 0, 1, 4,12, 0, 3),
+ keepOneUser(0, 0, 0, 4, 0, 0, 2),
+ betweenChkp(),
+ unmergedBcs(0, 0, 1, 0,16, 2, 3),
+ btwnChkpUBC(0, 0, 1, 0,16, 2, 3));
+ assertExists("1:/a");
+ assertExists("1:/b");
+
+ // now do another gc to get those documents actually deleted
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ stats = gc.gc(1, HOURS);
+ assertStatsCountsZero(stats);
+ if (!isModeOneOf(FullGCMode.NONE)) {
+ assertBranchRevisionRemovedFromAllDocuments(store, br1);
+ assertBranchRevisionRemovedFromAllDocuments(store, br2);
+ }
+ }
+
+ @Test
+ public void unmergedAddsThenMergedAddThenUnmergedRemovesChildren() throws Exception {
+ RevisionVector br1 = unmergedBranchCommit(b -> {
+ b.child("a");
+ b.child("b");
+ });
+ RevisionVector br2 = unmergedBranchCommit(b -> {
+ b.child("a");
+ b.child("b");
+ });
+ assertExists("1:/a");
+ assertExists("1:/b");
+ assertFalse(store.getDocumentStore().find(Collection.NODES, "1:/a").wasDeletedOnce());
+ assertFalse(store.getDocumentStore().find(Collection.NODES, "1:/b").wasDeletedOnce());
+
+ store.runBackgroundOperations();
+
+ mergedBranchCommit(b -> {
+ b.child("a");
+ b.child("b");
+ });
+
+ store.runBackgroundOperations();
+
+ RevisionVector br3 = unmergedBranchCommit(b -> {
+ b.child("a").remove();
+ b.child("b").remove();
+ });
+ RevisionVector br4 = unmergedBranchCommit(b -> {
+ b.child("a").remove();
+ b.child("b").remove();
+ });
+
+ store.runBackgroundOperations();
+
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ // clean everything older than one hour
+ VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS);
+
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(0, 0, 0, 0, 0, 0, 0),
+ gapOrphProp(0, 0, 0, 0, 0, 0, 0),
+ allOrphProp(0, 0, 0, 0, 0, 0, 0),
+ keepOneFull(0, 0, 1, 8,24, 0, 3),
+ keepOneUser(0, 0, 0, 8, 0, 0, 2),
+ betweenChkp(),
+ unmergedBcs(0, 0, 1, 0,32, 4, 3),
+ btwnChkpUBC(0, 0, 1, 0,32, 4, 3));
+ assertExists("1:/a");
+ assertExists("1:/b");
+
+ // now do another gc to get those documents actually deleted
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ stats = gc.gc(1, HOURS);
+ assertStatsCountsZero(stats);
+ if (!isModeOneOf(FullGCMode.NONE)) {
+ assertBranchRevisionRemovedFromAllDocuments(store, br1);
+ assertBranchRevisionRemovedFromAllDocuments(store, br2);
+ assertBranchRevisionRemovedFromAllDocuments(store, br3);
+ assertBranchRevisionRemovedFromAllDocuments(store, br4);
+ }
+ }
+
+ @Test
+ public void unmergedAddAndRemoveChild() throws Exception {
+ mergedBranchCommit(b -> {
+ b.child("foo");
+ b.child("test");
+ });
+ RevisionVector br = unmergedBranchCommit(b -> {
+ b.child("test").remove();
+ b.getChildNode("foo").child("childFoo");
+ });
+ store.runBackgroundOperations();
+
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ // clean everything older than one hour
+ VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS);
+
+ // first gc round now deletes it, via orphaned node deletion
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(1, 0, 0, 0, 0, 0, 1),
+ keepOneFull(1, 0, 0, 1, 6, 0, 4),
+ keepOneUser(1, 0, 0, 1, 0, 0, 2),
+ betweenChkp(1, 0, 0, 0, 0, 0, 1),
+ unmergedBcs(1, 0, 0, 0, 7, 1, 4),
+ btwnChkpUBC(1, 0, 0, 0, 7, 1, 4));
+
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ // now do second gc round - should not have anything left to clean up though
+ stats = gc.gc(1, HOURS);
+ assertStatsCountsZero(stats);
+ if (!isModeOneOf(FullGCMode.NONE)) {
+ assertBranchRevisionRemovedFromAllDocuments(store, br);
+ }
+ }
+
+ @Test
+ public void unmergedRemoveProperty() throws Exception {
+ mergedBranchCommit(b -> b.child("foo"));
+ mergedBranchCommit(b -> b.child("foo").setProperty("a", "b"));
+ // do a gc without waiting, to check that works fine
+ store.runBackgroundOperations();
+ stats = gc.gc(1, HOURS);
+ assertEquals(0, stats.updatedFullGCDocsCount);
+
+ RevisionVector br = unmergedBranchCommit(b -> b.child("foo").removeProperty("a"));
+ mergedBranchCommit(b -> b.child("foo").setProperty("c", "d"));
+ store.runBackgroundOperations();
+
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ // clean everything older than one hour
+ stats = gc.gc(1, HOURS);
+
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
+ keepOneFull(0, 0, 0, 1, 4, 0, 2),
+ keepOneUser(0, 0, 0, 1, 0, 0, 1),
+ betweenChkp(),
+ unmergedBcs(0, 0, 0, 1, 4, 1, 2),
+ btwnChkpUBC(0, 0, 0, 1, 4, 1, 2));
+ if (!isModeOneOf(FullGCMode.NONE)) {
+ assertBranchRevisionRemovedFromAllDocuments(store, br);
+ }
+ }
+
+ @Test
+ public void unmergedAddProperty() throws Exception {
+ mergedBranchCommit(b -> b.child("foo"));
+ RevisionVector br = unmergedBranchCommit(b -> b.child("foo").setProperty("a", "b"));
+ store.runBackgroundOperations();
+
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ // clean everything older than one hour
+ stats = gc.gc(1, HOURS);
- NodeBuilder b1 = store.getRoot().builder();
- b1.child("a");
- b1.child("b");
- persistToBranch(b1);
+ // 1 deleted prop: 1:/foo[a]
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 1, 0, 0, 0, 0, 1),
+ allOrphProp(0, 1, 0, 0, 0, 0, 1),
+ keepOneFull(0, 1, 0, 0, 4, 0, 2),
+ keepOneUser(0, 1, 0, 0, 0, 0, 1),
+ betweenChkp(0, 1, 0, 0, 0, 0, 1),
+ unmergedBcs(0, 1, 0, 0, 4, 1, 2),
+ btwnChkpUBC(0, 1, 0, 0, 4, 1, 2));
+ if (!isModeOneOf(FullGCMode.NONE)) {
+ assertBranchRevisionRemovedFromAllDocuments(store, br);
+ }
+ }
- // b1 must see 'a' and 'b'
- assertTrue(b1.hasChildNode("a"));
- assertTrue(b1.hasChildNode("b"));
+ @Test
+ public void unmergedRemoveChild() throws Exception {
+ mergedBranchCommit(b -> {
+ b.child("foo");
+ b.child("bar");
+ });
+ // do a gc without waiting, to check that works fine
+ store.runBackgroundOperations();
+ stats = gc.gc(1, HOURS);
+ assertEquals(0, stats.updatedFullGCDocsCount);
+ assertEquals(0, stats.deletedUnmergedBCCount);
+ final List<RevisionVector> brs = new LinkedList<>();
+ for (int j = 0; j < 10; j++) {
+ brs.add(unmergedBranchCommit(b -> b.child("foo").remove()));
+ }
store.runBackgroundOperations();
+ NodeDocument doc = store.getDocumentStore().find(Collection.NODES, "1:/foo");
+ Long originalModified = doc.getModified();
+
// wait two hours
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
- // clean everything older than one hours
- VersionGarbageCollector.VersionGCStats stats= gc.gc(1, HOURS);
+ // clean everything older than one hour
+ stats = gc.gc(1, HOURS);
+
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(0, 0, 0, 0, 0, 0, 0),
+ gapOrphProp(0, 0, 0, 0, 0, 0, 0),
+ allOrphProp(0, 0, 0, 0, 0, 0, 0),
+ keepOneFull(0, 0, 1,10,40, 0, 2),
+ keepOneUser(0, 0, 0,10, 0, 0, 1),
+ betweenChkp(),
+ unmergedBcs(0, 0, 1, 0,50,10, 2),
+ btwnChkpUBC(0, 0, 1, 0,50,10, 2));
- //This will fail as of now but will pass once BranchCommit GC code is merged.
- assertEquals(1, stats.deletedDocGCCount);
+ doc = store.getDocumentStore().find(Collection.NODES, "1:/foo");
+ Long finalModified = doc.getModified();
+
+ assertEquals(originalModified, finalModified);
+
+ if (!isModeOneOf(FullGCMode.NONE)) {
+ for (RevisionVector br : brs) {
+ assertBranchRevisionRemovedFromAllDocuments(store, br);
+ }
+ }
}
- @Ignore
@Test
- public void orphanedModifiedBranchCommitDetect() throws Exception {
+ public void unmergedMergedRemoveChild() throws Exception {
+ mergedBranchCommit(b -> {
+ b.child("foo");
+ b.child("bar");
+ });
+ // do a gc without waiting, to check that works fine
+ store.runBackgroundOperations();
+ stats = gc.gc(1, HOURS);
+ assertEquals(0, stats.updatedFullGCDocsCount);
+ assertEquals(0, stats.deletedUnmergedBCCount);
- NodeBuilder b = store.getRoot().builder();
- b.child("foo");
- b.child("test");
- persistToBranch(b);
+ for (int i = 0; i < 50; i++) {
+ mergedBranchCommit(b -> b.child("foo").remove());
+ mergedBranchCommit(b -> b.child("foo"));
+ }
+ final List<RevisionVector> brs = new LinkedList<>();
+ for (int j = 0; j < 10; j++) {
+ brs.add(unmergedBranchCommit(b -> b.child("foo").remove()));
+ }
+ store.runBackgroundOperations();
- // b must see 'foo' and 'test'
- assertTrue(b.hasChildNode("foo"));
- assertTrue(b.hasChildNode("test"));
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ // clean everything older than one hour
+ stats = gc.gc(1, HOURS);
+
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(0, 0, 0, 0, 0, 0, 0),
+ gapOrphProp(0, 0, 0, 0, 0, 0, 0),
+ allOrphProp(0, 0, 0, 0, 0, 0, 0),
+ keepOneFull(0, 0, 2,10,30, 0, 2),
+ keepOneUser(0, 0, 0,10, 0, 0, 1),
+ betweenChkp(),
+ unmergedBcs(0, 0, 2, 0,40,10, 2),
+ btwnChkpUBC(0, 0, 2, 0,40,10, 2));
+ if (!isModeOneOf(FullGCMode.NONE)) {
+ for (RevisionVector br : brs) {
+ assertBranchRevisionRemovedFromAllDocuments(store, br);
+ }
+ }
+ }
- merge(b);
+ @Test
+ public void unmergedThenMergedRemoveProperty() throws Exception {
+ mergedBranchCommit(b -> b.child("foo"));
+ mergedBranchCommit(b -> b.child("foo").setProperty("a", "b"));
+ mergedBranchCommit(b -> b.child("foo").setProperty("c", "d"));
+ // do a gc without waiting, to check that works fine
+ store.runBackgroundOperations();
+ stats = gc.gc(1, HOURS);
+ assertEquals(0, stats.updatedFullGCDocsCount);
+ assertEquals(0, stats.deletedUnmergedBCCount);
+ assertEquals(0, stats.deletedPropsCount);
+ assertEquals(0, stats.deletedPropRevsCount);
- b = store.getRoot().builder();
- b.child("test").remove();
- b.getChildNode("foo").child("childfoo");
- persistToBranch(b);
+ RevisionVector br = unmergedBranchCommit(b -> {
+ b.setProperty("rootProp", "v");
+ b.child("foo").removeProperty("a");
+ });
store.runBackgroundOperations();
+ DocumentNodeStore store2 = newStore(2);
+ FullGCHelper.mergedBranchCommit(store2, b -> b.child("foo").removeProperty("a"));
+ store2.runBackgroundOperations();
+ store2.dispose();
+ store.runBackgroundOperations();
+
+ {
+ // some flakyness diagnostics
+ NodeDocument d = store.getDocumentStore().find(Collection.NODES, "0:/", -1);
+ assertNotNull(d);
+ assertEquals(1, d.getLocalMap("rootProp").size());
+ assertEquals(1, d.getLocalMap("_collisions").size());
+ d = store.getDocumentStore().find(Collection.NODES, "1:/foo", -1);
+ assertNotNull(d);
+ assertEquals(3, d.getLocalMap("a").size());
+ }
// wait two hours
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
- // clean everything older than one hours
- VersionGarbageCollector.VersionGCStats stats= gc.gc(1, HOURS);
+ // clean everything older than one hour
+ stats = gc.gc(1, HOURS);
- //This will fail as of now but will pass once BranchCommit GC code is merged.
- assertEquals(1, stats.deletedDocGCCount);
+ // deleted props: 0:/[rootProp], 1:/foo[a]
+ // deleted internal prop : 0:/ _collision
+ // deleted properties are 0:/ -> rootProp, _collisions & 1:/foo -> a
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 2, 0, 0, 0, 0, 2),
+ allOrphProp(0, 2, 0, 0, 0, 0, 2),
+ keepOneFull(0, 2, 1, 0, 8, 0, 2),
+ keepOneUser(0, 2, 0, 0, 0, 0, 2),
+ betweenChkp(0, 2, 0, 0, 0, 0, 2),
+ unmergedBcs(0, 2, 1, 0, 4, 1, 2),
+ btwnChkpUBC(0, 2, 1, 0, 4, 1, 2));
+ if (isModeOneOf(FullGCMode.NONE, FullGCMode.GAP_ORPHANS)) {
+ return;
+ }
+
+ {
+ // some flakyness diagnostics
+ NodeDocument d = store.getDocumentStore().find(Collection.NODES, "0:/", -1);
+ assertNotNull(d);
+ assertEquals(0, d.getLocalMap("rootProp").size());
+ if (VersionGarbageCollector.getFullGcMode() == FullGCMode.ORPHANS_EMPTYPROPS_KEEP_ONE_ALL_PROPS
+ || VersionGarbageCollector.getFullGcMode() == FullGCMode.ORPHANS_EMPTYPROPS_UNMERGED_BC
+ || VersionGarbageCollector.getFullGcMode() == FullGCMode.ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_WITH_UNMERGED_BC) {
+ assertEquals(0, d.getLocalMap("_collisions").size());
+ } else {
+ assertEquals(1, d.getLocalMap("_collisions").size());
+ }
+ d = store.getDocumentStore().find(Collection.NODES, "1:/foo", -1);
+ assertNotNull(d);
+ assertEquals(0, d.getLocalMap("a").size());
+ }
+ assertBranchRevisionRemovedFromAllDocuments(store, br);
}
- private void merge(NodeBuilder builder)
- throws Exception {
- store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ // helper methods
+
+ private DocumentNodeStore newStore(int clusterId) {
+ Builder builder = builderProvider.newBuilder().clock(clock)
+ .setLeaseCheckMode(LeaseCheckMode.DISABLED).setFullGCEnabled(true)
+ .setAsyncDelay(0).setDocumentStore(store.getDocumentStore());
+ if (clusterId > 0) {
+ builder.setClusterId(clusterId);
+ }
+ return builder.getNodeStore();
+ }
+
+ private RevisionVector mergedBranchCommit(Consumer<NodeBuilder> buildFunction) throws Exception {
+ return build(store, true, true, buildFunction);
+ }
+
+ private RevisionVector unmergedBranchCommit(Consumer<NodeBuilder> buildFunction) throws Exception {
+ RevisionVector result = build(store, true, false, buildFunction);
+ assertTrue(result.isBranch());
+ return result;
+ }
+
+ private void assertNotExists(String id) {
+ NodeDocument doc = store.getDocumentStore().find(Collection.NODES, id);
+ assertNull("doc exists but was expected not to : id=" + id, doc);
+ }
+
+ private void assertExists(String id) {
+ NodeDocument doc = store.getDocumentStore().find(Collection.NODES, id);
+ assertNotNull("doc does not exist : id=" + id, doc);
+ }
+
+ private void assertNoEmptyProperties() {
+ for (NodeDocument nd : Utils.getAllDocuments(store.getDocumentStore())) {
+ for (Entry<String, Object> e : nd.data.entrySet()) {
+ Object v = e.getValue();
+ if (v instanceof Map) {
+ @SuppressWarnings("rawtypes")
+ Map m = (Map) v;
+ if (m.isEmpty() && (e.getKey().equals("_commitRoot") || e.getKey().equals("_collisions"))
+ && Objects.equals(nd.getId(), "0:/")) {
+ // skip : root document apparently has an empty _commitRoot:{}
+ continue;
+ }
+ assertFalse("document has empty property : id=" + nd.getId() +
+ ", property=" + e.getKey() + ", document=" + nd.asString(), m.isEmpty());
+ }
+ }
+ }
}
}
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceConfigurationTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceConfigurationTest.java
index 3576ed6ab6..c15e1f4cbb 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceConfigurationTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceConfigurationTest.java
@@ -33,6 +33,8 @@ import org.osgi.framework.BundleContext;
import org.osgi.service.cm.ConfigurationAdmin;
import org.osgi.service.component.ComponentContext;
+import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FULL_GC_ENABLED;
+import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_EMBEDDED_VERIFICATION_ENABLED;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_THROTTLING_ENABLED;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
@@ -84,6 +86,8 @@ public class DocumentNodeStoreServiceConfigurationTest {
assertEquals(Arrays.asList("/"), Arrays.asList(config.persistentCacheIncludes()));
assertEquals("STRICT", config.leaseCheckMode());
assertEquals(DEFAULT_THROTTLING_ENABLED, config.throttlingEnabled());
+ assertEquals(DEFAULT_FULL_GC_ENABLED, config.fullGCEnabled());
+ assertEquals(DEFAULT_EMBEDDED_VERIFICATION_ENABLED, config.embeddedVerificationEnabled());
assertEquals(CommitQueue.DEFAULT_SUSPEND_TIMEOUT, config.suspendTimeoutMillis());
}
@@ -103,6 +107,22 @@ public class DocumentNodeStoreServiceConfigurationTest {
assertEquals(throttleDocStore, config.throttlingEnabled());
}
+ @Test
+ public void fullGCEnabled() throws Exception {
+ boolean fullGCDocStore = true;
+ addConfigurationEntry(preset, "fullGCEnabled", fullGCDocStore);
+ Configuration config = createConfiguration();
+ assertEquals(fullGCDocStore, config.fullGCEnabled());
+ }
+
+ @Test
+ public void embeddedVerificationEnabled() throws Exception {
+ boolean embeddedVerificationEnabled = false;
+ addConfigurationEntry(preset, "embeddedVerificationEnabled", embeddedVerificationEnabled);
+ Configuration config = createConfiguration();
+ assertEquals(embeddedVerificationEnabled, config.embeddedVerificationEnabled());
+ }
+
@Test
public void presetSocketKeepAlive() throws Exception {
boolean keepAlive = !DocumentNodeStoreService.DEFAULT_SO_KEEP_ALIVE;
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/FailingDocumentStore.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/FailingDocumentStore.java
index a466d377ab..adfccbe0ee 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/FailingDocumentStore.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/FailingDocumentStore.java
@@ -55,6 +55,8 @@ class FailingDocumentStore extends DocumentStoreWrapper {
private boolean afterOp = false;
+ private boolean noDispose = false;
+
class Fail {
private Fail() {
@@ -309,4 +311,16 @@ class FailingDocumentStore extends DocumentStoreWrapper {
private void reportRemainingOps(List<UpdateOp> remainingOps) {
listeners.forEach(listener -> remainingOps.forEach(listener::failed));
}
+
+ public void noDispose() {
+ noDispose = true;
+ }
+
+ @Override
+ public void dispose() {
+ if (!noDispose) {
+ super.dispose();
+ }
+ }
+
}
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/FullGCHelper.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/FullGCHelper.java
new file mode 100644
index 0000000000..8b3b3de4ce
--- /dev/null
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/FullGCHelper.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.FullGCMode;
+import org.apache.jackrabbit.oak.plugins.document.util.Utils;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Consumer;
+
+import static org.apache.commons.lang3.reflect.FieldUtils.writeField;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class FullGCHelper {
+
+ public static void enableFullGC(final VersionGarbageCollector vgc) throws IllegalAccessException {
+ if (vgc != null) {
+ writeField(vgc, "fullGCEnabled", true, true);
+ }
+ }
+
+ public static void disableFullGC(final VersionGarbageCollector vgc) throws IllegalAccessException {
+ if (vgc != null) {
+ writeField(vgc, "fullGCEnabled", false, true);
+ }
+ }
+ public static void enableFullGCDryRun(final VersionGarbageCollector vgc) throws IllegalAccessException {
+ if (vgc != null) {
+ writeField(vgc, "isFullGCDryRun", true, true);
+ }
+ }
+
+ public static void disableFullGCDryRun(final VersionGarbageCollector vgc) throws IllegalAccessException {
+ if (vgc != null) {
+ writeField(vgc, "isFullGCDryRun", false, true);
+ }
+ }
+
+ public static RevisionVector mergedBranchCommit(final NodeStore store, final Consumer<NodeBuilder> buildFunction) throws Exception {
+ return build(store, true, true, buildFunction);
+ }
+
+ public static RevisionVector unmergedBranchCommit(final NodeStore store, final Consumer<NodeBuilder> buildFunction) throws Exception {
+ RevisionVector result = build(store, true, false, buildFunction);
+ assertTrue(result.isBranch());
+ return result;
+ }
+
+ public static void assertBranchRevisionRemovedFromAllDocuments(
+ final DocumentNodeStore store, final RevisionVector br,
+ final String... exceptIds) {
+ assertTrue(br.isBranch());
+ if (VersionGarbageCollector.getFullGcMode() == FullGCMode.GAP_ORPHANS
+ || VersionGarbageCollector.getFullGcMode() == FullGCMode.GAP_ORPHANS_EMPTYPROPS
+ || VersionGarbageCollector.getFullGcMode() == FullGCMode.ALL_ORPHANS_EMPTYPROPS
+ || VersionGarbageCollector.getFullGcMode() == FullGCMode.ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_NO_UNMERGED_BC) {
+ // then we must skip these asserts, as we cannot guarantee
+ // that all revisions are cleaned up in this mode
+ return;
+ }
+ Revision br1 = br.getRevision(1);
+ assert br1 != null;
+ Revision r1 = br1.asTrunkRevision();
+ Set<String> except = Set.of(exceptIds);
+ for (NodeDocument nd : Utils.getAllDocuments(store.getDocumentStore())) {
+ if (nd.getId().equals(Utils.getIdFromPath(Path.ROOT))
+ || except.contains(nd.getId())) {
+ // skip root and the provided ids
+ continue;
+ }
+ NodeDocument target = new NodeDocument(store.getDocumentStore());
+ nd.deepCopy(target);
+ // ignore the _revisions entry, as we cannot always cleanup everything in there
+ target.remove(NodeDocument.REVISIONS);
+ for (Entry<String, Object> e : target.data.entrySet()) {
+ String k = e.getKey();
+ final boolean internal = k.startsWith("_");
+ final boolean dgcSupportsInternalPropCleanup = (VersionGarbageCollector.getFullGcMode() != FullGCMode.ORPHANS_EMPTYPROPS_KEEP_ONE_USER_PROPS);
+ if (internal && !dgcSupportsInternalPropCleanup) {
+ // skip
+ continue;
+ }
+ assertFalse("document not cleaned up for prop " + k + " : " + e,
+ e.toString().contains(r1.toString()));
+ }
+ }
+ }
+
+ public static void assertBranchRevisionNotRemovedFromAllDocuments(final DocumentNodeStore store, final RevisionVector br) {
+ assertTrue(br.isBranch());
+ Revision br1 = br.getRevision(1);
+ assert br1 != null;
+ Revision r1 = br1.asTrunkRevision();
+ for (NodeDocument nd : Utils.getAllDocuments(store.getDocumentStore())) {
+ if (Objects.equals(nd.getId(), "0:/")) {
+ NodeDocument target = new NodeDocument(store.getDocumentStore());
+ nd.deepCopy(target);
+ assertTrue("document not cleaned up: " + nd.asString(), nd.asString().contains(r1.toString()));
+ }
+ }
+ }
+
+ static RevisionVector build(final NodeStore store, final boolean persistToBranch, final boolean merge,
+ final Consumer<NodeBuilder> buildFunction) throws Exception {
+ if (!persistToBranch && !merge) {
+ throw new IllegalArgumentException("must either persistToBranch or merge");
+ }
+ NodeBuilder b = store.getRoot().builder();
+ buildFunction.accept(b);
+ RevisionVector result = null;
+ if (persistToBranch) {
+ DocumentRootBuilder drb = (DocumentRootBuilder) b;
+ drb.persist();
+ DocumentNodeState ns = (DocumentNodeState) drb.getNodeState();
+ result = ns.getLastRevision();
+ }
+ if (merge) {
+ DocumentNodeState dns = (DocumentNodeState) merge(store, b);
+ result = dns.getLastRevision();
+ }
+ return result;
+ }
+
+ private static NodeState merge(final NodeStore store, final NodeBuilder builder) throws Exception {
+ return store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ }
+}
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/FullGCStatsCollectorImplTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/FullGCStatsCollectorImplTest.java
new file mode 100644
index 0000000000..6c46684749
--- /dev/null
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/FullGCStatsCollectorImplTest.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
+import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
+import org.apache.jackrabbit.oak.plugins.metric.MetricStatisticsProvider;
+import org.apache.jackrabbit.oak.stats.MeterStats;
+import org.junit.After;
+import org.junit.Test;
+
+import java.util.concurrent.ScheduledExecutorService;
+
+import static java.lang.management.ManagementFactory.getPlatformMBeanServer;
+import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.commons.lang3.reflect.FieldUtils.readField;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCStatsCollectorImpl.COLLECT_DELETED_OLD_REVS_TIMER;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCStatsCollectorImpl.COLLECT_DELETED_PROPS_TIMER;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCStatsCollectorImpl.COLLECT_FULL_GC_TIMER;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCStatsCollectorImpl.COLLECT_ORPHAN_NODES_TIMER;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCStatsCollectorImpl.COLLECT_UNMERGED_BC_TIMER;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCStatsCollectorImpl.COUNTER;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCStatsCollectorImpl.DELETED_ORPHAN_NODE;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCStatsCollectorImpl.DELETED_PROPERTY;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCStatsCollectorImpl.DELETED_UNMERGED_BC;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCStatsCollectorImpl.DELETE_FULL_GC_DOCS_TIMER;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCStatsCollectorImpl.FULL_GC;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCStatsCollectorImpl.FULL_GC_ACTIVE_TIMER;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCStatsCollectorImpl.FULL_GC_TIMER;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCStatsCollectorImpl.FAILURE_COUNTER;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCStatsCollectorImpl.READ_DOC;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCStatsCollectorImpl.SKIPPED_DOC;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCStatsCollectorImpl.UPDATED_DOC;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Unit Cases for {@link FullGCStatsCollectorImpl}
+ */
+public class FullGCStatsCollectorImplTest {
+
+ private final ScheduledExecutorService executor = newSingleThreadScheduledExecutor();
+ private final MetricStatisticsProvider statsProvider = new MetricStatisticsProvider(getPlatformMBeanServer(), executor);
+ private final FullGCStatsCollectorImpl stats = new FullGCStatsCollectorImpl(statsProvider);
+
+ @After
+ public void shutDown(){
+ statsProvider.close();
+ new ExecutorCloser(executor).close();
+ }
+
+ @Test
+ public void getReadDocCount() throws IllegalAccessException {
+ final Meter m = getMeter(READ_DOC);
+ long count = m.getCount();
+ stats.documentRead();
+ assertEquals(count + 1, m.getCount());
+ assertEquals(count + 1, ((MeterStats) readField(stats, "readDoc", true)).getCount());
+ }
+
+ @Test
+ public void getDocumentsSkippedUpdationCount() throws IllegalAccessException {
+ Meter m = getMeter(SKIPPED_DOC);
+ long count = m.getCount();
+ stats.documentsUpdateSkipped(17);
+ assertEquals(count + 17, m.getCount());
+ assertEquals(count + 17, ((MeterStats) readField(stats, "skippedDoc", true)).getCount());
+ }
+
+ @Test
+ public void getOrphanNodesDeletedCount() throws IllegalAccessException {
+ Meter m = getMeter(DELETED_ORPHAN_NODE);
+ long count = m.getCount();
+ stats.orphanNodesDeleted(10);
+ assertEquals(count + 10, m.getCount());
+ assertEquals(count + 10, ((MeterStats) readField(stats, "deletedOrphanNode", true)).getCount());
+ }
+
+ @Test
+ public void getPropertiesDeletedCount() throws IllegalAccessException {
+ Meter m = getMeter(DELETED_PROPERTY);
+ long count = m.getCount();
+ stats.propertiesDeleted(10);
+ assertEquals(count + 10, m.getCount());
+ assertEquals(count + 10, ((MeterStats) readField(stats, "deletedProperty", true)).getCount());
+ }
+
+ @Test
+ public void getUnmergedBCDeletedCount() throws IllegalAccessException {
+ Meter m = getMeter(DELETED_UNMERGED_BC);
+ long count = m.getCount();
+ stats.unmergedBranchCommitsDeleted(10);
+ assertEquals(count + 10, m.getCount());
+ assertEquals(count + 10, ((MeterStats) readField(stats, "deletedUnmergedBC", true)).getCount());
+ }
+
+ @Test
+ public void getDocumentsUpdatedCount() throws IllegalAccessException {
+ Meter m = getMeter(UPDATED_DOC);
+ long count = m.getCount();
+ stats.documentsUpdated(10);
+ assertEquals(count + 10, m.getCount());
+ assertEquals(count + 10, ((MeterStats) readField(stats, "updatedDoc", true)).getCount());
+ }
+
+ @Test
+ public void timers() {
+ final VersionGarbageCollector.VersionGCStats vgcs = new VersionGarbageCollector.VersionGCStats();
+ vgcs.fullGCActiveElapsed = MILLISECONDS.toMicros(2);
+ vgcs.fullGCDocsElapsed = MILLISECONDS.toMicros(3);
+ vgcs.collectFullGCElapsed = MILLISECONDS.toMicros(5);
+ vgcs.collectOrphanNodesElapsed = MILLISECONDS.toMicros(6);
+ vgcs.collectDeletedPropsElapsed = MILLISECONDS.toMicros(7);
+ vgcs.collectDeletedOldRevsElapsed = MILLISECONDS.toMicros(11);
+ vgcs.collectUnmergedBCElapsed = MILLISECONDS.toMicros(13);
+ vgcs.deleteFullGCDocsElapsed = MILLISECONDS.toMicros(15);
+ vgcs.fullGCActive.start();
+
+ stats.finished(vgcs);
+ assertTimer(2, FULL_GC_ACTIVE_TIMER);
+ assertTimer(3, FULL_GC_TIMER);
+ assertTimer(5, COLLECT_FULL_GC_TIMER);
+ assertTimer(6, COLLECT_ORPHAN_NODES_TIMER);
+ assertTimer(7, COLLECT_DELETED_PROPS_TIMER);
+ assertTimer(11, COLLECT_DELETED_OLD_REVS_TIMER);
+ assertTimer(13, COLLECT_UNMERGED_BC_TIMER);
+ assertTimer(15, DELETE_FULL_GC_DOCS_TIMER);
+ }
+
+ @Test
+ public void counters() {
+ Counter counter = getCounter(COUNTER);
+ Counter failureCounter = getCounter(FAILURE_COUNTER);
+
+ VersionGarbageCollector.VersionGCStats vgcs = new VersionGarbageCollector.VersionGCStats();
+ stats.started();
+ stats.finished(vgcs);
+ assertEquals(1, counter.getCount());
+ assertEquals(0, failureCounter.getCount());
+
+ vgcs.success = false;
+ stats.started();
+ stats.finished(vgcs);
+ assertEquals(2, counter.getCount());
+ assertEquals(1, failureCounter.getCount());
+ }
+
+ private void assertTimer(long expected, String name) {
+ assertEquals(expected, NANOSECONDS.toMillis(getTimer(name).getSnapshot().getMax()));
+ }
+
+ private Timer getTimer(String name) {
+ return statsProvider.getRegistry().getTimers().get(FULL_GC + "." + name);
+ }
+
+ private Meter getMeter(String name) {
+ return statsProvider.getRegistry().getMeters().get(FULL_GC + "." + name);
+ }
+
+ private Counter getCounter(String name) {
+ return statsProvider.getRegistry().getCounters().get(FULL_GC + "." + name);
+ }
+
+}
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleanerTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleanerTest.java
new file mode 100644
index 0000000000..2bd8fe6f8d
--- /dev/null
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleanerTest.java
@@ -0,0 +1,576 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class NodeDocumentRevisionCleanerTest {
+
+ @Mock
+ DocumentStore documentStore;
+
+ @Mock
+ DocumentNodeStore documentNodeStore;
+
+ @Mock
+ Checkpoints checkpoints;
+
+ NodeDocument workingDocument;
+
+ NodeDocumentRevisionCleaner nodeDocumentRevisionCleaner;
+
+ @Before
+ public void setUp() {
+ MockitoAnnotations.openMocks(this);
+ workingDocument = new NodeDocument(documentStore);
+
+ Mockito.when(documentStore.find(Mockito.eq(NODES), Mockito.anyString())).thenReturn(workingDocument);
+ Mockito.when(documentNodeStore.getBranches()).thenReturn(new UnmergedBranches());
+ nodeDocumentRevisionCleaner = new NodeDocumentRevisionCleaner(documentNodeStore, workingDocument);
+ }
+
+ @After
+ public void tearDown() {
+ Mockito.reset(documentStore, documentNodeStore, checkpoints);
+ }
+
+ @Test
+ public void testMarkCheckpointRevisionsToPreserveOnePropertyOneCluster() throws IOException {
+ Revision revisionA = Revision.fromString("r100000000-0-1");
+ Revision revisionB = Revision.fromString("r105000000-0-1");
+ Revision revisionC = Revision.fromString("r110000000-0-1");
+ Revision revisionD = Revision.fromString("r115000000-0-1");
+ Revision revisionE = Revision.fromString("r120000000-0-1");
+ Revision revisionF = Revision.fromString("r125000000-0-1");
+
+ Revision checkpoint1 = Revision.fromString("r109000000-0-1");
+ Revision checkpoint2 = Revision.fromString("r119000000-0-1");
+
+ String jsonProperties = "{" +
+ "'prop1': {'" + revisionA + "': 'value1', '" + revisionB + "': 'value2', '" + revisionC + "': 'value3', '" + revisionD + "': 'value4', '" + revisionE + "': 'value5', '" + revisionF + "': 'value6'}, " +
+ "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c', '" + revisionC + "': 'c', '" + revisionD + "': 'c', '" + revisionE + "': 'c', '" + revisionF + "': 'c'}" +
+ "}";
+ String jsonCheckpoints = "{" +
+ "'" + checkpoint1 + "': {'expires':'200000000','rv':'r109000000-0-1'}," +
+ "'" + checkpoint2 + "': {'expires':'200000000','rv':'r119000000-0-1'}" +
+ "}";
+
+
+ prepareDocumentMock(jsonProperties);
+ prepareCheckpointsMock(jsonCheckpoints);
+
+ nodeDocumentRevisionCleaner.classifyRevisionsAndProperties();
+ nodeDocumentRevisionCleaner.markCheckpointRevisionsToPreserve();
+ nodeDocumentRevisionCleaner.removeCandidatesInList();
+
+ // The revisions blocked should be:
+ // - r105000000-0-1 (blocked by checkpoint r109000000-0-1)
+ // - r115000000-0-1 (blocked by checkpoint r119000000-0-1)
+ assertEquals(Set.of(revisionB, revisionD), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1));
+ assertEquals(Set.of(revisionA, revisionC, revisionE, revisionF), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
+ }
+
+ @Test
+ public void testRecentRevisionsArePreserved() throws IOException {
+ StringBuilder jsonPropBuilder = new StringBuilder("'prop1': {");
+ StringBuilder jsonRevisionsBuilder = new StringBuilder("'_revisions': {");
+
+ Instant currentTime = Instant.now();
+ List<Revision> revisions = new ArrayList<>();
+ for (int i = 29; i >= 0; i--) {
+ long timestamp = currentTime.minus(i, ChronoUnit.HOURS).toEpochMilli();
+ Revision revision = new Revision(timestamp, 0, 1);
+ revisions.add(revision);
+
+ jsonPropBuilder.append("'").append(revision).append("': 'value").append(i).append("', ");
+ jsonRevisionsBuilder.append("'").append(revision).append("': 'c', ");
+ }
+
+ // 23 hours and 59 minutes ago -> Should be preserved
+ long timestamp = currentTime.minus(23, ChronoUnit.HOURS).minus(59, ChronoUnit.MINUTES).toEpochMilli();
+ Revision revision = new Revision(timestamp, 0, 1);
+ revisions.add(revision); // revisions[30]
+ jsonPropBuilder.append("'").append(revision).append("': 'value").append(30).append("', ");
+ jsonRevisionsBuilder.append("'").append(revision).append("': 'c', ");
+
+ // 24 hours and 1 minute ago -> Should be candidate to clean
+ timestamp = currentTime.minus(24, ChronoUnit.HOURS).minus(1, ChronoUnit.MINUTES).toEpochMilli();
+ revision = new Revision(timestamp, 0, 1);
+ revisions.add(revision); // revisions[31]
+ jsonPropBuilder.append("'").append(revision).append("': 'value").append(30).append("'");
+ jsonRevisionsBuilder.append("'").append(revision).append("': 'c'");
+
+ jsonPropBuilder.append("}");
+ jsonRevisionsBuilder.append("}");
+ StringBuilder jsonBuilder = new StringBuilder("{");
+ jsonBuilder.append(jsonPropBuilder).append(", ").append(jsonRevisionsBuilder).append("}");
+
+ // revisions.forEach(rev -> System.out.println(rev.toReadableString()));
+
+ prepareDocumentMock(jsonBuilder.toString());
+
+ nodeDocumentRevisionCleaner.classifyRevisionsAndProperties();
+ nodeDocumentRevisionCleaner.markRevisionsNewerThanThresholdToPreserve(24, ChronoUnit.HOURS);
+ nodeDocumentRevisionCleaner.removeCandidatesInList();
+
+ // The candidate revisions should be the 6 oldest ones (current time -29, -28, -27, -26, -25, -24)
+ // and the one created 24 hours and 1 minute ago
+ Set<Revision> expectedCandidateRevisions = Stream.concat(
+ revisions.subList(0, 6).stream(), Stream.of(revisions.get(31))
+ ).collect(Collectors.toSet());
+
+ // Blocked revisions are all the 24 revisions created in the last 24 hours, and the one created 23 hours and 59 minutes ago
+ Set<Revision> expectedBlockedRevisions = revisions.subList(6, 31).stream().collect(Collectors.toSet());
+
+ assertEquals(expectedCandidateRevisions, nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
+ assertEquals(expectedBlockedRevisions, nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1));
+ }
+
+ @Test
+ public void testInitializeCleanupProcessMultipleClusters() throws IOException {
+ List<Revision> revs = new ArrayList<>();
+
+ // Some initial revisions (0-2)
+ revs.add(Revision.fromString("r100000000-0-1"));
+ revs.add(Revision.fromString("r100000001-0-2"));
+ revs.add(Revision.fromString("r100000002-0-3"));
+
+ // Blocked by first checkpoint r109000000 (3-5)
+ revs.add(Revision.fromString("r106000003-0-1"));
+ revs.add(Revision.fromString("r107000004-0-2"));
+ revs.add(Revision.fromString("r108000005-0-3"));
+
+ // Some unblocked revisions in middle (6-10)
+ revs.add(Revision.fromString("r110000006-0-3"));
+ revs.add(Revision.fromString("r110000006-1-3"));
+ revs.add(Revision.fromString("r114000007-0-2"));
+ revs.add(Revision.fromString("r115000008-0-1"));
+ revs.add(Revision.fromString("r118000009-0-1"));
+
+ // Blocked by second checkpoint r118000000 (11-13)
+ revs.add(Revision.fromString("r118000009-1-1"));
+ revs.add(Revision.fromString("r118000010-0-2"));
+ revs.add(Revision.fromString("r118000011-0-3"));
+
+ // Some more unblocked revisions (14-19)
+ revs.add(Revision.fromString("r120000012-0-1"));
+ revs.add(Revision.fromString("r122000013-0-2"));
+ revs.add(Revision.fromString("r122000013-1-2"));
+ revs.add(Revision.fromString("r123000014-0-2"));
+ revs.add(Revision.fromString("r125000015-0-3"));
+ revs.add(Revision.fromString("r130000016-0-1"));
+
+ // Last revision (20-22)
+ revs.add(Revision.fromString("r130000016-1-1"));
+ revs.add(Revision.fromString("r130000017-0-2"));
+ revs.add(Revision.fromString("r130000018-0-3"));
+
+ // Checkpoint revisions
+ Revision checkpoint1 = Revision.fromString("r109000000-0-1");
+ Revision checkpoint2 = Revision.fromString("r119000000-0-1");
+
+ StringBuilder jsonPropBuilder = new StringBuilder("'prop1': {");
+ StringBuilder jsonRevisionsBuilder = new StringBuilder("'_revisions': {");
+
+ for (int i = 0; i < revs.size(); i++) {
+ jsonPropBuilder.append("'").append(revs.get(i)).append("': 'value").append(i).append("'");
+ jsonRevisionsBuilder.append("'").append(revs.get(i)).append("': 'c'");
+ if (i < revs.size() - 1) {
+ jsonPropBuilder.append(", ");
+ jsonRevisionsBuilder.append(", ");
+ }
+ }
+
+ jsonPropBuilder.append("}");
+ jsonRevisionsBuilder.append("}");
+ StringBuilder jsonBuilder = new StringBuilder("{");
+ jsonBuilder.append(jsonPropBuilder).append(", ").append(jsonRevisionsBuilder).append("}");
+
+ String jsonCheckpoints = "{" +
+ "'" + checkpoint1 + "': {'expires':'200000000','rv':'r106500003-0-1,r107500004-0-2,r108500005-0-3'}," +
+ "'" + checkpoint2 + "': {'expires':'200000000','rv':'r118000009-1-1,r118000010-0-2,r118000011-0-3'}" +
+ "}";
+
+ prepareDocumentMock(jsonBuilder.toString());
+ prepareCheckpointsMock(jsonCheckpoints);
+
+ final UpdateOp op = new UpdateOp(requireNonNull("0:/"), false);
+ nodeDocumentRevisionCleaner.collectOldRevisions(op);
+
+ // The revisions blocked should be:
+ // - r106000003-0-3, r118000004-0-2, r108000005-0-3 (referenced by checkpoint 1)
+ // - r118000009-1-1, r118000010-0-2, r118000011-0-3 (referenced by checkpoint 2)
+ // - r130000016-1-1, r130000017-0-2, r130000018-0-3 (last revision)
+ assertEquals(Set.of(revs.get(3), revs.get(11), revs.get(20)), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1));
+ assertEquals(Set.of(revs.get(4), revs.get(12), revs.get(21)), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2));
+ assertEquals(Set.of(revs.get(5), revs.get(13), revs.get(22)), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3));
+
+ // Rest of revisions are candidates to clean
+ assertEquals(Set.of(revs.get(0), revs.get(9), revs.get(10), revs.get(14), revs.get(19)), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
+ assertEquals(Set.of(revs.get(1), revs.get(8), revs.get(15), revs.get(16), revs.get(17)), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2));
+ assertEquals(Set.of(revs.get(2), revs.get(6), revs.get(7), revs.get(18)), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3));
+
+ assertTrue(Collections.disjoint(nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)));
+ assertTrue(Collections.disjoint(nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2)));
+ assertTrue(Collections.disjoint(nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3)));
+ }
+
+ @Test
+ public void testCheckpointedRevisionFallback() throws IOException {
+ List<Revision> revs = new ArrayList<>();
+
+ revs.add(Revision.fromString("r100000000-0-1"));
+ revs.add(Revision.fromString("r100000001-0-2"));
+ revs.add(Revision.fromString("r100000002-0-3"));
+
+ revs.add(Revision.fromString("r101100003-0-1"));
+ revs.add(Revision.fromString("r101200004-0-2"));
+ revs.add(Revision.fromString("r101300005-0-3"));
+
+ revs.add(Revision.fromString("r102000006-0-1"));
+ revs.add(Revision.fromString("r102000007-0-3"));
+
+ revs.add(Revision.fromString("r103000008-0-1"));
+ revs.add(Revision.fromString("r103000009-0-2"));
+ revs.add(Revision.fromString("r103000010-0-3"));
+
+ // Checkpoint
+ Revision checkpoint1 = Revision.fromString("r102000000-0-1");
+
+ StringBuilder jsonPropBuilder = new StringBuilder("'prop1': {");
+ StringBuilder jsonRevisionsBuilder = new StringBuilder("'_revisions': {");
+
+ for (int i = 0; i < revs.size(); i++) {
+ jsonPropBuilder.append("'").append(revs.get(i)).append("': 'value").append(i).append("'");
+ jsonRevisionsBuilder.append("'").append(revs.get(i)).append("': 'c'");
+ if (i < revs.size() - 1) {
+ jsonPropBuilder.append(", ");
+ jsonRevisionsBuilder.append(", ");
+ }
+ }
+
+ jsonPropBuilder.append("}");
+ jsonRevisionsBuilder.append("}");
+ StringBuilder jsonBuilder = new StringBuilder("{");
+ jsonBuilder.append(jsonPropBuilder).append(", ").append(jsonRevisionsBuilder).append("}");
+
+ String jsonCheckpoints = "{" +
+ "'" + checkpoint1 + "': {'expires':'200000000','rv':'r102000007-0-1,r102000000-0-2,r102000008-0-3'}" +
+ "}";
+
+ prepareDocumentMock(jsonBuilder.toString());
+ prepareCheckpointsMock(jsonCheckpoints);
+
+ final UpdateOp op = new UpdateOp(requireNonNull("0:/"), false);
+ nodeDocumentRevisionCleaner.collectOldRevisions(op);
+
+ // The revisions blocked should be:
+ // - r103000008-0-1, r103000009-0-2, r103000010-0-3 (last revisions)
+ // - r102000006-0-1, r101300005-0-3, r102000007-0-3 (referenced by checkpoint for clusters 1, 2 and 3 respectively)
+ assertEquals(Set.of(revs.get(6), revs.get(8)), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1));
+ assertEquals(Set.of(revs.get(9)), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2));
+ assertEquals(Set.of(revs.get(5), revs.get(7), revs.get(10)), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3));
+
+ // Rest of revisions are candidates to clean
+ assertEquals(Set.of(revs.get(0), revs.get(3)), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
+ assertEquals(Set.of(revs.get(1), revs.get(4)), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2));
+ assertEquals(Set.of(revs.get(2)), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3));
+ }
+
+ @Test
+ public void testLastRevisionIsBlocked() throws IOException {
+ Revision revisionA = new Revision(111111111L, 0, 1);
+ Revision revisionB = new Revision(222222222L, 0, 1);
+ Revision revisionC = new Revision(333333333L, 0, 1);
+
+ String jsonProperties = "{" +
+ "'prop1': {'" + revisionA + "': 'value1', '" + revisionB + "': 'value2', '" + revisionC + "': 'value3'}, " +
+ "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c'}" +
+ "}";
+ prepareDocumentMock(jsonProperties);
+
+ nodeDocumentRevisionCleaner.classifyRevisionsAndProperties();
+ nodeDocumentRevisionCleaner.markLastRevisionForEachProperty();
+
+ assertFalse(nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA));
+ assertFalse(nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionB));
+ assertEquals(Set.of(revisionC), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1));
+ assertEquals(Set.of(revisionA, revisionB, revisionC), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
+ }
+
+ @Test
+ public void testFirstDeletedRevisionIsBlocked() throws Exception {
+ Revision revisionA = new Revision(111111111L, 0, 1);
+ Revision revisionB = new Revision(222222222L, 0, 1);
+ Revision revisionC = new Revision(333333333L, 0, 1);
+
+ String jsonProperties = "{" +
+ "'_deleted': {'" + revisionA + "': 'false', '" + revisionB + "': 'true', '" + revisionC + "': 'false'}," +
+ "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c'}" +
+ "}";
+ prepareDocumentMock(jsonProperties);
+
+ nodeDocumentRevisionCleaner.classifyRevisionsAndProperties();
+
+ assertTrue(nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA));
+ assertEquals(Set.of(revisionA, revisionB, revisionC), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
+ }
+
+ @Test
+ public void testFirstDeletedRevisionIsBlockedUnordered() throws Exception {
+ Revision revisionA = new Revision(111111111L, 0, 1);
+ Revision revisionB = new Revision(222222222L, 0, 1);
+ Revision revisionC = new Revision(333333333L, 0, 1);
+
+ String jsonProperties = "{" +
+ "'_deleted': {'" + revisionC + "': 'false', '" + revisionA + "': 'true', '" + revisionB + "': 'false'}," +
+ "'_revisions': {'" + revisionC + "': 'c', '" + revisionB + "': 'c', '" + revisionA + "': 'c'}" +
+ "}";
+ prepareDocumentMock(jsonProperties);
+
+ nodeDocumentRevisionCleaner.classifyRevisionsAndProperties();
+
+ assertTrue(nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA));
+ assertEquals(Set.of(revisionA, revisionB, revisionC), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
+ }
+
+ @Test
+ public void testClassifyAndMapRevisionsAndPropertiesWithDeleted() throws IOException {
+ Revision revisionA = new Revision(111111111L, 0, 1);
+ Revision revisionB = new Revision(222222222L, 0, 1);
+ Revision revisionC = new Revision(333333333L, 0, 1);
+
+ String jsonProperties = "{" +
+ "'prop1': {'" + revisionA + "': 'value1', '" + revisionB + "': 'value2', '" + revisionC + "': 'value3'}, " +
+ "'prop2': {'" + revisionB + "': 'value4'}, " +
+ "'_deleted': {'" + revisionA + "': 'false'}," +
+ "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c'}" +
+ "}";
+ prepareDocumentMock(jsonProperties);
+
+ nodeDocumentRevisionCleaner.classifyRevisionsAndProperties();
+
+ SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = nodeDocumentRevisionCleaner.getPropertiesModifiedByRevision();
+ assertEquals(Set.of("prop1", "_deleted"), propertiesModifiedByRevision.get(revisionA));
+ assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB));
+ assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC));
+
+ SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = nodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster();
+ assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("prop1").get(1));
+ assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop2").get(1));
+ assertEquals(Set.of(revisionA), revisionsModifyingProperty.get("_deleted").get(1));
+ }
+
+ @Test
+ public void testClassifyAndMapRevisionsMultipleDeleted() throws IOException {
+ Revision revisionA = new Revision(111111111L, 0, 1);
+ Revision revisionB = new Revision(222222222L, 0, 1);
+ Revision revisionC = new Revision(333333333L, 0, 1);
+
+ String jsonProperties = "{" +
+ "'_deleted': {'" + revisionA + "': 'false', '" + revisionB + "': 'true', '" + revisionC + "': 'false'}," +
+ "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c'}" +
+ "}";
+ prepareDocumentMock(jsonProperties);
+
+ nodeDocumentRevisionCleaner.classifyRevisionsAndProperties();
+
+ SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = nodeDocumentRevisionCleaner.getPropertiesModifiedByRevision();
+ assertEquals(Set.of("_deleted"), propertiesModifiedByRevision.get(revisionA));
+ assertEquals(Set.of("_deleted"), propertiesModifiedByRevision.get(revisionB));
+ assertEquals(Set.of("_deleted"), propertiesModifiedByRevision.get(revisionC));
+
+ SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = nodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster();
+ assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("_deleted").get(1));
+ }
+
+ @Test
+ public void testClassifyAndMapRevisionsAndPropertiesWithoutDeleted() throws IOException {
+ Revision revisionA = new Revision(111111111L, 0, 1);
+ Revision revisionB = new Revision(222222222L, 0, 1);
+ Revision revisionC = new Revision(333333333L, 0, 1);
+
+ String jsonProperties = "{" +
+ "'prop1': {'" + revisionA + "': 'value1', '" + revisionB + "': 'value2', '" + revisionC + "': 'value3'}, " +
+ "'prop2': {'" + revisionB + "': 'value4'}, " +
+ "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c'}" +
+ "}";
+ prepareDocumentMock(jsonProperties);
+
+ nodeDocumentRevisionCleaner.classifyRevisionsAndProperties();
+
+ SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = nodeDocumentRevisionCleaner.getPropertiesModifiedByRevision();
+ assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionA));
+ assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB));
+ assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC));
+
+ SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = nodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster();
+ assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("prop1").get(1));
+ assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop2").get(1));
+ }
+
+ @Test
+ public void testClassifyAndMapRevisionsAndPropertiesNotCommitted() throws IOException {
+ Revision revisionA = new Revision(111111111L, 0, 1);
+ Revision revisionB = new Revision(222222222L, 0, 1);
+ Revision revisionC = new Revision(333333333L, 0, 1);
+ Revision revisionD = new Revision(444444444L, 0, 1);
+
+ String jsonProperties = "{" +
+ "'prop1': {'" + revisionA + "': 'value1', '" + revisionB + "': 'value2', '" + revisionC + "': 'value3', '" + revisionD + "': 'value5'}, " +
+ "'prop2': {'" + revisionB + "': 'value4', '" + revisionD + "': 'value5'}, " +
+ "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c', '" + revisionD + "': 'nc'}" +
+ "}";
+ prepareDocumentMock(jsonProperties);
+
+ nodeDocumentRevisionCleaner.classifyRevisionsAndProperties();
+
+ // Modifications done in revisionD should be ignored, as it is not committed
+ SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = nodeDocumentRevisionCleaner.getPropertiesModifiedByRevision();
+ assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionA));
+ assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB));
+ assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC));
+ assertNull(propertiesModifiedByRevision.get(revisionD));
+
+ SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = nodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster();
+ assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("prop1").get(1));
+ assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop2").get(1));
+ }
+
+ @Test
+ public void testClassifyAndMapRevisionsAndPropertiesDifferentClusters() throws IOException {
+ Revision revisionA = new Revision(111111111L, 0, 1);
+ Revision revisionB = new Revision(222222222L, 0, 2);
+ Revision revisionC = new Revision(333333333L, 0, 3);
+ Revision revisionD = new Revision(444444444L, 0, 1);
+
+ String jsonProperties = "{" +
+ "'prop1': {'" + revisionA + "': 'value1', '" + revisionB + "': 'value2', '" + revisionC + "': 'value3', '" + revisionD + "': 'value5'}, " +
+ "'prop2': {'" + revisionB + "': 'value4', '" + revisionD + "': 'value5'}, " +
+ "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c', '" + revisionD + "': 'c'}" +
+ "}";
+ prepareDocumentMock(jsonProperties);
+
+ nodeDocumentRevisionCleaner.classifyRevisionsAndProperties();
+
+ SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = nodeDocumentRevisionCleaner.getPropertiesModifiedByRevision();
+ assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionA));
+ assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB));
+ assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC));
+ assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionD));
+
+ SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = nodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster();
+ assertEquals(Set.of(revisionA, revisionD), revisionsModifyingProperty.get("prop1").get(1));
+ assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop1").get(2));
+ assertEquals(Set.of(revisionC), revisionsModifyingProperty.get("prop1").get(3));
+ assertEquals(Set.of(revisionD), revisionsModifyingProperty.get("prop2").get(1));
+ assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop2").get(2));
+ assertNull(revisionsModifyingProperty.get("prop2").get(3));
+ }
+
+ private void prepareDocumentMock(String jsonProperties) throws IOException {
+ String json = jsonProperties.replace("'", "\"");
+
+ ObjectMapper objectMapper = new ObjectMapper();
+ Map<String, Map<String, String>> data = objectMapper.readValue(json, new TypeReference<>() {});
+
+ SortedMap<String, Object> entries = new TreeMap<>();
+ SortedMap<Revision, String> allRevisions = new TreeMap<>(StableRevisionComparator.INSTANCE);
+ for (Map.Entry<String, Map<String, String>> entry : data.entrySet()) {
+ String property = entry.getKey();
+ Map<String, String> revisions = entry.getValue();
+
+ SortedMap<Revision, String> sortedRevisions = new TreeMap<>(StableRevisionComparator.INSTANCE);
+ for (Map.Entry<String, String> revisionEntry : revisions.entrySet()) {
+ String revisionStr = revisionEntry.getKey();
+ String value = revisionEntry.getValue();
+
+ String[] parts = revisionStr.split("-");
+ // The timestamp part of the revision string (first part) is parsed as hexadecimal (radix 16)
+ long timestamp = Long.parseLong(parts[0].substring(1), 16);
+ int counter = Integer.parseInt(parts[1]);
+ int clusterId = Integer.parseInt(parts[2]);
+
+ Revision revision = new Revision(timestamp, counter, clusterId);
+ sortedRevisions.put(revision, value);
+
+ // Add all revisions to the "_revisions" list
+ allRevisions.put(revision, value);
+ }
+ entries.put(property, sortedRevisions);
+ }
+
+ workingDocument.data = entries;
+ }
+
+ private void prepareCheckpointsMock(String jsonCheckpoints) throws IOException {
+ String json = jsonCheckpoints.replace("'", "\"");
+
+ ObjectMapper objectMapper = new ObjectMapper();
+ Map<String, Map<String, Object>> data = objectMapper.readValue(json, new TypeReference<>() {});
+
+ SortedMap<Revision, Checkpoints.Info> checkpoints = new TreeMap<>(StableRevisionComparator.REVERSE);
+ for (Map.Entry<String, Map<String, Object>> entry : data.entrySet()) {
+ String checkpointStr = entry.getKey();
+ Map<String, Object> checkpointData = entry.getValue();
+
+ String[] parts = checkpointStr.split("-");
+ long timestamp = Long.parseLong(parts[0].substring(1), 16);
+ int counter = Integer.parseInt(parts[1]);
+ int clusterId = Integer.parseInt(parts[2]);
+
+ Revision checkpoint = new Revision(timestamp, counter, clusterId);
+ String checkpointDataJson = objectMapper.writeValueAsString(checkpointData);
+
+ Checkpoints.Info info = Checkpoints.Info.fromString(checkpointDataJson);
+ checkpoints.put(checkpoint, info);
+ }
+
+ Mockito.when(documentNodeStore.getCheckpoints()).thenReturn(this.checkpoints);
+ Mockito.when(documentNodeStore.getCheckpoints().getCheckpoints()).thenReturn(checkpoints);
+ }
+}
\ No newline at end of file
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentTest.java
index b99897dfb0..2adcc9f86a 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentTest.java
@@ -449,6 +449,38 @@ public class NodeDocumentTest {
assertEquals(140, uncommittedRevisions);
}
+ @Test
+ public void getPropertyNames() throws CommitFailedException {
+ DocumentStore store = new MemoryDocumentStore();
+ DocumentNodeStore ns = new DocumentMK.Builder().setDocumentStore(store).setAsyncDelay(0).getNodeStore();
+
+ // add properties
+ for (int i = 0; i < 10; i++) {
+ NodeBuilder nb = ns.getRoot().builder();
+ nb.child("x").setProperty("p"+i, i);
+ merge(ns, nb);
+ }
+
+ final NodeDocument nodeDocument = store.find(NODES, "1:/x");
+ assert nodeDocument != null;
+ assertEquals(10, nodeDocument.getPropertyNames().size());
+ }
+
+ @Test
+ public void getNoPropertyNames() throws CommitFailedException {
+ DocumentStore store = new MemoryDocumentStore();
+ DocumentNodeStore ns = new DocumentMK.Builder().setDocumentStore(store).setAsyncDelay(0).getNodeStore();
+
+ // add no property
+ NodeBuilder nb = ns.getRoot().builder();
+ nb.child("x");
+ merge(ns, nb);
+
+ final NodeDocument nodeDocument = store.find(NODES, "1:/x");
+ assert nodeDocument != null;
+ assertEquals(0, nodeDocument.getPropertyNames().size());
+ }
+
@Test
public void getNewestRevisionTooExpensive() throws Exception {
final int NUM_CHANGES = 200;
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/PausableDocumentStore.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/PausableDocumentStore.java
index 2de97d1c4b..06bd6213e3 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/PausableDocumentStore.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/PausableDocumentStore.java
@@ -153,4 +153,8 @@ public class PausableDocumentStore extends DocumentStoreWrapper {
pauseWith(nextCallback);
} // else continue using the same pauseCallback
}
+
+ public void noDispose() {
+ getFailingDocumentStore().noDispose();
+ }
}
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCInitTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCInitTest.java
index 055188be46..2e21b927ca 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCInitTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCInitTest.java
@@ -18,15 +18,28 @@
*/
package org.apache.jackrabbit.oak.plugins.document;
+import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCStats;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
+import static java.util.concurrent.TimeUnit.DAYS;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
+import static org.apache.jackrabbit.oak.plugins.document.Collection.SETTINGS;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.enableFullGC;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.enableFullGCDryRun;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MIN_ID_VALUE;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_FULL_GC_DRY_RUN_DOCUMENT_ID_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_FULL_GC_DRY_RUN_TIMESTAMP_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_ID;
+import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getIdFromPath;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
-import java.util.concurrent.TimeUnit;
-
public class VersionGCInitTest {
@Rule
@@ -42,13 +55,104 @@ public class VersionGCInitTest {
@Test
public void lazyInitialize() throws Exception {
DocumentStore store = ns.getDocumentStore();
- Document vgc = store.find(Collection.SETTINGS, "versionGC");
+ Document vgc = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
+ assertNull(vgc);
+
+ ns.getVersionGarbageCollector().gc(1, DAYS);
+
+ vgc = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
+ assertNotNull(vgc);
+ assertEquals(0L, vgc.get(SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP));
+ assertNull(vgc.get(SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP));
+ }
+
+ @Test
+ public void lazyInitializeWithFullGC() throws Exception {
+ DocumentStore store = ns.getDocumentStore();
+ Document vgc = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
assertNull(vgc);
- ns.getVersionGarbageCollector().gc(1, TimeUnit.DAYS);
+ enableFullGC(ns.getVersionGarbageCollector());
+ long offset = SECONDS.toMillis(42);
+ String id = getIdFromPath("/node");
+ Revision r = new Revision(offset, 0, 1);
+ UpdateOp op = new UpdateOp(id, true);
+ NodeDocument.setModified(op, r);
+ store.createOrUpdate(NODES, op);
+ VersionGCStats stats = ns.getVersionGarbageCollector().gc(1, DAYS);
- vgc = store.find(Collection.SETTINGS, "versionGC");
+ vgc = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
assertNotNull(vgc);
+ assertEquals(stats.oldestModifiedDocTimeStamp, vgc.get(SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP));
+ assertEquals(stats.oldestModifiedDocId, vgc.get(SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP));
+ assertEquals(MIN_ID_VALUE, vgc.get(SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP));
}
+ @Test
+ public void lazyInitializeWithFullGCWithNoData() throws Exception {
+ DocumentStore store = ns.getDocumentStore();
+ Document vgc = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
+ assertNull(vgc);
+
+ enableFullGC(ns.getVersionGarbageCollector());
+ VersionGCStats stats = ns.getVersionGarbageCollector().gc(1, DAYS);
+
+ vgc = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
+ assertNotNull(vgc);
+ assertEquals(stats.oldestModifiedDocTimeStamp, vgc.get(SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP));
+ assertEquals(stats.oldestModifiedDocId, vgc.get(SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP));
+ assertEquals(MIN_ID_VALUE, vgc.get(SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP));
+ }
+
+ @Test
+ public void lazyInitializeWithFullGCDryRun() throws Exception {
+ DocumentStore store = ns.getDocumentStore();
+ Document vgc = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
+ assertNull(vgc);
+
+ enableFullGC(ns.getVersionGarbageCollector());
+ enableFullGCDryRun(ns.getVersionGarbageCollector());
+
+ long offset = SECONDS.toMillis(42);
+ String id = getIdFromPath("/node");
+ Revision r = new Revision(offset, 0, 1);
+ UpdateOp op = new UpdateOp(id, true);
+ NodeDocument.setModified(op, r);
+ store.createOrUpdate(NODES, op);
+ VersionGCStats stats = ns.getVersionGarbageCollector().gc(1, DAYS);
+
+ vgc = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
+ assertNotNull(vgc);
+ // fullGC values shouldn't have been updated in dryRun mode
+ System.out.println(stats.oldestModifiedDocId);
+ System.out.println(stats.oldestModifiedDocTimeStamp);
+ System.out.println(vgc);
+ assertNull(vgc.get(SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP));
+ assertNull(vgc.get(SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP));
+
+ // dryRun mode values should have been updated
+ assertEquals(stats.oldestModifiedDocTimeStamp, vgc.get(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_TIMESTAMP_PROP));
+ assertEquals(stats.oldestModifiedDocId, vgc.get(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_DOCUMENT_ID_PROP));
+ }
+
+ @Test
+ public void lazyInitializeWithFullGCDryRunWithNoData() throws Exception {
+ DocumentStore store = ns.getDocumentStore();
+ Document vgc = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
+ assertNull(vgc);
+
+ enableFullGC(ns.getVersionGarbageCollector());
+ enableFullGCDryRun(ns.getVersionGarbageCollector());
+ VersionGCStats stats = ns.getVersionGarbageCollector().gc(1, DAYS);
+
+ vgc = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
+ assertNotNull(vgc);
+ // fullGC values shouldn't have been updated in dryRun mode
+ assertNull(vgc.get(SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP));
+ assertNull(vgc.get(SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP));
+
+ // dryRun mode values should have been updated
+ assertEquals(stats.oldestModifiedDocTimeStamp, vgc.get(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_TIMESTAMP_PROP));
+ assertEquals(stats.oldestModifiedDocId, vgc.get(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_DOCUMENT_ID_PROP));
+ }
}
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCQueryTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCQueryTest.java
index 73eda05759..0b8453011e 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCQueryTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCQueryTest.java
@@ -107,7 +107,7 @@ public class VersionGCQueryTest {
clock.waitUntil(clock.getTime() + TimeUnit.HOURS.toMillis(1));
VersionGarbageCollector gc = new VersionGarbageCollector(
- ns, new VersionGCSupport(store));
+ ns, new VersionGCSupport(store), false, false, false);
prevDocIds.clear();
VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
assertEquals(11, stats.deletedDocGCCount);
@@ -140,7 +140,7 @@ public class VersionGCQueryTest {
clock.waitUntil(clock.getTime() + TimeUnit.HOURS.toMillis(1));
VersionGarbageCollector gc = new VersionGarbageCollector(
- ns, new VersionGCSupport(store));
+ ns, new VersionGCSupport(store), false, false, false);
prevDocIds.clear();
VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
assertEquals(1, stats.deletedDocGCCount);
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCStatsTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCStatsTest.java
index 13448e5403..04793a0992 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCStatsTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCStatsTest.java
@@ -31,21 +31,11 @@ import static org.junit.Assert.assertEquals;
public class VersionGCStatsTest {
- private static final Callable START = new Callable() {
- @Override
- public void call(Stopwatch watch) {
- watch.start();
- }
- };
+ private static final Callable START = Stopwatch::start;
- private static final Callable STOP = new Callable() {
- @Override
- public void call(Stopwatch watch) {
- watch.stop();
- }
- };
+ private static final Callable STOP = Stopwatch::stop;
- private VersionGCStats stats = new VersionGCStats();
+ private final VersionGCStats stats = new VersionGCStats();
@Before
public void before() throws Exception {
@@ -61,12 +51,20 @@ public class VersionGCStatsTest {
VersionGCStats cumulative = new VersionGCStats();
cumulative.addRun(stats);
assertEquals(stats.active.elapsed(MICROSECONDS), cumulative.activeElapsed);
+ assertEquals(stats.fullGCActive.elapsed(MICROSECONDS), cumulative.fullGCActiveElapsed);
assertEquals(stats.collectDeletedDocs.elapsed(MICROSECONDS), cumulative.collectDeletedDocsElapsed);
assertEquals(stats.checkDeletedDocs.elapsed(MICROSECONDS), cumulative.checkDeletedDocsElapsed);
assertEquals(stats.deleteDeletedDocs.elapsed(MICROSECONDS), cumulative.deleteDeletedDocsElapsed);
assertEquals(stats.collectAndDeleteSplitDocs.elapsed(MICROSECONDS), cumulative.collectAndDeleteSplitDocsElapsed);
assertEquals(stats.sortDocIds.elapsed(MICROSECONDS), cumulative.sortDocIdsElapsed);
assertEquals(stats.updateResurrectedDocuments.elapsed(MICROSECONDS), cumulative.updateResurrectedDocumentsElapsed);
+ assertEquals(stats.fullGCDocs.elapsed(MICROSECONDS), cumulative.fullGCDocsElapsed);
+ assertEquals(stats.deleteFullGCDocs.elapsed(MICROSECONDS), cumulative.deleteFullGCDocsElapsed);
+ assertEquals(stats.collectFullGC.elapsed(MICROSECONDS), cumulative.collectFullGCElapsed);
+ assertEquals(stats.collectOrphanNodes.elapsed(MICROSECONDS), cumulative.collectOrphanNodesElapsed);
+ assertEquals(stats.collectDeletedProps.elapsed(MICROSECONDS), cumulative.collectDeletedPropsElapsed);
+ assertEquals(stats.collectDeletedOldRevs.elapsed(MICROSECONDS), cumulative.collectDeletedOldRevsElapsed);
+ assertEquals(stats.collectUnmergedBC.elapsed(MICROSECONDS), cumulative.collectUnmergedBCElapsed);
}
@Test
@@ -77,22 +75,38 @@ public class VersionGCStatsTest {
cumulative.addRun(cumulative);
// now the stats must have doubled
assertEquals(stats.active.elapsed(MICROSECONDS) * 2, cumulative.activeElapsed);
+ assertEquals(stats.fullGCActive.elapsed(MICROSECONDS) * 2, cumulative.fullGCActiveElapsed);
assertEquals(stats.collectDeletedDocs.elapsed(MICROSECONDS) * 2, cumulative.collectDeletedDocsElapsed);
assertEquals(stats.checkDeletedDocs.elapsed(MICROSECONDS) * 2, cumulative.checkDeletedDocsElapsed);
assertEquals(stats.deleteDeletedDocs.elapsed(MICROSECONDS) * 2, cumulative.deleteDeletedDocsElapsed);
assertEquals(stats.collectAndDeleteSplitDocs.elapsed(MICROSECONDS) * 2, cumulative.collectAndDeleteSplitDocsElapsed);
assertEquals(stats.sortDocIds.elapsed(MICROSECONDS) * 2, cumulative.sortDocIdsElapsed);
assertEquals(stats.updateResurrectedDocuments.elapsed(MICROSECONDS) * 2, cumulative.updateResurrectedDocumentsElapsed);
+ assertEquals(stats.fullGCDocs.elapsed(MICROSECONDS) * 2, cumulative.fullGCDocsElapsed);
+ assertEquals(stats.deleteFullGCDocs.elapsed(MICROSECONDS) * 2, cumulative.deleteFullGCDocsElapsed);
+ assertEquals(stats.collectFullGC.elapsed(MICROSECONDS) * 2, cumulative.collectFullGCElapsed);
+ assertEquals(stats.collectOrphanNodes.elapsed(MICROSECONDS) * 2, cumulative.collectOrphanNodesElapsed);
+ assertEquals(stats.collectDeletedProps.elapsed(MICROSECONDS) * 2, cumulative.collectDeletedPropsElapsed);
+ assertEquals(stats.collectDeletedOldRevs.elapsed(MICROSECONDS) * 2, cumulative.collectDeletedOldRevsElapsed);
+ assertEquals(stats.collectUnmergedBC.elapsed(MICROSECONDS) * 2, cumulative.collectUnmergedBCElapsed);
}
private void forEachStopwatch(VersionGCStats stats, Callable c) {
c.call(stats.active);
+ c.call(stats.fullGCActive);
c.call(stats.collectDeletedDocs);
c.call(stats.checkDeletedDocs);
c.call(stats.deleteDeletedDocs);
c.call(stats.collectAndDeleteSplitDocs);
c.call(stats.sortDocIds);
c.call(stats.updateResurrectedDocuments);
+ c.call(stats.fullGCDocs);
+ c.call(stats.deleteFullGCDocs);
+ c.call(stats.collectFullGC);
+ c.call(stats.collectOrphanNodes);
+ c.call(stats.collectDeletedProps);
+ c.call(stats.collectDeletedOldRevs);
+ c.call(stats.collectUnmergedBC);
}
private interface Callable {
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupportTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupportTest.java
index 831d2c89ec..eb848fe4cc 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupportTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupportTest.java
@@ -18,10 +18,9 @@
*/
package org.apache.jackrabbit.oak.plugins.document;
+import java.util.ArrayList;
import java.util.List;
-import org.apache.jackrabbit.guava.common.collect.Iterables;
-import org.apache.jackrabbit.guava.common.collect.Lists;
import com.mongodb.ReadPreference;
import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore;
@@ -30,30 +29,44 @@ import org.apache.jackrabbit.oak.plugins.document.mongo.MongoVersionGCSupport;
import org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStore;
import org.apache.jackrabbit.oak.plugins.document.rdb.RDBVersionGCSupport;
import org.apache.jackrabbit.oak.plugins.document.util.Utils;
-import org.apache.jackrabbit.oak.stats.Clock;
import org.junit.After;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import static java.lang.Long.MAX_VALUE;
+import static java.util.Comparator.comparing;
+import static java.util.List.of;
+import static java.util.Optional.ofNullable;
import static java.util.concurrent.TimeUnit.SECONDS;
+import static java.util.stream.StreamSupport.stream;
+import static org.apache.jackrabbit.guava.common.collect.Comparators.isInOrder;
+import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
+import static org.apache.jackrabbit.oak.plugins.document.Document.ID;
import static org.apache.jackrabbit.oak.plugins.document.DocumentStoreFixture.MEMORY;
import static org.apache.jackrabbit.oak.plugins.document.DocumentStoreFixture.MONGO;
import static org.apache.jackrabbit.oak.plugins.document.DocumentStoreFixture.RDB_H2;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MIN_ID_VALUE;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MODIFIED_IN_SECS;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.NULL;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.setModified;
+import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getIdFromPath;
+import static org.apache.jackrabbit.oak.stats.Clock.SIMPLE;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
@RunWith(Parameterized.class)
public class VersionGCSupportTest {
- private DocumentStoreFixture fixture;
- private DocumentStore store;
- private VersionGCSupport gcSupport;
- private List<String> ids = Lists.newArrayList();
+ private final DocumentStoreFixture fixture;
+ private final DocumentStore store;
+ private final VersionGCSupport gcSupport;
+ private final List<String> ids = new ArrayList<>();
@Parameterized.Parameters(name="{0}")
public static java.util.Collection<DocumentStoreFixture> fixtures() {
- List<DocumentStoreFixture> fixtures = Lists.newArrayList();
+ List<DocumentStoreFixture> fixtures = new ArrayList<>();
if (RDB_H2.isAvailable()) {
fixtures.add(RDB_H2);
}
@@ -102,7 +115,7 @@ public class VersionGCSupportTest {
UpdateOp op = new UpdateOp(id, true);
NodeDocument.setModified(op, r);
NodeDocument.setDeleted(op, r, true);
- store.create(Collection.NODES, Lists.newArrayList(op));
+ store.create(Collection.NODES, of(op));
}
assertPossiblyDeleted(0, 41, 0);
@@ -126,6 +139,39 @@ public class VersionGCSupportTest {
assertPossiblyDeleted(51, 60, 0);
}
+ @Test
+ public void getPossiblyModifiedDocs() {
+ long offset = SECONDS.toMillis(42);
+ for (int i = 0; i < 5; i++) {
+ Revision r = new Revision(offset + SECONDS.toMillis(i), 0, 1);
+ String id = getIdFromPath("/doc-modified" + i);
+ ids.add(id);
+ UpdateOp op = new UpdateOp(id, true);
+ setModified(op, r);
+ store.create(NODES, of(op));
+ }
+
+ assertModified(0, 41, 0);
+ assertModified(0, 42, 0);
+ assertModified(0, 44, 0);
+ assertModified(0, 45, 3);
+ assertModified(0, 46, 3);
+ assertModified(0, 49, 3);
+ assertModified(0, 50, 5);
+ assertModified(0, 51, 5);
+ assertModified(39, 60, 5);
+ assertModified(40, 60, 5);
+ assertModified(41, 60, 5);
+ assertModified(42, 60, 5);
+ assertModified(44, 60, 5);
+ assertModified(45, 60, 2);
+ assertModified(47, 60, 2);
+ assertModified(48, 60, 2);
+ assertModified(49, 60, 2);
+ assertModified(50, 60, 0);
+ assertModified(51, 60, 0);
+ }
+
@Test
public void findOldest() {
// see OAK-8476
@@ -137,14 +183,207 @@ public class VersionGCSupportTest {
UpdateOp op = new UpdateOp(id, true);
NodeDocument.setModified(op, r);
NodeDocument.setDeleted(op, r, true);
- store.create(Collection.NODES, Lists.newArrayList(op));
+ store.create(Collection.NODES, of(op));
+
+ long reportedSecs = gcSupport.getOldestDeletedOnceTimestamp(SIMPLE, 1) / SECONDS.toMillis(1);
+ assertTrue("diff (s) should be < 5: " + Math.abs(secs - reportedSecs), Math.abs(secs - reportedSecs) < 5);
+ }
+
+ @Test
+ public void findDocument() {
+ long secs = 1234567;
+ long offset = SECONDS.toMillis(secs);
+ Revision r = new Revision(offset, 0, 1);
+ String id = getIdFromPath("/doc");
+ ids.add(id);
+ UpdateOp op = new UpdateOp(id, true);
+ setModified(op, r);
+ store.create(NODES, of(op));
+
+ NodeDocument doc = gcSupport.getDocument(id, of(ID)).orElse(NULL);
+ assertEquals(id, doc.getId());
+ assertEquals(1, doc.keySet().size());
+ }
- long reportedsecs = gcSupport.getOldestDeletedOnceTimestamp(Clock.SIMPLE, 1) / SECONDS.toMillis(1);
- assertTrue("diff (s) should be < 5: " + Math.abs(secs - reportedsecs), Math.abs(secs - reportedsecs) < 5);
+ @Test
+ public void findDocumentWhenNotExist() {
+ long secs = 1234567;
+ long offset = SECONDS.toMillis(secs);
+ Revision r = new Revision(offset, 0, 1);
+ String id = getIdFromPath("/doc/3");
+ ids.add(id);
+ UpdateOp op = new UpdateOp(id, true);
+ setModified(op, r);
+ store.create(NODES, of(op));
+
+ NodeDocument doc = gcSupport.getDocument(getIdFromPath("/doc/4"), of()).orElse(NULL);
+ assertNotEquals(id, doc.getId());
+ }
+
+ @Test
+ public void findDocumentWithProjection() {
+ long secs = 1234567;
+ long offset = SECONDS.toMillis(secs);
+ Revision r = new Revision(offset, 0, 1);
+ String id = getIdFromPath("/doc/2");
+ ids.add(id);
+ UpdateOp op = new UpdateOp(id, true);
+ setModified(op, r);
+ store.create(NODES, of(op));
+
+ NodeDocument doc = gcSupport.getDocument(id, of(ID, MODIFIED_IN_SECS)).orElse(NULL);
+ assertEquals(id, doc.getId());
+ assertEquals(2, doc.keySet().size());
+ }
+
+ @Test
+ public void findOldestModified() {
+ long secs = 1234567;
+ long offset = SECONDS.toMillis(secs);
+ Revision r = new Revision(offset, 0, 1);
+ String id = getIdFromPath("/doc-modified");
+ ids.add(id);
+ UpdateOp op = new UpdateOp(id, true);
+ setModified(op, r);
+ store.create(NODES, of(op));
+
+ NodeDocument oldestModifiedDoc = gcSupport.getOldestModifiedDoc(SIMPLE).orElse(NULL);
+ String oldestModifiedDocId = oldestModifiedDoc.getId();
+ long reportedSecs = ofNullable(oldestModifiedDoc.getModified()).orElse(0L);
+ assertTrue("diff (s) should be < 5: " + Math.abs(secs - reportedSecs), Math.abs(secs - reportedSecs) < 5);
+ assertEquals(id, oldestModifiedDocId);
+ }
+
+ @Test
+ public void findModifiedDocsWhenModifiedIsDifferent() {
+ long secs = 42;
+ long offset = SECONDS.toMillis(secs);
+ List<UpdateOp> updateOps = new ArrayList<>(5_000);
+ for (int i = 0; i < 5_000; i++) {
+ Revision r = new Revision(offset + (i * 5), 0, 1);
+ String id = getIdFromPath("/x" + i);
+ ids.add(id);
+ UpdateOp op = new UpdateOp(id, true);
+ setModified(op, r);
+ updateOps.add(op);
+ }
+ // create 5_000 nodes
+ store.create(NODES, updateOps);
+
+ NodeDocument oldestModifiedDoc = gcSupport.getOldestModifiedDoc(SIMPLE).orElse(NULL);
+ String oldestModifiedDocId = oldestModifiedDoc.getId();
+ long oldestModifiedDocTs = ofNullable(oldestModifiedDoc.getModified()).orElse(0L);
+ assertEquals(40L, oldestModifiedDocTs);
+ assertEquals("1:/x0", oldestModifiedDocId);
+ oldestModifiedDocId = MIN_ID_VALUE;
+
+ for(int i = 0; i < 5; i++) {
+ Iterable<NodeDocument> modifiedDocs = gcSupport.getModifiedDocs(SECONDS.toMillis(oldestModifiedDocTs), MAX_VALUE, 1000, oldestModifiedDocId);
+ assertTrue(isInOrder(modifiedDocs, (o1, o2) -> comparing(NodeDocument::getModified).thenComparing(Document::getId).compare(o1, o2)));
+ long count = stream(modifiedDocs.spliterator(), false).count();
+ assertEquals(1000, count);
+ for (NodeDocument modifiedDoc : modifiedDocs) {
+ oldestModifiedDoc = modifiedDoc;
+ }
+ oldestModifiedDocId = oldestModifiedDoc.getId();
+ oldestModifiedDocTs = ofNullable(oldestModifiedDoc.getModified()).orElse(0L);
+ }
+ }
+
+ @Test
+ public void findModifiedDocsWhenOldestDocIsPresent() {
+ long offset = SECONDS.toMillis(42);
+ List<UpdateOp> updateOps = new ArrayList<>(5_001);
+ for (int i = 0; i < 5_001; i++) {
+ Revision r = new Revision(offset, 0, 1);
+ String id = getIdFromPath("/x" + i);
+ ids.add(id);
+ UpdateOp op = new UpdateOp(id, true);
+ setModified(op, r);
+ updateOps.add(op);
+ }
+ // create 5_001 nodes
+ store.create(NODES, updateOps);
+
+ NodeDocument oldestModifiedDoc = gcSupport.getOldestModifiedDoc(SIMPLE).orElse(NULL);
+ String oldestModifiedDocId = oldestModifiedDoc.getId();
+ long oldestModifiedDocTs = ofNullable(oldestModifiedDoc.getModified()).orElse(0L);
+ assertEquals(40L, oldestModifiedDocTs);
+ assertEquals("1:/x0", oldestModifiedDocId);
+ oldestModifiedDocId = MIN_ID_VALUE;
+
+ for(int i = 0; i < 5; i++) {
+ Iterable<NodeDocument> modifiedDocs = gcSupport.getModifiedDocs(SECONDS.toMillis(oldestModifiedDocTs), MAX_VALUE, 1000, oldestModifiedDocId);
+ assertTrue(isInOrder(modifiedDocs, (o1, o2) -> comparing(NodeDocument::getModified).thenComparing(Document::getId).compare(o1, o2)));
+ long count = stream(modifiedDocs.spliterator(), false).count();
+ assertEquals(1000, count);
+ for (NodeDocument modifiedDoc : modifiedDocs) {
+ oldestModifiedDoc = modifiedDoc;
+ }
+ oldestModifiedDocId = oldestModifiedDoc.getId();
+ oldestModifiedDocTs = ofNullable(oldestModifiedDoc.getModified()).orElse(0L);
+ }
+
+ // fetch last remaining document now
+ Iterable<NodeDocument> modifiedDocs = gcSupport.getModifiedDocs(SECONDS.toMillis(oldestModifiedDocTs), MAX_VALUE, 1000, oldestModifiedDocId);
+ assertEquals(1, stream(modifiedDocs.spliterator(), false).count());
+ assertTrue(isInOrder(modifiedDocs, (o1, o2) -> comparing(NodeDocument::getModified).thenComparing(Document::getId).compare(o1, o2)));
+ oldestModifiedDoc = modifiedDocs.iterator().next();
+ oldestModifiedDocId = oldestModifiedDoc.getId();
+ oldestModifiedDocTs = ofNullable(oldestModifiedDoc.getModified()).orElse(0L);
+
+ // all documents had been fetched, now we won't get any document
+ modifiedDocs = gcSupport.getModifiedDocs(SECONDS.toMillis(oldestModifiedDocTs), MAX_VALUE, 1000, oldestModifiedDocId);
+ assertEquals(0, stream(modifiedDocs.spliterator(), false).count());
+
+ }
+
+ @Test
+ public void findModifiedDocsWhenOldestDocIsAbsent() {
+
+ NodeDocument oldestModifiedDoc = gcSupport.getOldestModifiedDoc(SIMPLE).orElse(NULL);
+ String oldestModifiedDocId = MIN_ID_VALUE;
+ long oldestModifiedDocTs = 0L;
+ assertEquals(NULL, oldestModifiedDoc);
+
+ long offset = SECONDS.toMillis(42);
+ List<UpdateOp> updateOps = new ArrayList<>(5_000);
+ for (int i = 0; i < 5_000; i++) {
+ Revision r = new Revision(offset, 0, 1);
+ String id = getIdFromPath("/x" + i);
+ ids.add(id);
+ UpdateOp op = new UpdateOp(id, true);
+ setModified(op, r);
+ updateOps.add(op);
+ }
+ // create 5_000 nodes
+ store.create(NODES, updateOps);
+
+ for(int i = 0; i < 5; i++) {
+ Iterable<NodeDocument> modifiedDocs = gcSupport.getModifiedDocs(SECONDS.toMillis(oldestModifiedDocTs), MAX_VALUE, 1000, oldestModifiedDocId);
+ assertTrue(isInOrder(modifiedDocs, (o1, o2) -> comparing(NodeDocument::getModified).thenComparing(Document::getId).compare(o1, o2)));
+ long count = stream(modifiedDocs.spliterator(), false).count();
+ assertEquals(1000, count);
+ for (NodeDocument modifiedDoc : modifiedDocs) {
+ oldestModifiedDoc = modifiedDoc;
+ }
+ oldestModifiedDocId = oldestModifiedDoc.getId();
+ oldestModifiedDocTs = ofNullable(oldestModifiedDoc.getModified()).orElse(0L);
+ }
+
+ // all documents had been fetched, now we won't get any document
+ Iterable<NodeDocument> modifiedDocs = gcSupport.getModifiedDocs(SECONDS.toMillis(oldestModifiedDocTs), MAX_VALUE, 1000, oldestModifiedDocId);
+ assertEquals(0, stream(modifiedDocs.spliterator(), false).count());
}
private void assertPossiblyDeleted(long fromSeconds, long toSeconds, long num) {
Iterable<NodeDocument> docs = gcSupport.getPossiblyDeletedDocs(SECONDS.toMillis(fromSeconds), SECONDS.toMillis(toSeconds));
- assertEquals(num, Iterables.size(docs));
+ assertEquals(num, stream(docs.spliterator(), false).count());
+ }
+
+ private void assertModified(long fromSeconds, long toSeconds, long num) {
+ Iterable<NodeDocument> docs = gcSupport.getModifiedDocs(SECONDS.toMillis(fromSeconds), SECONDS.toMillis(toSeconds), 10, MIN_ID_VALUE);
+ assertEquals(num, stream(docs.spliterator(), false).count());
+ assertTrue(isInOrder(docs, (o1, o2) -> comparing(NodeDocument::getModified).thenComparing(Document::getId).compare(o1, o2)));
}
}
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
index a3e7a5e1e3..1ad175344c 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
@@ -47,10 +47,22 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
+import static java.lang.System.currentTimeMillis;
import static java.util.concurrent.TimeUnit.HOURS;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.jackrabbit.oak.plugins.document.Collection.SETTINGS;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.disableFullGC;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.disableFullGCDryRun;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.enableFullGC;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.enableFullGCDryRun;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_FULL_GC_DRY_RUN_DOCUMENT_ID_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_FULL_GC_DRY_RUN_TIMESTAMP_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_ID;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -64,7 +76,7 @@ public class VersionGCTest {
private ExecutorService execService;
- private TestStore store = new TestStore();
+ private final TestStore store = new TestStore();
private DocumentNodeStore ns;
@@ -94,6 +106,8 @@ public class VersionGCTest {
@After
public void tearDown() throws Exception {
+ disableFullGC(gc);
+ disableFullGCDryRun(gc);
execService.shutdown();
execService.awaitTermination(1, MINUTES);
}
@@ -185,6 +199,128 @@ public class VersionGCTest {
}
}
+ // OAK-10199
+ @Test
+ public void cancelMustNotUpdateLastOldestModifiedTimeStamp() throws Exception {
+ // get previous entry from SETTINGS
+ String versionGCId = SETTINGS_COLLECTION_ID;
+ String fullGCTimestamp = SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP;
+ enableFullGC(gc);
+ gc.gc(30, SECONDS);
+ Document statusBefore = store.find(SETTINGS, versionGCId);
+ // block gc call
+ store.semaphore.acquireUninterruptibly();
+ Future<VersionGCStats> stats = gc();
+ boolean gcBlocked = false;
+ for (int i = 0; i < 10; i ++) {
+ if (store.semaphore.hasQueuedThreads()) {
+ gcBlocked = true;
+ break;
+ }
+ Thread.sleep(100);
+ }
+ assertTrue(gcBlocked);
+ // now cancel the GC
+ gc.cancel();
+ store.semaphore.release();
+ assertTrue(stats.get().canceled);
+
+ // ensure a canceled GC doesn't update that versionGC SETTINGS entry
+ Document statusAfter = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
+ if (statusBefore == null) {
+ assertNull(statusAfter);
+ } else {
+ assertNotNull(statusAfter);
+ assertEquals(
+ "canceled GC shouldn't change the " + fullGCTimestamp + " property on " + versionGCId
+ + " settings entry",
+ statusBefore.get(fullGCTimestamp), statusAfter.get(fullGCTimestamp));
+ }
+ }
+
+ @Test
+ public void cancelMustNotUpdateLastOldestModifiedDocId() throws Exception {
+ // get previous entry from SETTINGS
+ String versionGCId = SETTINGS_COLLECTION_ID;
+ String oldestModifiedDocId = SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP;
+ enableFullGC(gc);
+ gc.gc(30, SECONDS);
+ Document statusBefore = store.find(SETTINGS, versionGCId);
+ // block gc call
+ store.semaphore.acquireUninterruptibly();
+ Future<VersionGCStats> stats = gc();
+ boolean gcBlocked = false;
+ for (int i = 0; i < 10; i ++) {
+ if (store.semaphore.hasQueuedThreads()) {
+ gcBlocked = true;
+ break;
+ }
+ Thread.sleep(100);
+ }
+ assertTrue(gcBlocked);
+ // now cancel the GC
+ gc.cancel();
+ store.semaphore.release();
+ assertTrue(stats.get().canceled);
+
+ // ensure a canceled GC doesn't update that versionGC SETTINGS entry
+ Document statusAfter = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
+ if (statusBefore == null) {
+ assertNull(statusAfter);
+ } else {
+ assertNotNull(statusAfter);
+ assertEquals(
+ "canceled GC shouldn't change the " + oldestModifiedDocId + " property on " + versionGCId
+ + " settings entry",
+ statusBefore.get(oldestModifiedDocId), statusAfter.get(oldestModifiedDocId));
+ }
+ }
+
+ // END - OAK-10199
+
+ // OAK-10370
+ @Test
+ public void dryRunMustNotUpdateLastOldestModifiedTimeStamp() throws Exception {
+ // get previous entry from SETTINGS
+ String versionGCId = SETTINGS_COLLECTION_ID;
+ String fullGCTimestamp = SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP;
+ enableFullGC(gc);
+ gc.gc(30, SECONDS);
+ Document statusBefore = store.find(SETTINGS, versionGCId);
+ // now run GC in dryRun mode
+ enableFullGCDryRun(gc);
+
+ gc.gc(30, SECONDS);
+
+ // ensure a dryRun GC doesn't update that versionGC SETTINGS entries
+ Document statusAfter = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
+ assertNotNull(statusAfter);
+ assert statusBefore != null;
+ assertEquals("canceled GC shouldn't change the " + fullGCTimestamp + " property on " + versionGCId
+ + " settings entry", statusBefore.get(fullGCTimestamp), statusAfter.get(fullGCTimestamp));
+ }
+
+ @Test
+ public void dryRunMustNotUpdateLastOldestModifiedDocId() throws Exception {
+ // get previous entry from SETTINGS
+ String versionGCId = SETTINGS_COLLECTION_ID;
+ String oldestModifiedDocId = SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP;
+ enableFullGC(gc);
+ gc.gc(30, SECONDS);
+ final Document statusBefore = store.find(SETTINGS, versionGCId);
+ // now run GC in dryRun mode
+ enableFullGCDryRun(gc);
+ gc.gc(30, SECONDS);
+ // ensure a dryRun GC doesn't update that versionGC SETTINGS entry
+ final Document statusAfter = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
+ assertNotNull(statusAfter);
+ assert statusBefore != null;
+ assertEquals("canceled GC shouldn't change the " + oldestModifiedDocId + " property on " + versionGCId
+ + " settings entry", statusBefore.get(oldestModifiedDocId), statusAfter.get(oldestModifiedDocId));
+ }
+
+ // END - OAK-10370
+
@Test
public void getInfo() throws Exception {
gc.gc(1, TimeUnit.HOURS);
@@ -238,7 +374,7 @@ public class VersionGCTest {
VersionGCSupport localgcsupport = fakeVersionGCSupport(ns.getDocumentStore(), oneYearAgo, twelveTimesTheLimit);
VersionGCRecommendations rec = new VersionGCRecommendations(secondsPerDay, ns.getCheckpoints(), ns.getClock(), localgcsupport,
- options, new TestGCMonitor());
+ options, new TestGCMonitor(), false, false);
// should select a duration of roughly one month
long duration= rec.scope.getDurationMs();
@@ -252,7 +388,7 @@ public class VersionGCTest {
assertTrue(stats.needRepeat);
rec = new VersionGCRecommendations(secondsPerDay, ns.getCheckpoints(), ns.getClock(), localgcsupport, options,
- new TestGCMonitor());
+ new TestGCMonitor(), false, false);
// new duration should be half
long nduration = rec.scope.getDurationMs();
@@ -281,7 +417,7 @@ public class VersionGCTest {
// loop until the recommended interval is at 60s (precisionMS)
do {
rec = new VersionGCRecommendations(secondsPerDay, ns.getCheckpoints(), ns.getClock(), localgcsupport, options,
- testmonitor);
+ testmonitor, false, false);
stats = new VersionGCStats();
stats.limitExceeded = true;
rec.evaluate(stats);
@@ -298,7 +434,7 @@ public class VersionGCTest {
deletedCount -= deleted;
localgcsupport = fakeVersionGCSupport(ns.getDocumentStore(), oldestDeleted, deletedCount);
rec = new VersionGCRecommendations(secondsPerDay, ns.getCheckpoints(), ns.getClock(), localgcsupport, options,
- testmonitor);
+ testmonitor, false, false);
stats = new VersionGCStats();
stats.limitExceeded = false;
stats.deletedDocGCCount = deleted;
@@ -322,7 +458,7 @@ public class VersionGCTest {
deletedOnceCountCalls.incrementAndGet();
return Iterables.size(Utils.getSelectedDocuments(store, NodeDocument.DELETED_ONCE, 1));
}
- });
+ }, false, false, false);
// run first RGC
gc.gc(1, TimeUnit.HOURS);
@@ -338,6 +474,63 @@ public class VersionGCTest {
}
}
+ // OAK-10199
+ @Test
+ public void testFullGCDocumentRead_disabled() throws Exception {
+ disableFullGC(gc);
+ VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
+ assertNotNull(stats);
+ assertEquals(0, stats.fullGCDocsElapsed);
+ }
+
+ @Test
+ public void testFullGCDocumentRead_enabled() throws Exception {
+ enableFullGC(gc);
+ VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
+ assertNotNull(stats);
+ assertNotEquals(0, stats.fullGCDocsElapsed);
+ }
+
+ // OAK-10199 END
+
+ // OAK-10370
+ @Test
+ public void testFullGCDryRunModeEnabled() throws Exception {
+ enableFullGC(gc);
+ enableFullGCDryRun(gc);
+ VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
+ assertNotNull(stats);
+ assertTrue(stats.fullGCDryRunMode);
+ }
+
+ @Test
+ public void testResetFullGCDryRunMode() throws Exception {
+ enableFullGC(gc);
+ enableFullGCDryRun(gc);
+ VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
+ assertNotNull(stats);
+
+ // add dryRun fields data
+ final UpdateOp updateOp = new UpdateOp(SETTINGS_COLLECTION_ID, true);
+ updateOp.set(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_DOCUMENT_ID_PROP, "docId");
+ updateOp.set(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_TIMESTAMP_PROP, currentTimeMillis());
+ store.createOrUpdate(Collection.SETTINGS, updateOp);
+
+ final Document settingsBefore = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
+ assertNotNull(settingsBefore);
+ assertNotNull(settingsBefore.get(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_DOCUMENT_ID_PROP));
+ assertNotNull(settingsBefore.get(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_TIMESTAMP_PROP));
+
+ gc.resetDryRun();
+ final Document settingsAfter = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
+ assertNotNull(settingsAfter);
+ assertNull(settingsAfter.get(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_DOCUMENT_ID_PROP));
+ assertNull(settingsAfter.get(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_TIMESTAMP_PROP));
+
+ }
+
+ // OAK-10370 END
+
private Future<VersionGCStats> gc() {
// run gc in a separate thread
return execService.submit(new Callable<VersionGCStats>() {
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
index 1a5e510f28..776842125b 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
@@ -19,12 +19,17 @@
package org.apache.jackrabbit.oak.plugins.document;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
import java.util.Set;
+import java.util.SortedMap;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
@@ -34,27 +39,65 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
-
+import java.util.function.BiConsumer;
+
+import static java.util.List.of;
+import static java.util.Objects.requireNonNull;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.StreamSupport.stream;
+import static org.apache.commons.lang3.reflect.FieldUtils.readField;
+import static org.apache.commons.lang3.reflect.FieldUtils.writeField;
+import static org.apache.commons.lang3.reflect.FieldUtils.writeStaticField;
+import static org.apache.jackrabbit.guava.common.base.Strings.repeat;
import static org.apache.jackrabbit.guava.common.collect.Iterables.filter;
import static org.apache.jackrabbit.guava.common.collect.Iterables.size;
import static java.util.concurrent.TimeUnit.HOURS;
import static java.util.concurrent.TimeUnit.MINUTES;
+import static org.apache.jackrabbit.oak.api.Type.NAME;
+import static org.apache.jackrabbit.oak.api.Type.STRING;
+import static org.apache.jackrabbit.oak.api.Type.STRINGS;
+import static org.apache.jackrabbit.oak.plugins.document.ClusterNodeInfo.DEFAULT_LEASE_DURATION_MILLIS;
import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
+import static org.apache.jackrabbit.oak.plugins.document.Collection.SETTINGS;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.assertBranchRevisionNotRemovedFromAllDocuments;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.assertBranchRevisionRemovedFromAllDocuments;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.enableFullGC;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.enableFullGCDryRun;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.mergedBranchCommit;
+import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.unmergedBranchCommit;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MIN_ID_VALUE;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.NUM_REVS_THRESHOLD;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.PREV_SPLIT_FACTOR;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.isDeletedEntry;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.setModified;
+import static org.apache.jackrabbit.oak.plugins.document.Revision.getCurrentTimestamp;
+import static org.apache.jackrabbit.oak.plugins.document.Revision.newRevision;
import static org.apache.jackrabbit.oak.plugins.document.TestUtils.NO_BINARY;
+import static org.apache.jackrabbit.oak.plugins.document.TestUtils.createChild;
+import static org.apache.jackrabbit.oak.plugins.document.TestUtils.disposeQuietly;
+import static org.apache.jackrabbit.oak.plugins.document.TestUtils.childBuilder;
+import static org.apache.jackrabbit.oak.plugins.document.UpdateOp.Operation.Type.SET_MAP_ENTRY;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_FULL_GC_DRY_RUN_DOCUMENT_ID_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_FULL_GC_DRY_RUN_TIMESTAMP_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_ID;
+import static org.apache.jackrabbit.oak.plugins.document.bundlor.DocumentBundlor.META_PROP_PATTERN;
+import static org.apache.jackrabbit.oak.plugins.document.util.Utils.PATH_LONG;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import static org.junit.Assume.assumeTrue;
import org.apache.jackrabbit.guava.common.base.Function;
import org.apache.jackrabbit.guava.common.base.Predicate;
-import org.apache.jackrabbit.guava.common.base.Strings;
+import org.apache.jackrabbit.guava.common.cache.Cache;
import org.apache.jackrabbit.guava.common.collect.AbstractIterator;
import org.apache.jackrabbit.guava.common.collect.ImmutableList;
import org.apache.jackrabbit.guava.common.collect.Iterators;
@@ -64,11 +107,15 @@ import org.apache.jackrabbit.guava.common.collect.Sets;
import org.apache.jackrabbit.guava.common.util.concurrent.Atomics;
import com.mongodb.ReadPreference;
+import org.apache.jackrabbit.oak.InitialContent;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.plugins.document.DocumentStoreFixture.RDBFixture;
+import org.apache.jackrabbit.oak.plugins.document.FailingDocumentStore.FailedUpdateOpListener;
+import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.FullGCMode;
import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCStats;
+import org.apache.jackrabbit.oak.plugins.document.bundlor.BundlingConfigInitializer;
import org.apache.jackrabbit.oak.plugins.document.mongo.MongoTestUtils;
import org.apache.jackrabbit.oak.plugins.document.rdb.RDBOptions;
import org.apache.jackrabbit.oak.plugins.document.util.Utils;
@@ -83,14 +130,62 @@ import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
@RunWith(Parameterized.class)
public class VersionGarbageCollectorIT {
- private DocumentStoreFixture fixture;
+ static class GCCounts {
+ private final FullGCMode mode;
+ int deletedDocGCCount, deletedPropsCount, deletedInternalPropsCount,
+ deletedPropRevsCount, deletedInternalPropRevsCount,
+ deletedUnmergedBCCount, updatedFullGCDocsCount;
+
+ public GCCounts(FullGCMode mode) {
+ this(mode, 0, 0, 0, 0, 0, 0, 0);
+ }
+
+ public GCCounts(FullGCMode mode, int deletedDocGCCount, int deletedPropsCount,
+ int deletedInternalPropsCount, int deletedPropRevsCount,
+ int deletedInternalPropRevsCount, int deletedUnmergedBCCount,
+ int updatedFullGCDocsCount) {
+ this.mode = mode;
+ assertTrue(deletedDocGCCount != -1);
+ assertTrue(deletedPropsCount != -1);
+ assertTrue(deletedInternalPropsCount != -1);
+ assertTrue(deletedPropRevsCount != -1);
+ assertTrue(deletedInternalPropRevsCount != -1);
+ assertTrue(deletedUnmergedBCCount != -1);
+ assertTrue(updatedFullGCDocsCount != -1);
+ if (mode == FullGCMode.GAP_ORPHANS
+ || mode == FullGCMode.GAP_ORPHANS_EMPTYPROPS
+ || mode == FullGCMode.ALL_ORPHANS_EMPTYPROPS) {
+ assertEquals(0, deletedPropRevsCount);
+ assertEquals(0, deletedInternalPropRevsCount);
+ assertEquals(0, deletedUnmergedBCCount);
+ } else if (mode == FullGCMode.ORPHANS_EMPTYPROPS_KEEP_ONE_USER_PROPS) {
+ assertEquals(0, deletedInternalPropsCount);
+ assertEquals(0, deletedInternalPropRevsCount);
+ }
+ this.deletedDocGCCount = deletedDocGCCount;
+ this.deletedPropsCount = deletedPropsCount;
+ this.deletedInternalPropsCount = deletedInternalPropsCount;
+ this.deletedPropRevsCount = deletedPropRevsCount;
+ this.deletedInternalPropRevsCount = deletedInternalPropRevsCount;
+ this.deletedUnmergedBCCount = deletedUnmergedBCCount;
+ this.updatedFullGCDocsCount = updatedFullGCDocsCount;
+ }
+ }
+
+ @Parameter(0)
+ public DocumentStoreFixture fixture;
+
+ @Parameter(1)
+ public FullGCMode fullGcMode;
private Clock clock;
@@ -104,20 +199,26 @@ public class VersionGarbageCollectorIT {
private ExecutorService execService;
- public VersionGarbageCollectorIT(DocumentStoreFixture fixture) {
- this.fixture = fixture;
- }
+ private FullGCMode originalFullGcMode;
- @Parameterized.Parameters(name="{0}")
- public static Collection<Object[]> fixtures() throws IOException {
- return AbstractDocumentStoreTest.fixtures();
+ @Parameterized.Parameters(name="{index}: {0} with {1}")
+ public static java.util.Collection<Object[]> params() throws IOException {
+ java.util.Collection<Object[]> params = new LinkedList<>();
+ for (Object[] fixture : AbstractDocumentStoreTest.fixtures()) {
+ DocumentStoreFixture f = (DocumentStoreFixture)fixture[0];
+ for (FullGCMode gcType : FullGCMode.values()) {
+ params.add(new Object[] {f, gcType});
+ }
+ }
+ return params;
}
@Before
- public void setUp() throws InterruptedException {
+ public void setUp() throws Exception {
execService = Executors.newCachedThreadPool();
clock = new Clock.Virtual();
clock.waitUntil(System.currentTimeMillis());
+ ClusterNodeInfo.setClock(clock);
Revision.setClock(clock);
createPrimaryStore();
@@ -129,23 +230,28 @@ public class VersionGarbageCollectorIT {
// test using a virtual clock
MongoTestUtils.setReadPreference(store1, ReadPreference.primary());
gc = store1.getVersionGarbageCollector();
+
+ originalFullGcMode = VersionGarbageCollector.getFullGcMode();
+ writeStaticField(VersionGarbageCollector.class, "fullGcMode", fullGcMode, true);
}
@After
public void tearDown() throws Exception {
+ writeStaticField(VersionGarbageCollector.class, "fullGcMode", originalFullGcMode, true);
if (store2 != null) {
store2.dispose();
}
if (store1 != null) {
store1.dispose();
}
+ ClusterNodeInfo.resetClockToDefault();
Revision.resetClockToDefault();
execService.shutdown();
execService.awaitTermination(1, MINUTES);
fixture.dispose();
}
- private String rdbTablePrefix = "T" + Long.toHexString(System.currentTimeMillis());
+ private final String rdbTablePrefix = "T" + Long.toHexString(System.currentTimeMillis());
private void createPrimaryStore() {
if (fixture instanceof RDBFixture) {
@@ -160,114 +266,2284 @@ public class VersionGarbageCollectorIT {
}
private void createSecondaryStore(LeaseCheckMode leaseCheckNode) {
+ createSecondaryStore(leaseCheckNode, false);
+ }
+
+ private void createSecondaryStore(LeaseCheckMode leaseCheckNode, boolean withFailingDS) {
if (fixture instanceof RDBFixture) {
((RDBFixture) fixture).setRDBOptions(
new RDBOptions().tablePrefix(rdbTablePrefix).dropTablesOnClose(false));
}
ds2 = fixture.createDocumentStore();
+ if (withFailingDS) {
+ FailingDocumentStore failingDs = new FailingDocumentStore(ds2);
+ failingDs.noDispose();
+ ds2 = failingDs;
+ }
DocumentMK.Builder documentMKBuilder2 = new DocumentMK.Builder().clock(clock).setClusterId(2)
.setLeaseCheckMode(leaseCheckNode)
.setDocumentStore(ds2).setAsyncDelay(0);
store2 = documentMKBuilder2.getNodeStore();
}
- private static Set<Thread> tbefore = new HashSet<>();
+ private static final Set<Thread> tbefore = new HashSet<>();
+
+ @BeforeClass
+ public static void before() throws Exception {
+ tbefore.addAll(Thread.getAllStackTraces().keySet());
+ }
+
+ @AfterClass
+ public static void after() throws Exception {
+ for (Thread t : Thread.getAllStackTraces().keySet()) {
+ if (!tbefore.contains(t)) {
+ System.err.println("potentially leaked thread: " + t);
+ }
+ }
+ }
+
+ @Test
+ public void gcIgnoredForCheckpoint() throws Exception {
+ long expiryTime = 100, maxAge = 20;
+
+ Revision cp = Revision.fromString(store1.checkpoint(expiryTime));
+
+ //Fast forward time to future but before expiry of checkpoint
+ clock.waitUntil(cp.getTimestamp() + expiryTime - maxAge);
+ VersionGCStats stats = gc(gc, maxAge, TimeUnit.MILLISECONDS);
+ assertTrue(stats.ignoredGCDueToCheckPoint);
+ assertFalse(stats.ignoredFullGCDueToCheckPoint);
+ assertFalse(stats.fullGCDryRunMode);
+ assertTrue(stats.canceled);
+
+ //Fast forward time to future such that checkpoint get expired
+ clock.waitUntil(clock.getTime() + expiryTime + 1);
+ stats = gc(gc, maxAge, TimeUnit.MILLISECONDS);
+ assertFalse("GC should be performed", stats.ignoredGCDueToCheckPoint);
+ assertFalse("Full GC shouldn't be performed", stats.ignoredFullGCDueToCheckPoint);
+ assertFalse(stats.canceled);
+ }
+
+ @Test
+ public void testGCDeletedDocument() throws Exception{
+ //1. Create nodes
+ NodeBuilder b1 = store1.getRoot().builder();
+ b1.child("x").child("y");
+ b1.child("z");
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ long maxAge = 1; //hours
+ long delta = TimeUnit.MINUTES.toMillis(10);
+ //1. Go past GC age and check no GC done as nothing deleted
+ clock.waitUntil(Revision.getCurrentTimestamp() + maxAge);
+ VersionGCStats stats = gc(gc, maxAge, HOURS);
+ assertEquals(0, stats.deletedDocGCCount);
+
+ //Remove x/y
+ NodeBuilder b2 = store1.getRoot().builder();
+ b2.child("x").child("y").remove();
+ store1.merge(b2, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ store1.runBackgroundOperations();
+
+ //2. Check that a deleted doc is not collected before
+ //maxAge
+ //Clock cannot move back (it moved forward in #1) so double the maxAge
+ clock.waitUntil(clock.getTime() + delta);
+ stats = gc(gc, maxAge*2, HOURS);
+ assertEquals(0, stats.deletedDocGCCount);
+ assertEquals(0, stats.deletedLeafDocGCCount);
+
+ //3. Check that deleted doc does get collected post maxAge
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
+
+ stats = gc(gc, maxAge*2, HOURS);
+ assertEquals(1, stats.deletedDocGCCount);
+ assertEquals(1, stats.deletedLeafDocGCCount);
+
+ //4. Check that a revived doc (deleted and created again) does not get gc
+ NodeBuilder b3 = store1.getRoot().builder();
+ b3.child("z").remove();
+ store1.merge(b3, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ NodeBuilder b4 = store1.getRoot().builder();
+ b4.child("z");
+ store1.merge(b4, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
+ stats = gc(gc, maxAge*2, HOURS);
+ assertEquals(0, stats.deletedDocGCCount);
+ assertEquals(0, stats.deletedLeafDocGCCount);
+ assertEquals(1, stats.updateResurrectedGCCount);
+ }
+
+ @Test
+ public void gcSplitDocs() throws Exception {
+ gcSplitDocsInternal("foo");
+ }
+
+ @Test
+ public void gcLongPathSplitDocs() throws Exception {
+ gcSplitDocsInternal(repeat("sub", 120));
+ }
+
+ private VersionGCStats gc(VersionGarbageCollector gc, long maxRevisionAge, TimeUnit unit) throws IOException {
+ final VersionGCStats stats = gc.gc(maxRevisionAge, unit);
+ if (stats.skippedFullGCDocsCount != 0) {
+ (new Exception("here: " + stats.skippedFullGCDocsCount)).printStackTrace(System.out);
+ }
+ assertEquals(0, stats.skippedFullGCDocsCount);
+ return stats;
+ }
+
+ // OAK-10199
+ @Test
+ public void testFullGCNeedRepeat() throws Exception {
+ long expiryTime = 5001, maxAge = 20, batchSize = /*PROGRESS_BATCH_SIZE+1*/ 10001;
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+
+ // create a bunch of garbage
+ NodeBuilder b1 = store1.getRoot().builder();
+ for( int i = 0; i < batchSize; i++ ) {
+ b1.child("c" + i).setProperty("test", "t", STRING);
+ }
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ for( int i = 0; i < batchSize; i++ ) {
+ b1.child("c" + i).removeProperty("test");
+ }
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ //Fast forward time to future such that checkpoint get expired
+ clock.waitUntil(clock.getTime() + expiryTime);
+ VersionGCStats stats = gc(gc, maxAge, TimeUnit.MILLISECONDS);
+ assertFalse("Full GC should be performed", stats.ignoredFullGCDueToCheckPoint);
+ assertFalse(stats.canceled);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, (int)batchSize, 0, 0, 0, 0, (int)batchSize),
+ allOrphProp(0, (int)batchSize, 0, 0, 0, 0, (int)batchSize),
+ keepOneFull(0, (int)batchSize, 0, 0, 0, 0, (int)batchSize),
+ keepOneUser(0, (int)batchSize, 0, 0, 0, 0, (int)batchSize),
+ unmergedBcs(0, (int)batchSize, 0, 0, 0, 0, (int)batchSize),
+ betweenChkp(0, (int)batchSize, 0, 0, 2, 0, (int)batchSize + 1),
+ btwnChkpUBC(0, (int)batchSize, 0, 0, 2, 0, (int)batchSize + 1));
+ assertFalse(stats.needRepeat);
+ }
+
+ // OAK-10199
+ @Test
+ public void fullGCIgnoredForCheckpoint() throws Exception {
+ long expiryTime = 100, maxAge = 20;
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+
+ Revision cp = Revision.fromString(store1.checkpoint(expiryTime));
+
+ //Fast forward time to future but before expiry of checkpoint
+ clock.waitUntil(cp.getTimestamp() + expiryTime - maxAge);
+ VersionGCStats stats = gc(gc, maxAge, TimeUnit.MILLISECONDS);
+ assertTrue(stats.ignoredFullGCDueToCheckPoint);
+ assertTrue(stats.canceled);
+
+ //Fast forward time to future such that checkpoint get expired
+ clock.waitUntil(clock.getTime() + expiryTime + 1);
+ stats = gc(gc, maxAge, TimeUnit.MILLISECONDS);
+ assertFalse("Full GC should be performed", stats.ignoredFullGCDueToCheckPoint);
+ assertFalse(stats.canceled);
+ }
+
+ @Test
+ public void testFullGCNotIgnoredForRGCCheckpoint() throws Exception {
+
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+
+ //1. Create nodes with properties
+ NodeBuilder b1 = store1.getRoot().builder();
+
+ // Add property to node & save
+ b1.child("x").setProperty("test", "t", STRING);
+ b1.child("z").setProperty("test", "t", STRING);
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ //Remove property
+ NodeBuilder b2 = store1.getRoot().builder();
+ b2.getChildNode("x").removeProperty("test");
+ store1.merge(b2, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ //2. move clock forward with 2 hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+
+ //3. Create a checkpoint now with expiry of 1 hour
+ long expiryTime = 1, delta = MINUTES.toMillis(10);
+ NodeBuilder b3 = store1.getRoot().builder();
+ b3.getChildNode("z").removeProperty("test");
+ store1.merge(b3, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ Revision.fromString(store1.checkpoint(HOURS.toMillis(expiryTime)));
+
+ //4. move clock forward by 10 mins
+ clock.waitUntil(clock.getTime() + delta);
+
+ // 5. Remove a node
+ NodeBuilder b4 = store1.getRoot().builder();
+ b4.getChildNode("z").remove();
+ store1.merge(b4, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ // 6. Now run gc after checkpoint and see removed properties gets collected
+ clock.waitUntil(clock.getTime() + delta*2);
+ VersionGCStats stats = gc(gc, delta, MILLISECONDS);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 1, 0, 0, 0, 0, 1),
+ allOrphProp(0, 1, 0, 0, 0, 0, 1),
+ keepOneFull(0, 1, 0, 0, 0, 0, 1),
+ keepOneUser(0, 1, 0, 0, 0, 0, 1),
+ unmergedBcs(0, 1, 0, 0, 0, 0, 1),
+ betweenChkp(0, 1, 0, 0, 0, 0, 1),
+ btwnChkpUBC(0, 1, 0, 0, 0, 0, 1));
+ assertTrue(stats.ignoredGCDueToCheckPoint);
+ assertTrue(stats.ignoredFullGCDueToCheckPoint);
+ assertTrue(stats.canceled);
+ }
+
+ @Test
+ public void testGCDeletedProps() throws Exception {
+ //1. Create nodes with properties
+ NodeBuilder b1 = store1.getRoot().builder();
+
+ // Add property to node & save
+ b1.child("x").setProperty("test", "t", STRING);
+ b1.child("z").setProperty("prop", "foo", STRING);
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ // update the property
+ b1 = store1.getRoot().builder();
+ b1.getChildNode("z").setProperty("prop", "bar", STRING);
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ // update property again
+ b1 = store1.getRoot().builder();
+ b1.getChildNode("z").setProperty("prop", "baz", STRING);
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+ long maxAge = 1; //hours
+ long delta = MINUTES.toMillis(10);
+ //1. Go past GC age and check no GC done as nothing deleted
+ clock.waitUntil(getCurrentTimestamp() + maxAge);
+ VersionGCStats stats = gc(gc, maxAge, HOURS);
+ assertStatsCountsZero(stats);
+
+ //Remove property
+ NodeBuilder b2 = store1.getRoot().builder();
+ b2.getChildNode("z").removeProperty("prop");
+ store1.merge(b2, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ store1.runBackgroundOperations();
+
+ //2. Check that a deleted property is not collected before maxAge
+ //Clock cannot move back (it moved forward in #1) so double the maxAge
+ clock.waitUntil(clock.getTime() + delta);
+ stats = gc(gc, maxAge*2, HOURS);
+ assertStatsCountsZero(stats);
+
+ //3. Check that deleted property does get collected post maxAge
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
+
+ stats = gc(gc, maxAge*2, HOURS);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 1, 0, 0, 0, 0, 1),
+ allOrphProp(0, 1, 0, 0, 0, 0, 1),
+ keepOneFull(0, 1, 0, 0, 0, 0, 1),
+ keepOneUser(0, 1, 0, 0, 0, 0, 1),
+ unmergedBcs(0, 1, 0, 0, 0, 0, 1),
+ betweenChkp(0, 1, 0, 0, 3, 0, 2),
+ btwnChkpUBC(0, 1, 0, 0, 3, 0, 2));
+ assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
+
+ //4. Check that a revived property (deleted and created again) does not get gc
+ NodeBuilder b3 = store1.getRoot().builder();
+ b3.child("x").removeProperty("test");
+ store1.merge(b3, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ NodeBuilder b4 = store1.getRoot().builder();
+ b4.child("x").setProperty("test", "t", STRING);
+ store1.merge(b4, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
+
+ stats = gc(gc, maxAge*2, HOURS);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
+ keepOneFull(0, 0, 0, 2, 0, 0, 1),
+ keepOneUser(0, 0, 0, 2, 0, 0, 1),
+ unmergedBcs(),
+ betweenChkp(0, 0, 0, 1, 0, 0, 1),
+ btwnChkpUBC(0, 0, 0, 1, 0, 0, 1));
+ assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
+ }
+
+ @Test
+ public void testGCDeletedProps_MoreThan_1000_WithSameRevision() throws Exception {
+ //1. Create nodes with properties
+ NodeBuilder b1 = store1.getRoot().builder();
+
+ // Add property to node & save
+ for (int i = 0; i < 5_000; i++) {
+ for (int j = 0; j < 10; j++) {
+ b1.child("z"+i).setProperty("prop"+j, "foo", STRING);
+ }
+ }
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+ long maxAge = 1; //hours
+ long delta = TimeUnit.MINUTES.toMillis(10);
+
+ //Remove property
+ NodeBuilder b2 = store1.getRoot().builder();
+ for (int i = 0; i < 5_000; i++) {
+ for (int j = 0; j < 10; j++) {
+ b2.getChildNode("z"+i).removeProperty("prop"+j);
+ }
+ }
+ store1.merge(b2, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ store1.runBackgroundOperations();
+
+ //3. Check that deleted property does get collected post maxAge
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
+
+ VersionGCStats stats = gc(gc, maxAge*2, HOURS);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 50_000, 0, 0, 0, 0, 5_000),
+ allOrphProp(0, 50_000, 0, 0, 0, 0, 5_000),
+ keepOneFull(0, 50_000, 0, 0, 0, 0, 5_000),
+ keepOneUser(0, 50_000, 0, 0, 0, 0, 5_000),
+ unmergedBcs(0, 50_000, 0, 0, 0, 0, 5_000),
+ betweenChkp(0, 50_000, 0, 0, 2, 0, 5_000 + 1),
+ btwnChkpUBC(0, 50_000, 0, 0, 2, 0, 5_000 + 1));
+ assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
+ }
+
+ @Test
+ public void testGCDeletedProps_MoreThan_1000_WithDifferentRevision() throws Exception {
+ //1. Create nodes with properties
+ NodeBuilder b1 = store1.getRoot().builder();
+ for (int k = 0; k < 50; k ++) {
+ // Add property to node & save
+ for (int i = 0; i < 100; i++) {
+ for (int j = 0; j < 10; j++) {
+ b1.child(k + "z" + i).setProperty("prop" + j, "foo", STRING);
+ }
+ }
+ }
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+ long maxAge = 1; //hours
+ long delta = MINUTES.toMillis(20);
+
+ //Remove property
+ NodeBuilder b2 = store1.getRoot().builder();
+ for (int k = 0; k < 50; k ++) {
+ b2 = store1.getRoot().builder();
+ for (int i = 0; i < 100; i++) {
+ for (int j = 0; j < 10; j++) {
+ b2.getChildNode(k + "z" + i).removeProperty("prop" + j);
+ }
+ }
+ store1.merge(b2, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ // increase the clock to create new revision for next batch
+ clock.waitUntil(getCurrentTimestamp() + (k * 5));
+ }
+
+ store1.runBackgroundOperations();
+ //3. Check that deleted property does get collected post maxAge
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
+
+ VersionGCStats stats = gc(gc, maxAge, HOURS);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 50_000, 0, 0, 0, 0, 5_000),
+ allOrphProp(0, 50_000, 0, 0, 0, 0, 5_000),
+ keepOneFull(0, 50_000, 0, 0, 0, 0, 5_000),
+ keepOneUser(0, 50_000, 0, 0, 0, 0, 5_000),
+ unmergedBcs(0, 50_000, 0, 0, 0, 0, 5_000),
+ betweenChkp(0, 50_000, 0, 0, 51, 0, 5_000 + 1),
+ btwnChkpUBC(0, 50_000, 0, 0, 51, 0, 5_000 + 1));
+ assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
+ }
+
+ @Test
+ public void testGC_WithNoDeletedProps_And_MoreThan_10_000_DocWithDifferentRevision() throws Exception {
+ //1. Create nodes with properties
+ NodeBuilder b1 = store1.getRoot().builder();
+ for (int k = 0; k < 50; k ++) {
+ b1 = store1.getRoot().builder();
+ // Add property to node & save
+ for (int i = 0; i < 500; i++) {
+ for (int j = 0; j < 10; j++) {
+ b1.child(k + "z" + i).setProperty("prop" + j, "foo", STRING);
+ }
+ }
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ // increase the clock to create new revision for next batch
+ clock.waitUntil(getCurrentTimestamp() + (k * 5));
+ }
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+ long maxAge = 1; //hours
+ long delta = MINUTES.toMillis(20);
+
+
+ store1.runBackgroundOperations();
+ //3. Check that deleted property does get collected post maxAge
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
+
+ for (int i = 0; i < 3 ; i++) {
+
+ VersionGCStats stats = gc(gc, maxAge, HOURS);
+ String oldestModifiedDocId = stats.oldestModifiedDocId;
+ long oldestModifiedDocTimeStamp = stats.oldestModifiedDocTimeStamp;
+
+ Document document = store1.getDocumentStore().find(SETTINGS, SETTINGS_COLLECTION_ID);
+ assert document != null;
+ assertEquals(document.get(SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP), oldestModifiedDocTimeStamp);
+ assertEquals(document.get(SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP), oldestModifiedDocId);
+ }
+ }
+
+ @Test
+ public void testGCDeletedPropsAlreadyGCed() throws Exception {
+ //1. Create nodes with properties
+ NodeBuilder b1 = store1.getRoot().builder();
+ // Add property to node & save
+ for (int i = 0; i < 10; i++) {
+ b1.child("z" + i).setProperty("prop" + i, "foo", STRING);
+ }
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+ long maxAge = 1; //hours
+ long delta = MINUTES.toMillis(10);
+ //1. Go past GC age and check no GC done as nothing deleted
+ clock.waitUntil(getCurrentTimestamp() + maxAge);
+ VersionGCStats stats = gc(gc, maxAge, HOURS);
+ assertStatsCountsZero(stats);
+
+ //Remove property
+ NodeBuilder b2 = store1.getRoot().builder();
+ for (int i = 0; i < 10; i++) {
+ b2.getChildNode("z" + i).removeProperty("prop" + i);
+ }
+ store1.merge(b2, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ //2. Check that deleted property does get collected post maxAge
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
+
+ stats = gc(gc, maxAge*2, HOURS);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 10, 0, 0, 0, 0, 10),
+ allOrphProp(0, 10, 0, 0, 0, 0, 10),
+ keepOneFull(0, 10, 0, 0, 0, 0, 10),
+ keepOneUser(0, 10, 0, 0, 0, 0, 10),
+ unmergedBcs(0, 10, 0, 0, 0, 0, 10),
+ betweenChkp(0, 10, 0, 0, 2, 0, 11),
+ btwnChkpUBC(0, 10, 0, 0, 2, 0, 11));
+ assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
+
+ //3. now reCreate those properties again
+ NodeBuilder b3 = store1.getRoot().builder();
+ // Add property to node & save
+ for (int i = 0; i < 10; i++) {
+ b3.child("z" + i).setProperty("prop" + i, "bar", STRING);
+ }
+ store1.merge(b3, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ //Remove properties again
+ NodeBuilder b4 = store1.getRoot().builder();
+ for (int i = 0; i < 10; i++) {
+ b4.getChildNode("z" + i).removeProperty("prop" + i);
+ }
+ store1.merge(b4, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+
+ //4. Check that deleted property does get collected again
+ // increment the clock again by more than 2 hours + delta
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
+ stats = gc(gc, maxAge*2, HOURS);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 10, 0, 0, 0, 0, 10),
+ allOrphProp(0, 10, 0, 0, 0, 0, 10),
+ keepOneFull(0, 10, 0, 0, 0, 0, 10),
+ keepOneUser(0, 10, 0, 0, 0, 0, 10),
+ unmergedBcs(0, 10, 0, 0, 0, 0, 10),
+ betweenChkp(0, 10, 0, 0, 2, 0, 11),
+ btwnChkpUBC(0, 10, 0, 0, 2, 0, 11));
+ assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
+ }
+
+ @Test
+ public void testGCDeletedPropsAfterSystemCrash() throws Exception {
+ if (store1 != null) {
+ store1.dispose();
+ }
+ final FailingDocumentStore fds = new FailingDocumentStore(fixture.createDocumentStore(), 42) {
+ @Override
+ public void dispose() {}
+ };
+ store1 = new DocumentMK.Builder().clock(clock).setLeaseCheckMode(LeaseCheckMode.DISABLED)
+ .setDocumentStore(fds).setAsyncDelay(0).getNodeStore();
+
+ assertTrue(store1.getDocumentStore() instanceof FailingDocumentStore);
+
+ MongoTestUtils.setReadPreference(store1, ReadPreference.primary());
+ gc = store1.getVersionGarbageCollector();
+
+ //1. Create nodes with properties
+ NodeBuilder b1 = store1.getRoot().builder();
+ // Add property to node & save
+ for (int i = 0; i < 10; i++) {
+ b1.child("z" + i).setProperty("prop" + i, "foo", STRING);
+ }
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ //2. Remove property
+ NodeBuilder b2 = store1.getRoot().builder();
+ for (int i = 0; i < 10; i++) {
+ b2.getChildNode("z" + i).removeProperty("prop" + i);
+ }
+ store1.merge(b2, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+ long maxAge = 1; //hours
+ long delta = MINUTES.toMillis(10);
+
+ //3. Check that deleted property does get collected again
+ // increment the clock again by more than 2 hours + delta
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
+
+ fds.fail().after(0).eternally();
+ try {
+ store1.dispose();
+ fail("dispose() must fail with an exception");
+ } catch (DocumentStoreException e) {
+ // expected
+ }
+ fds.fail().never();
+
+ // create new store
+ store1 = new DocumentMK.Builder().clock(clock).setLeaseCheckMode(LeaseCheckMode.DISABLED)
+ .setDocumentStore(fds).setAsyncDelay(0)
+ .getNodeStore();
+ assertTrue(store1.getDocumentStore() instanceof FailingDocumentStore);
+ MongoTestUtils.setReadPreference(store1, ReadPreference.primary());
+ gc = store1.getVersionGarbageCollector();
+ store1.runBackgroundOperations();
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+
+ //4. Check that deleted property does get collected again
+ // increment the clock again by more than 2 hours + delta
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
+ VersionGCStats stats = gc(gc, maxAge*2, HOURS);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 10, 0, 0, 0, 0, 10),
+ allOrphProp(0, 10, 0, 0, 0, 0, 10),
+ keepOneFull(0, 10, 0, 0, 0, 0, 10),
+ keepOneUser(0, 10, 0, 0, 0, 0, 10),
+ unmergedBcs(0, 10, 0, 0, 0, 0, 10),
+ betweenChkp(0, 10, 0, 0, 2, 0, 11),
+ btwnChkpUBC(0, 10, 0, 0, 2, 0, 11));
+ assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
+ }
+
+ @Test
+ public void testGCDeletedEscapeProps() throws Exception {
+ //1. Create nodes with properties
+ NodeBuilder b1 = store1.getRoot().builder();
+
+ // Add property to node & save
+ for (int i = 0; i < 10; i++) {
+ b1.child("x").setProperty("test."+i, "t", STRING);
+ }
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+ long maxAge = 1; //hours
+ long delta = MINUTES.toMillis(10);
+ //1. Go past GC age and check no GC done as nothing deleted
+ clock.waitUntil(getCurrentTimestamp() + maxAge);
+ VersionGCStats stats = gc(gc, maxAge, HOURS);
+ assertStatsCountsZero(stats);
+
+ //Remove property
+ NodeBuilder b2 = store1.getRoot().builder();
+ for (int i = 0; i < 10; i++) {
+ b2.getChildNode("x").removeProperty("test."+i);
+ }
+ store1.merge(b2, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ store1.runBackgroundOperations();
+
+ //2. Check that a deleted property is not collected before maxAge
+ //Clock cannot move back (it moved forward in #1) so double the maxAge
+ clock.waitUntil(clock.getTime() + delta);
+ stats = gc(gc, maxAge*2, HOURS);
+ assertStatsCountsZero(stats);
+
+ //3. Check that deleted property does get collected post maxAge
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
+
+ stats = gc(gc, maxAge*2, HOURS);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 10, 0, 0, 0, 0, 1),
+ allOrphProp(0, 10, 0, 0, 0, 0, 1),
+ keepOneFull(0, 10, 0, 0, 0, 0, 1),
+ keepOneUser(0, 10, 0, 0, 0, 0, 1),
+ unmergedBcs(0, 10, 0, 0, 0, 0, 1),
+ betweenChkp(0, 10, 0, 0, 1, 0, 2),
+ btwnChkpUBC(0, 10, 0, 0, 1, 0, 2));
+
+ //4. Check that a revived property (deleted and created again) does not get gc
+ NodeBuilder b4 = store1.getRoot().builder();
+ for (int i = 0; i < 10; i++) {
+ b4.child("x").setProperty("test."+i, "t", STRING);
+ }
+ store1.merge(b4, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
+ stats = gc(gc, maxAge*2, HOURS);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
+ keepOneFull(),
+ keepOneUser(),
+ unmergedBcs(),
+ betweenChkp(0, 0, 0, 0, 1, 0, 1),
+ btwnChkpUBC(0, 0, 0, 0, 1, 0, 1));
+ }
+
+ @Test
+ public void testGCDeletedLongPathProps() throws Exception {
+ //1. Create nodes with properties
+ NodeBuilder b1 = store1.getRoot().builder();
+ String longPath = repeat("p", PATH_LONG + 1);
+ b1.child(longPath);
+
+ // Add property to node & save
+ for (int i = 0; i < 10; i++) {
+ b1.child(longPath).child("foo").setProperty("test"+i, "t", STRING);
+ }
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+ long maxAge = 1; //hours
+ long delta = MINUTES.toMillis(10);
+ //1. Go past GC age and check no GC done as nothing deleted
+ clock.waitUntil(getCurrentTimestamp() + maxAge);
+ VersionGCStats stats = gc(gc, maxAge, HOURS);
+ assertStatsCountsZero(stats);
+
+ //Remove property
+ NodeBuilder b2 = store1.getRoot().builder();
+ for (int i = 0; i < 10; i++) {
+ b2.child(longPath).child("foo").removeProperty("test"+i);
+ }
+ store1.merge(b2, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ store1.runBackgroundOperations();
+
+ //2. Check that a deleted property is not collected before maxAge
+ //Clock cannot move back (it moved forward in #1) so double the maxAge
+ clock.waitUntil(clock.getTime() + delta);
+ stats = gc(gc, maxAge*2, HOURS);
+ assertStatsCountsZero(stats);
+
+ //3. Check that deleted property does get collected post maxAge
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
+
+ stats = gc(gc, maxAge*2, HOURS);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 10, 0, 0, 0, 0, 1),
+ allOrphProp(0, 10, 0, 0, 0, 0, 1),
+ keepOneFull(0, 10, 0, 0, 0, 0, 1),
+ keepOneUser(0, 10, 0, 0, 0, 0, 1),
+ unmergedBcs(0, 10, 0, 0, 0, 0, 1),
+ betweenChkp(0, 10, 0, 0, 1, 0, 2),
+ btwnChkpUBC(0, 10, 0, 0, 1, 0, 2));
+
+ //4. Check that a revived property (deleted and created again) does not get gc
+ NodeBuilder b4 = store1.getRoot().builder();
+ for (int i = 0; i < 10; i++) {
+ b4.child(longPath).child("foo").setProperty("test"+i, "t", STRING);
+ }
+ store1.merge(b4, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
+ stats = gc(gc, maxAge*2, HOURS);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
+ keepOneFull(),
+ keepOneUser(),
+ unmergedBcs(),
+ betweenChkp(0, 0, 0, 0, 1, 0, 1),
+ btwnChkpUBC(0, 0, 0, 0, 1, 0, 1));
+ }
+
+ @Test
+ public void testGCDeletedNonBundledProps() throws Exception {
+
+ //0. Initialize bundling configs
+ final NodeBuilder builder = store1.getRoot().builder();
+ new InitialContent().initialize(builder);
+ BundlingConfigInitializer.INSTANCE.initialize(builder);
+ merge(store1, builder);
+ store1.runBackgroundOperations();
+
+ //1. Create nodes with properties
+ NodeBuilder b1 = store1.getRoot().builder();
+ b1.child("x").setProperty("jcr:primaryType", "nt:file", NAME);
+
+ // Add property to node & save
+ for (int i = 0; i < 10; i++) {
+ b1.child("x").child("jcr:content").setProperty("prop"+i, "t", STRING);
+ b1.child("x").setProperty(META_PROP_PATTERN, of("jcr:content"), STRINGS);
+ b1.child("x").setProperty("prop"+i, "bar", STRING);
+ }
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+ long maxAge = 1; //hours
+ long delta = MINUTES.toMillis(10);
+ //1. Go past GC age and check no GC done as nothing deleted
+ clock.waitUntil(getCurrentTimestamp() + maxAge);
+ VersionGCStats stats = gc(gc, maxAge, HOURS);
+ assertStatsCountsZero(stats);
+
+ //Remove property
+ NodeBuilder b2 = store1.getRoot().builder();
+ for (int i = 0; i < 10; i++) {
+ b2.getChildNode("x").removeProperty("prop"+i);
+ }
+ store1.merge(b2, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ store1.runBackgroundOperations();
+
+ //2. Check that a deleted property is not collected before maxAge
+ //Clock cannot move back (it moved forward in #1) so double the maxAge
+ clock.waitUntil(clock.getTime() + delta);
+ stats = gc(gc, maxAge*2, HOURS);
+ assertStatsCountsZero(stats);
+
+ //3. Check that deleted property does get collected post maxAge
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
+
+ stats = gc(gc, maxAge*2, HOURS);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 10, 0, 0, 0, 0, 1),
+ allOrphProp(0, 10, 0, 0, 0, 0, 1),
+ keepOneFull(0, 10, 0, 0, 0, 0, 1),
+ keepOneUser(0, 10, 0, 0, 0, 0, 1),
+ unmergedBcs(0, 10, 0, 0, 0, 0, 1),
+ betweenChkp(0, 10, 0, 0, 1, 0, 2),
+ btwnChkpUBC(0, 10, 0, 0, 1, 0, 2));
+ }
+
+ @Test
+ public void testGCDeletedBundledProps() throws Exception {
+
+ //0. Initialize bundling configs
+ final NodeBuilder builder = store1.getRoot().builder();
+ new InitialContent().initialize(builder);
+ BundlingConfigInitializer.INSTANCE.initialize(builder);
+ merge(store1, builder);
+ store1.runBackgroundOperations();
+
+ //1. Create nodes with properties
+ NodeBuilder b1 = store1.getRoot().builder();
+ b1.child("x").setProperty("jcr:primaryType", "nt:file", NAME);
+
+ // Add property to node & save
+ // adding 11 to have the 10th being a special case as we're not removing it below
+ for (int i = 0; i < 11; i++) {
+ b1.child("x").child("jcr:content").setProperty("prop"+i, "t", STRING);
+ b1.child("x").setProperty(META_PROP_PATTERN, of("jcr:content"), STRINGS);
+ b1.child("x").setProperty("prop"+i, "bar", STRING);
+ }
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ NodeState x = store1.getRoot().getChildNode("x");
+ assertTrue(x.exists());
+ assertTrue(x.hasProperty("prop0"));
+ assertTrue(x.hasProperty("prop10"));
+ NodeState jcrContent = x.getChildNode("jcr:content");
+ assertTrue(jcrContent.exists());
+ assertTrue(jcrContent.hasProperty("prop10"));
+ assertTrue(jcrContent.hasProperty("prop0"));
+
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+ long maxAge = 1; //hours
+ long delta = MINUTES.toMillis(10);
+ //1. Go past GC age and check no GC done as nothing deleted
+ clock.waitUntil(getCurrentTimestamp() + maxAge);
+ VersionGCStats stats = gc(gc, maxAge, HOURS);
+ assertStatsCountsZero(stats);
+
+ x = store1.getRoot().getChildNode("x");
+ assertTrue(x.exists());
+ assertTrue(x.hasProperty("prop0"));
+ assertTrue(x.hasProperty("prop10"));
+ jcrContent = x.getChildNode("jcr:content");
+ assertTrue(jcrContent.exists());
+ assertTrue(jcrContent.hasProperty("prop10"));
+ assertTrue(jcrContent.hasProperty("prop0"));
+
+ //Remove property
+ NodeBuilder b2 = store1.getRoot().builder();
+ for (int i = 0; i < 10; i++) {
+ b2.getChildNode("x").getChildNode("jcr:content").removeProperty("prop"+i);
+ }
+ store1.merge(b2, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ store1.runBackgroundOperations();
+
+ //2. Check that a deleted property is not collected before maxAge
+ //Clock cannot move back (it moved forward in #1) so double the maxAge
+ clock.waitUntil(clock.getTime() + delta);
+ stats = gc(gc, maxAge*2, HOURS);
+ assertStatsCountsZero(stats);
+
+ x = store1.getRoot().getChildNode("x");
+ assertTrue(x.exists());
+ assertTrue(x.hasProperty("prop0"));
+ assertTrue(x.hasProperty("prop10"));
+ jcrContent = x.getChildNode("jcr:content");
+ assertTrue(jcrContent.exists());
+ assertTrue(jcrContent.hasProperty("prop10"));
+ assertFalse(jcrContent.hasProperty("prop0"));
+
+ //3. Check that deleted property does get collected post maxAge
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
+
+ stats = gc(gc, maxAge*2, HOURS);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 10, 0, 0, 0, 0, 1),
+ allOrphProp(0, 10, 0, 0, 0, 0, 1),
+ keepOneFull(0, 10, 0, 0, 0, 0, 1),
+ keepOneUser(0, 10, 0, 0, 0, 0, 1),
+ unmergedBcs(0, 10, 0, 0, 0, 0, 1),
+ betweenChkp(0, 10, 0, 0, 1, 0, 2),
+ btwnChkpUBC(0, 10, 0, 0, 1, 0, 2));
+
+ x = store1.getRoot().getChildNode("x");
+ assertTrue(x.exists());
+ assertTrue(x.hasProperty("prop0"));
+ assertTrue(x.hasProperty("prop10"));
+ jcrContent = x.getChildNode("jcr:content");
+ assertTrue(jcrContent.exists());
+ assertTrue(jcrContent.hasProperty("prop10"));
+ assertFalse(jcrContent.hasProperty("prop0"));
+ }
+
+ static void assertStatsCountsZero(VersionGCStats stats) {
+ GCCounts c = new GCCounts(VersionGarbageCollector.getFullGcMode());
+ assertStatsCountsEqual(stats, c);
+ }
+
+ static void assertStatsCountsEqual(VersionGCStats stats, GCCounts... counts) {
+ GCCounts c = null;
+ for (GCCounts a : counts) {
+ if (a.mode == VersionGarbageCollector.getFullGcMode()) {
+ c = a;
+ break;
+ }
+ }
+ if (c == null && VersionGarbageCollector.getFullGcMode() == FullGCMode.NONE) {
+ c = new GCCounts(FullGCMode.NONE);
+ }
+ assertNotNull(stats);
+ assertNotNull(c);
+ assertEquals(c.mode + "/docGC", c.deletedDocGCCount, stats.deletedDocGCCount);
+ assertEquals(c.mode + "/props", c.deletedPropsCount, stats.deletedPropsCount);
+ assertEquals(c.mode + "/internalProps", c.deletedInternalPropsCount, stats.deletedInternalPropsCount);
+ assertEquals(c.mode + "/propRevs", c.deletedPropRevsCount, stats.deletedPropRevsCount);
+ assertEquals(c.mode + "/internalPropRevs", c.deletedInternalPropRevsCount, stats.deletedInternalPropRevsCount);
+ assertEquals(c.mode + "/unmergedBC", c.deletedUnmergedBCCount, stats.deletedUnmergedBCCount);
+ assertEquals(c.mode + "/updatedFullGCDocsCount", c.updatedFullGCDocsCount, stats.updatedFullGCDocsCount);
+ }
+
+ @Test
+ public void testGCDeletedPropsWhenModifiedConcurrently() throws Exception {
+ //1. Create nodes with properties
+ NodeBuilder b1 = store1.getRoot().builder();
+
+ // Add property to node & save
+ for (int i = 0; i < 10; i++) {
+ b1.child("x"+i).setProperty("test"+i, "t", STRING);
+ }
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+ long maxAge = 1; //hours
+ long delta = MINUTES.toMillis(10);
+ //1. Go past GC age and check no GC done as nothing deleted
+ clock.waitUntil(getCurrentTimestamp() + maxAge);
+ VersionGCStats stats = gc(gc, maxAge, HOURS);
+ assertEquals(0, stats.deletedPropsCount);
+ assertEquals(0, stats.updatedFullGCDocsCount);
+
+ //Remove property
+ NodeBuilder b2 = store1.getRoot().builder();
+ for (int i = 0; i < 10; i++) {
+ b2.getChildNode("x"+i).removeProperty("test"+i);
+ }
+ store1.merge(b2, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ //2. Check that a deleted property is not collected before maxAge
+ //Clock cannot move back (it moved forward in #1) so double the maxAge
+ clock.waitUntil(clock.getTime() + delta);
+ stats = gc(gc, maxAge*2, HOURS);
+ assertEquals(0, stats.deletedPropsCount);
+ assertEquals(0, stats.updatedFullGCDocsCount);
+ assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId); // as GC hadn't run
+
+ //3. Check that deleted property does get collected post maxAge
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
+
+ VersionGCSupport gcSupport = new VersionGCSupport(store1.getDocumentStore()) {
+
+ @Override
+ public Iterable<NodeDocument> getModifiedDocs(long fromModified, long toModified, int limit, @NotNull String fromId) {
+ Iterable<NodeDocument> modifiedDocs = super.getModifiedDocs(fromModified, toModified, limit, fromId);
+ List<NodeDocument> result = stream(modifiedDocs.spliterator(), false).collect(toList());
+ final Revision updateRev = newRevision(1);
+ store1.getDocumentStore().findAndUpdate(NODES, stream(modifiedDocs.spliterator(), false)
+ .map(doc -> {
+ UpdateOp op = new UpdateOp(requireNonNull(doc.getId()), false);
+ setModified(op, updateRev);
+ return op;
+ }).
+ collect(toList())
+ );
+ return result;
+ }
+ };
+
+ VersionGarbageCollector gc = new VersionGarbageCollector(store1, gcSupport, true, false, false);
+ stats = gc.gc(maxAge*2, HOURS);
+ assertEquals(0, stats.updatedFullGCDocsCount);
+ assertEquals(0, stats.deletedPropsCount);
+ assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
+ }
+
+ @Test
+ public void cancelFullGCAfterFirstBatch() throws Exception {
+ //1. Create nodes with properties
+ NodeBuilder b1 = store1.getRoot().builder();
+
+ // Add property to node & save
+ for (int i = 0; i < 5_000; i++) {
+ for (int j = 0; j < 10; j++) {
+ b1.child("z"+i).setProperty("prop"+j, "foo", STRING);
+ }
+ }
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+ long maxAge = 1; //hours
+ long delta = MINUTES.toMillis(10);
+ //1. Go past GC age and check no GC done as nothing deleted
+ clock.waitUntil(getCurrentTimestamp() + maxAge);
+ VersionGCStats stats = gc(gc, maxAge, HOURS);
+ assertStatsCountsZero(stats);
+
+ //Remove property
+ NodeBuilder b2 = store1.getRoot().builder();
+ for (int i = 0; i < 5_000; i++) {
+ for (int j = 0; j < 10; j++) {
+ b2.getChildNode("z"+i).removeProperty("prop"+j);
+ }
+ }
+ store1.merge(b2, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ final AtomicReference<VersionGarbageCollector> gcRef = Atomics.newReference();
+ final VersionGCSupport gcSupport = new VersionGCSupport(store1.getDocumentStore()) {
+
+ @Override
+ public Iterable<NodeDocument> getModifiedDocs(long fromModified, long toModified, int limit, @NotNull String fromId) {
+ return () -> new AbstractIterator<>() {
+ private final Iterator<NodeDocument> it = candidates(fromModified, toModified, limit, fromId);
+
+ @Override
+ protected NodeDocument computeNext() {
+ if (it.hasNext()) {
+ return it.next();
+ }
+ // cancel when we reach the end
+ gcRef.get().cancel();
+ return endOfData();
+ }
+ };
+ }
+
+ private Iterator<NodeDocument> candidates(long fromModified, long toModified, int limit, @NotNull String fromId) {
+ return super.getModifiedDocs(fromModified, toModified, limit, fromId).iterator();
+ }
+ };
+
+ gcRef.set(new VersionGarbageCollector(store1, gcSupport, true, false, false));
+
+ //3. Check that deleted property does get collected post maxAge
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
+ stats = gcRef.get().gc(maxAge*2, HOURS);
+ assertTrue(stats.canceled);
+ assertEquals(0, stats.updatedFullGCDocsCount);
+ assertEquals(0, stats.deletedPropsCount);
+ assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
+ }
+
+ // OAK-10199 END
+
+ @SuppressWarnings("unchecked")
+ /**
+ * Test when a revision on a parent is becoming garbage on a property
+ * but not on "_revision" as it is (potentially but in this case indeed)
+ * still required by a child (2:/parent/child "ck").
+ * Note that this is not involving and branch commits.
+ */
+ @Test
+ public void parentWithGarbageGCChildIndependent() throws Exception { // rename me
+ assumeTrue(fixture.hasSinglePersistence());
+ NodeBuilder nb = store1.getRoot().builder();
+ NodeBuilder parent = nb.child("parent");
+ parent.setProperty("pk", "pv");
+ parent.child("child").setProperty("ck", "cv");
+ store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+ // now make a change that later can ge GCed on parent but not on child
+ nb = store1.getRoot().builder();
+ nb.child("parent").setProperty("pk", "pv2");
+ nb.child("parent").child("child").setProperty("ck", "cv2");
+ store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ nb = store1.getRoot().builder();
+ nb.child("parent").setProperty("pk", "pv3");
+ store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ // not doing a bgops == sweep here to cause "cv2" revision
+ // not being considered as commited by sweep
+ // (the test goal is to cause that resolution to fail in case
+ // we would delete the parent's _revisions entry)
+
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+
+ // check state of 2:/parent/child.ck before GC
+ // invalidate a bunch of caches, in particular nodecache and commitvaluecache
+ store1.getNodeCache().invalidateAll();
+ store1.getNodeChildrenCache().invalidateAll();
+ CachingCommitValueResolver cvr = (CachingCommitValueResolver) readField(store1, "commitValueResolver", true);
+ Cache<Revision, String> commitValueCache = (Cache<Revision, String>) readField( cvr, "commitValueCache", true);
+ commitValueCache.invalidateAll();
+ PropertyState ckBefore = store1.getRoot().getChildNode("parent").getChildNode("child").getProperty("ck");
+ assertNotNull(ckBefore);
+
+ // now the GC
+ VersionGCStats stats = gc(gc, 1, HOURS);
+
+ // check state of 2:/parent/child.ck after GC - should match before GC state
+ // invalidate a bunch of caches, in particular nodecache and commitvaluecache
+ store1.getNodeCache().invalidateAll();
+ store1.getNodeChildrenCache().invalidateAll();
+ cvr = (CachingCommitValueResolver) readField(store1, "commitValueResolver", true);
+ commitValueCache = (Cache<Revision, String>) readField( cvr, "commitValueCache", true);
+ commitValueCache.invalidateAll();
+ PropertyState ckAfter = store1.getRoot().getChildNode("parent").getChildNode("child").getProperty("ck");
+ assertNotNull(ckAfter);
+ assertEquals(ckBefore.getValue(Type.STRING), ckAfter.getValue(Type.STRING));
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
+ keepOneFull(0, 0, 0, 3, 0, 0, 2),
+ keepOneUser(0, 0, 0, 3, 0, 0, 2),
+ unmergedBcs(),
+ betweenChkp(0, 0, 0, 1, 1, 0, 2),
+ btwnChkpUBC(0, 0, 0, 1, 1, 0, 2));
+ }
+
+ @Test
+ public void parentGCChildIndependent() throws Exception {
+ assumeTrue(fixture.hasSinglePersistence());
+ NodeBuilder nb = store1.getRoot().builder();
+ NodeBuilder parent = nb.child("parent");
+ parent.setProperty("pk", "pv");
+ parent.child("child").setProperty("ck", "cv");
+ store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ nb = store1.getRoot().builder();
+ nb.child("parent").setProperty("pk", "pv2");
+ store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+
+ // now make a child change so that only parent (and root) gets GCed
+ nb = store1.getRoot().builder();
+ nb.child("parent").child("child").setProperty("ck", "cv2");
+ store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ // now unlike usually, DONT do a store1.runBackgroundOperations() here
+ // this will leave the parent GC-able
+
+ // now the GC
+ VersionGCStats stats = gc(gc, 1, HOURS);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
+ keepOneFull(0, 0, 0, 1, 0, 0, 1),
+ keepOneUser(0, 0, 0, 1, 0, 0, 1),
+ unmergedBcs(),
+ betweenChkp(0, 0, 0, 0, 1, 0, 1),
+ btwnChkpUBC(0, 0, 0, 0, 1, 0, 1));
+ }
+
+ @Test
+ public void testPartialMergeRootCleanup() throws Exception {
+ assumeTrue(fixture.hasSinglePersistence());
+
+ createSecondaryStore(LeaseCheckMode.LENIENT, true);
+
+ NodeBuilder nb = store2.getRoot().builder();
+ nb.child("node1").setProperty("a", "1");
+ store2.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store2.runBackgroundOperations();
+
+ // create the orphaned paths
+ final FailingDocumentStore fds = (FailingDocumentStore) ds2;
+ fds.fail().after(1).eternally();
+
+ // create partial merge
+ nb = store2.getRoot().builder();
+ nb.child("node1").setProperty("a", "2");
+ nb.setProperty("rootProp1", "rootValue1");
+ try {
+ store2.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ fail("expected 'OakOak0001: write operation failed'");
+ } catch(CommitFailedException cfe) {
+ assertEquals("OakOak0001: write operation failed", cfe.getMessage());
+ }
+
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ // clean everything older than one hour
+
+ // before the gc, 1h+ later, do a last-minute modification (only) on /node1 (without root update)
+ nb = store1.getRoot().builder();
+ nb.child("node1").setProperty("b", "4");
+ try {
+ store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ fail("should fail");
+ } catch(Exception e) {
+ // expected to fail
+ }
+ VersionGCStats stats = gc(gc, 1, HOURS);
+ store1.runBackgroundOperations();
+ store1.runBackgroundOperations();
+ createSecondaryStore(LeaseCheckMode.LENIENT);
+ NodeState node1 = store2.getRoot().getChildNode("node1");
+ assertEquals("1", node1.getProperty("a").getValue(Type.STRING));
+ assertFalse(node1.hasProperty("b"));
+ assertStatsCountsZero(stats);
+ }
+
+ @Test
+ public void testUnmergedBCRootCleanup() throws Exception {
+ assumeTrue(fixture.hasSinglePersistence());
+ NodeBuilder nb = store1.getRoot().builder();
+ nb.child("node1").setProperty("a", "1");
+ store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ // create branch commits
+ RevisionVector br1 = unmergedBranchCommit(store1, b -> b.child("node1").setProperty("a", "2"));
+ store1.runBackgroundOperations();
+ invalidateCaches(store1);
+ assertEquals("1", store1.getRoot().getChildNode("node1").getProperty("a").getValue(Type.STRING));
+
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+
+ invalidateCaches(store1);
+ assertEquals("1", store1.getRoot().getChildNode("node1").getProperty("a").getValue(Type.STRING));
+
+ // clean everything older than one hour
+
+ // before the gc, 1h+ later, do a last-minute modification (only) on /node1 (without root update)
+ nb = store1.getRoot().builder();
+ nb.child("node1").setProperty("b", "4");
+ store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ VersionGCStats stats = gc(gc, 1, HOURS);
+ invalidateCaches(store1);
+ assertEquals("1", store1.getRoot().getChildNode("node1").getProperty("a").getValue(Type.STRING));
+ store1.runBackgroundOperations();
+ invalidateCaches(store1);
+ assertEquals("1", store1.getRoot().getChildNode("node1").getProperty("a").getValue(Type.STRING));
+ store1.runBackgroundOperations();
+ invalidateCaches(store1);
+ assertEquals("1", store1.getRoot().getChildNode("node1").getProperty("a").getValue(Type.STRING));
+ createSecondaryStore(LeaseCheckMode.LENIENT);
+
+ // while "2" was written to node1/a via an unmerged branch commit,
+ // it should not have been made visible through DGC/sweep combo
+ invalidateCaches(store2);
+ assertEquals("1", store2.getRoot().getChildNode("node1").getProperty("a").getValue(Type.STRING));
+ assertEquals("4", store2.getRoot().getChildNode("node1").getProperty("b").getValue(Type.STRING));
+ invalidateCaches(store2);
+ assertEquals("1", store2.getRoot().getChildNode("node1").getProperty("a").getValue(Type.STRING));
+ assertEquals("4", store2.getRoot().getChildNode("node1").getProperty("b").getValue(Type.STRING));
+
+ // deletedPropsCount=0 : _bc on /node1 and / CANNOT be removed
+ // deletedPropRevsCount=1 : (nothing on /node1[a, _commitRoot), /[_revisions]
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
+ keepOneFull(0, 0, 1, 0, 1, 0, 1),
+ keepOneUser(),
+ unmergedBcs(0, 0, 1, 0, 1, 1, 1),
+ betweenChkp(0, 0, 0, 0, 1, 0, 1),
+ btwnChkpUBC(0, 0, 1, 0, 2, 1, 1));
+ // checking for br1 revisino to have disappeared doesn't really make much sense,
+ // since 1:/node1 isn't GCed as it is young, and 0:/ being root cannot guarantee full removal
+ // (if br1 is deleted form 0:/ _bc, then the commit value resolution flips it to committed)
+ assertBranchRevisionRemovedFromAllDocuments(store1, br1, "1:/node1");
+ }
+
+ // OAK-8646
+ @Test
+ public void testDeletedPropsAndUnmergedBCWithoutCollision() throws Exception {
+ // create a node with property.
+ NodeBuilder nb = store1.getRoot().builder();
+ nb.child("bar").setProperty("prop", "value");
+ nb.child("bar").setProperty("x", "y");
+ store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ // remove the property
+ nb = store1.getRoot().builder();
+ nb.child("bar").removeProperty("prop");
+ store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ // create branch commits
+ mergedBranchCommit(store1, b -> b.child("foo").setProperty("p", "prop"));
+ RevisionVector br1 = unmergedBranchCommit(store1, b -> b.child("foo").setProperty("a", "b"));
+ RevisionVector br4 = unmergedBranchCommit(store1, b -> b.child("bar").setProperty("x", "z"));
+ mergedBranchCommit(store1, b -> b.child("foo").removeProperty("p"));
+ store1.runBackgroundOperations();
+
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ // clean everything older than one hour
+ VersionGCStats stats = gc(gc, 1, HOURS);
+
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 3, 0, 0, 0, 0, 2),
+ allOrphProp(0, 3, 0, 0, 0, 0, 2),
+ keepOneFull(0, 3, 1, 1, 9, 0, 3),
+ keepOneUser(0, 3, 0, 1, 0, 0, 2),
+ unmergedBcs(0, 3, 1, 1, 7, 2, 3),
+ betweenChkp(0, 3, 0, 0, 1, 0, 3),
+ btwnChkpUBC(0, 3, 1, 1, 8, 2, 3));
+ if (!isModeOneOf(FullGCMode.NONE, FullGCMode.GAP_ORPHANS, FullGCMode.GAP_ORPHANS_EMPTYPROPS)) {
+ assertBranchRevisionRemovedFromAllDocuments(store1, br1);
+ assertBranchRevisionRemovedFromAllDocuments(store1, br4);
+ }
+ }
+
+ @Test
+ public void testDeletedPropsAndUnmergedBCWithCollision() throws Exception {
+ // create a node with property.
+ NodeBuilder nb = store1.getRoot().builder();
+ nb.child("bar").setProperty("prop", "value");
+ nb.child("bar").setProperty("x", "y");
+ store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ // remove the property
+ nb = store1.getRoot().builder();
+ nb.child("bar").removeProperty("prop");
+ store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ // create branch commits
+ mergedBranchCommit(store1, b -> b.child("foo").setProperty("p", "prop"));
+ RevisionVector br1 = unmergedBranchCommit(store1, b -> b.child("foo").setProperty("a", "b"));
+ RevisionVector br2 = unmergedBranchCommit(store1, b -> b.child("foo").setProperty("a", "c"));
+ RevisionVector br3 = unmergedBranchCommit(store1, b -> b.child("foo").setProperty("a", "d"));
+ RevisionVector br4 = unmergedBranchCommit(store1, b -> b.child("bar").setProperty("x", "z"));
+ mergedBranchCommit(store1, b -> b.child("foo").removeProperty("p"));
+ store1.runBackgroundOperations();
+
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ // clean everything older than one hour
+
+ VersionGCStats stats = gc(gc, 1, HOURS);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 3, 0, 0, 0, 0, 2),
+ allOrphProp(0, 3, 0, 0, 0, 0, 2),
+ keepOneFull(0, 3, 2, 1, 17, 0, 3),
+ keepOneUser(0, 3, 0, 1, 0, 0, 2),
+ unmergedBcs(0, 3, 2, 1, 15, 4, 3),
+ betweenChkp(0, 3, 0, 0, 1, 0, 3),
+ btwnChkpUBC(0, 3, 2, 1, 16, 4, 3));
+ if (!isModeOneOf(FullGCMode.NONE, FullGCMode.GAP_ORPHANS, FullGCMode.GAP_ORPHANS_EMPTYPROPS)) {
+ assertBranchRevisionRemovedFromAllDocuments(store1, br1);
+ assertBranchRevisionRemovedFromAllDocuments(store1, br2);
+ assertBranchRevisionRemovedFromAllDocuments(store1, br3);
+ assertBranchRevisionRemovedFromAllDocuments(store1, br4);
+ }
+ }
+
+ static boolean isModeOneOf(FullGCMode... modes) {
+ for (FullGCMode rdgcType : modes) {
+ if (VersionGarbageCollector.getFullGcMode() == rdgcType) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ // OAK-8646 END
+
+ /**
+ * Tests whether FullGC properly deletes a late-written addChild "/grand/parent/a"
+ */
+ @Test
+ public void lateWriteCreateChildGC() throws Exception {
+ doLateWriteCreateChildrenGC(of("/grand/parent"), of("/grand/parent/a"), "/d",
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(1, 0, 0, 0, 0, 0, 1),
+ keepOneFull(1, 0, 0, 0, 0, 0, 1),
+ keepOneUser(1, 0, 0, 0, 0, 0, 1),
+ unmergedBcs(1, 0, 0, 0, 0, 0, 1),
+ betweenChkp(1, 0, 0, 0, 2, 0, 2),
+ btwnChkpUBC(1, 0, 0, 0, 2, 0, 2));
+ }
+
+ /**
+ * Tests whether FullGC can delete a whole subtree "/a/b/c/d/**" that was
+ * added via late-writes.
+ */
+ @Test
+ public void lateWriteCreateChildTreeGC() throws Exception {
+ doLateWriteCreateChildrenGC(of("/a", "/a/b/c"), of("/a/b/c/d", "/a/b/c/d/e/f"), "/d",
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(3, 0, 0, 0, 0, 0, 3),
+ keepOneFull(3, 0, 0, 0, 0, 0, 3),
+ keepOneUser(3, 0, 0, 0, 0, 0, 3),
+ unmergedBcs(3, 0, 0, 0, 0, 0, 3),
+ betweenChkp(3, 0, 0, 0, 3/*4*/, 0, 4),
+ btwnChkpUBC(3, 0, 0, 0, 3/*4*/, 0, 4));
+ }
+
+ /**
+ * Same as {@link #lateWriteCreateChildGC()} except with a long path
+ * (where the id becomes a hash)
+ */
+ @Test
+ public void lateWriteCreateChildGCLargePath() throws Exception {
+ String longPath = repeat("p", PATH_LONG + 1);
+ String path = "/grand/parent/" + longPath + "/longPathChild";
+ doLateWriteCreateChildrenGC(of("/grand/parent"), of(path), "/d",
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(2, 0, 0, 0, 0, 0, 2),
+ keepOneFull(2, 0, 0, 0, 0, 0, 2),
+ keepOneUser(2, 0, 0, 0, 0, 0, 2),
+ unmergedBcs(2, 0, 0, 0, 0, 0, 2),
+ betweenChkp(2, 0, 0, 0, 2/*3*/, 0, 3),
+ btwnChkpUBC(2, 0, 0, 0, 2/*3*/, 0, 3));
+ }
+
+ /**
+ * Tests whether FullGC can delete a large amount of randomly
+ * created orphans (that were added in a late-write manner)
+ */
+ @Test
+ public void lateWriteCreateManyChildrenGC() throws Exception {
+ List<String> nonOrphans = of("/a", "/b", "/c");
+ createNodes(nonOrphans);
+ Set<String> orphans = new HashSet<>();
+ Set<String> commonOrphanParents = new HashSet<>();
+ Random r = new Random(43);
+ for(int i = 0; i < 900; i++) {
+ String orphanParent = nonOrphans.get(r.nextInt(3)) + "/" + r.nextInt(42);
+ commonOrphanParents.add(orphanParent);
+ orphans.add(orphanParent + "/" + r.nextInt(24));
+ }
+ int expectedNumOrphanedDocs = orphans.size() + commonOrphanParents.size();
+ int expectedNumInternalPropRevsGCed = 4;//904=expectedNumOrphanedDocs + 1;
+ int expectedNumDocsUpdatedGCed = expectedNumOrphanedDocs + 1;
+ doLateWriteCreateChildrenGC(nonOrphans, orphans, "/d",
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(expectedNumOrphanedDocs, 0, 0, 0, 0, 0, expectedNumOrphanedDocs),
+ keepOneFull(expectedNumOrphanedDocs, 0, 0, 0, 0, 0, expectedNumOrphanedDocs),
+ keepOneUser(expectedNumOrphanedDocs, 0, 0, 0, 0, 0, expectedNumOrphanedDocs),
+ unmergedBcs(expectedNumOrphanedDocs, 0, 0, 0, 0, 0, expectedNumOrphanedDocs),
+ betweenChkp(expectedNumOrphanedDocs, 0, 0, 0, expectedNumInternalPropRevsGCed, 0, expectedNumDocsUpdatedGCed),
+ btwnChkpUBC(expectedNumOrphanedDocs, 0, 0, 0, expectedNumInternalPropRevsGCed, 0, expectedNumDocsUpdatedGCed));
+ }
+
+ @Test
+ public void lateWriteRemoveChildGC_noSweep() throws Exception {
+ assumeTrue(fixture.hasSinglePersistence());
+ enableFullGC(store1.getVersionGarbageCollector());
+ createNodes("/a/b/c/d");
+ lateWriteRemoveNodes(of("/a/b"), null);
+
+ assertTrue(getChildeNodeState(store1, "/a/b/c/d", true).exists());
+
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ // clean everything older than one hour
+ VersionGCStats stats = gc(store1.getVersionGarbageCollector(), 1, HOURS);
+ assertNotNull(stats);
+
+ assertNotNull(store1.getDocumentStore().find(NODES, "2:/a/b"));
+ assertNotNull(store1.getDocumentStore().find(NODES, "4:/a/b/c/d"));
+ assertTrue(getChildeNodeState(store1, "/a/b/c/d", true).exists());
+ // should be 3 as it should clean up the _deleted from /a/b, /a/b/c and /a/b/c/d
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
+ keepOneFull(0, 0, 0, 3, 3, 0, 3),
+ keepOneUser(0, 0, 0, 3, 0, 0, 3),
+ unmergedBcs(),
+ betweenChkp(0, 0, 0, 0, 1, 0, 1),
+ btwnChkpUBC(0, 0, 0, 0, 1, 0, 1));
+ }
+
+ /**
+ * This (re)produces a case where classic GC deletes nodes
+ * but they are still in the nodes cache, eg:
+ * org.apache.jackrabbit.oak.plugins.document.ConflictException:
+ * The node 4:/a/b/c/d does not exist or is already deleted
+ * at base revision r2-0-1,r1-0-2,
+ * branch: null, commit revision: re-0-1]
+ */
+ @Test
+ @Ignore(value = "OAK-10658 : fails currently as invalidation is missing (in classic GC) after late-write-then-sweep-then-GC")
+ public void lateWriteRemoveChildGC_withSweep() throws Exception {
+ assumeTrue(fixture.hasSinglePersistence());
+ enableFullGC(store1.getVersionGarbageCollector());
+ createNodes("/a/b/c/d");
+ lateWriteRemoveNodes(of("/a/b"), "/foo");
+
+ getChildeNodeState(store1, "/a/b/c/d", true);
+
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ // clean everything older than one hour
+ getChildeNodeState(store1, "/a/b/c/d", true);
+ assertNotNull(store1.getDocumentStore().find(NODES, "4:/a/b/c/d"));
+ assertNotNull(store1.getDocumentStore().find(NODES, "3:/a/b/c"));
+ assertNotNull(store1.getDocumentStore().find(NODES, "2:/a/b"));
+
+ VersionGCStats stats = gc(store1.getVersionGarbageCollector(), 1, HOURS);
+ assertNotNull(stats);
+
+ assertNull(store1.getDocumentStore().find(NODES, "4:/a/b/c/d"));
+ assertNull(store1.getDocumentStore().find(NODES, "3:/a/b/c"));
+ assertNull(store1.getDocumentStore().find(NODES, "2:/a/b"));
+
+ // invalidating store1's nodeCache would fix it
+ // but we need that to happen in prod code, not test
+// store1.getNodeCache().invalidateAll();
+
+ // creating /a/b/c again, below late-write-removed /a/b
+ // triggered a ConflictException
+ createNodes("/a/b/c/d/e");
+ getChildeNodeState(store1, "/a/b/c/d/e", true);
+ }
+
+ @Test
+ public void orphanedChildGC() throws Exception {
+ assumeTrue(fixture.hasSinglePersistence());
+ createSecondaryStore(LeaseCheckMode.LENIENT);
+ createNodes(store2, "/a/b/c", "/a/b/c/d/e", "/a/f/g");
+ ds2.remove(NODES, "3:/a/b/c");
+ store2.dispose();
+
+ store1.runBackgroundOperations();
+
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ // clean everything older than one hour
+ enableFullGC(store1.getVersionGarbageCollector());
+ VersionGCStats stats = gc(store1.getVersionGarbageCollector(), 1, HOURS);
+ assertNotNull(stats);
+ // expected 2 updated (deletions): /a/b/c/d and /a/b/c/d/e
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(2, 0, 0, 0, 0, 0, 2),
+ gapOrphProp(2, 0, 0, 0, 0, 0, 2),
+ allOrphProp(2, 0, 0, 0, 0, 0, 2),
+ keepOneFull(2, 0, 0, 0, 0, 0, 2),
+ keepOneUser(2, 0, 0, 0, 0, 0, 2),
+ unmergedBcs(2, 0, 0, 0, 0, 0, 2),
+ betweenChkp(2, 0, 0, 0, 3, 0, 4),
+ btwnChkpUBC(2, 0, 0, 0, 3, 0, 4));
+
+ if (isModeOneOf(FullGCMode.NONE)) {
+ // in these modes the inconsistency isn't cleaned up
+ // which means there will be a OakMerge0004 exception upon
+ // trying to create the node(s) again.
+ // hence we can't really do this in thsse modes
+ } else {
+ createNodes("/a/b/c/d/e");
+ }
+ }
+
+ @Ignore(value="this is a reminder to add bundling-fullGC tests in general, plus some of those cases combined with OAK-10542")
+ @Test
+ public void testBundlingAndLatestSplit() throws Exception {
+ fail("yet to be implemented");
+ }
+
+ @Test
+ public void testBundledPropUnmergedBCGC() throws Exception {
+ //0. Initialize bundling configs
+ final NodeBuilder builder = store1.getRoot().builder();
+ new InitialContent().initialize(builder);
+ BundlingConfigInitializer.INSTANCE.initialize(builder);
+ merge(store1, builder);
+ store1.runBackgroundOperations();
+
+ //1. Create nodes with properties
+ NodeBuilder b1 = store1.getRoot().builder();
+ b1.child("x").setProperty("jcr:primaryType", "nt:file", NAME);
+ String nasty_key1 = "nas.ty_key$%^";
+ String nasty_key2 = "nas.ty_key$%^2";
+ b1.child("x").setProperty(nasty_key1, "v", STRING);
+ b1.child("x").child("jcr:content").setProperty(nasty_key2, "v2", STRING);
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ NodeBuilder nb = store1.getRoot().builder();
+ nb.child("x").setProperty(nasty_key1, "v3", STRING);
+ nb.child("x").child("jcr:content").setProperty("prop", "value");
+ store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ // create branch commits
+ mergedBranchCommit(store1, b -> b.child("x").child("jcr:content").setProperty("p", "prop"));
+ unmergedBranchCommit(store1, b -> b.child("x").child("jcr:content").setProperty("a", "b"));
+ unmergedBranchCommit(store1, b -> b.child("x").child("jcr:content").setProperty(nasty_key2, "c"));
+ unmergedBranchCommit(store1, b -> b.child("x").child("jcr:content").setProperty(nasty_key2, "d"));
+ mergedBranchCommit(store1, b -> b.child("x").child("jcr:content").removeProperty("p"));
+ store1.runBackgroundOperations();
+
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+ long maxAgeHours = 1;
+ long maxAgeMillis = TimeUnit.HOURS.toMillis(maxAgeHours);
+ //1. Go past GC age and check no GC done as nothing deleted
+ clock.waitUntil(getCurrentTimestamp() + maxAgeMillis + 1);
+
+ VersionGCStats stats = gc(gc, maxAgeHours, HOURS);
+ // below is an example of how the different modes result in different cleanups
+ // this might help us narrow down differences in the modes
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 2, 0, 0, 0, 0, 1),
+ allOrphProp(0, 2, 0, 0, 0, 0, 1),
+ keepOneFull(0, 2, 2, 3, 11, 0, 2),
+ keepOneUser(0, 2, 0, 3, 0, 0, 1),
+ unmergedBcs(0, 2, 1, 2, 12, 3, 2),
+ betweenChkp(0, 2, 0, 0, 1, 0, 2),
+ btwnChkpUBC(0, 2, 1, 2, 13, 3, 2));
+ }
+
+ static GCCounts gapOrphOnly() {
+ return new GCCounts(FullGCMode.GAP_ORPHANS);
+ }
+
+ static GCCounts gapOrphOnly(int deletedDocGCCount, int deletedPropsCount,
+ int deletedInternalPropsCount, int deletedPropRevsCount,
+ int deletedInternalPropRevsCount, int deletedUnmergedBCCount,
+ int updatedFullGCDocsCount) {
+ assertEquals(0, deletedInternalPropsCount);
+ assertEquals(0, deletedPropRevsCount);
+ assertEquals(0, deletedInternalPropRevsCount);
+ assertEquals(0, deletedUnmergedBCCount);
+ return new GCCounts(FullGCMode.GAP_ORPHANS, deletedDocGCCount,
+ deletedPropsCount, deletedInternalPropsCount, deletedPropRevsCount,
+ deletedInternalPropRevsCount, deletedUnmergedBCCount,
+ updatedFullGCDocsCount);
+ }
+
+ static GCCounts gapOrphProp() {
+ return new GCCounts(FullGCMode.GAP_ORPHANS_EMPTYPROPS);
+ }
+
+ static GCCounts gapOrphProp(int deletedDocGCCount, int deletedPropsCount,
+ int deletedInternalPropsCount, int deletedPropRevsCount,
+ int deletedInternalPropRevsCount, int deletedUnmergedBCCount,
+ int updatedFullGCDocsCount) {
+ assertEquals(0, deletedInternalPropsCount);
+ assertEquals(0, deletedPropRevsCount);
+ assertEquals(0, deletedInternalPropRevsCount);
+ assertEquals(0, deletedUnmergedBCCount);
+ return new GCCounts(FullGCMode.GAP_ORPHANS_EMPTYPROPS, deletedDocGCCount,
+ deletedPropsCount, deletedInternalPropsCount, deletedPropRevsCount,
+ deletedInternalPropRevsCount, deletedUnmergedBCCount,
+ updatedFullGCDocsCount);
+ }
+
+ static GCCounts allOrphProp() {
+ return new GCCounts(FullGCMode.ALL_ORPHANS_EMPTYPROPS);
+ }
+
+ static GCCounts allOrphProp(int deletedDocGCCount, int deletedPropsCount,
+ int deletedInternalPropsCount, int deletedPropRevsCount,
+ int deletedInternalPropRevsCount, int deletedUnmergedBCCount,
+ int updatedFullGCDocsCount) {
+ assertEquals(0, deletedInternalPropsCount);
+ assertEquals(0, deletedPropRevsCount);
+ assertEquals(0, deletedInternalPropRevsCount);
+ assertEquals(0, deletedUnmergedBCCount);
+ return new GCCounts(FullGCMode.ALL_ORPHANS_EMPTYPROPS, deletedDocGCCount,
+ deletedPropsCount, deletedInternalPropsCount, deletedPropRevsCount,
+ deletedInternalPropRevsCount, deletedUnmergedBCCount,
+ updatedFullGCDocsCount);
+ }
+
+ static GCCounts keepOneFull() {
+ return new GCCounts(FullGCMode.ORPHANS_EMPTYPROPS_KEEP_ONE_ALL_PROPS);
+ }
+
+ static GCCounts keepOneFull(int deletedDocGCCount, int deletedPropsCount,
+ int deletedInternalPropsCount, int deletedPropRevsCount,
+ int deletedInternalPropRevsCount, int deletedUnmergedBCCount,
+ int updatedFullGCDocsCount) {
+ return new GCCounts(FullGCMode.ORPHANS_EMPTYPROPS_KEEP_ONE_ALL_PROPS, deletedDocGCCount,
+ deletedPropsCount, deletedInternalPropsCount, deletedPropRevsCount,
+ deletedInternalPropRevsCount, deletedUnmergedBCCount,
+ updatedFullGCDocsCount);
+ }
+
+ static GCCounts keepOneUser() {
+ return new GCCounts(FullGCMode.ORPHANS_EMPTYPROPS_KEEP_ONE_USER_PROPS);
+ }
+
+ static GCCounts keepOneUser(int deletedDocGCCount, int deletedPropsCount,
+ int deletedInternalPropsCount, int deletedPropRevsCount,
+ int deletedInternalPropRevsCount, int deletedUnmergedBCCount,
+ int updatedFullGCDocsCount) {
+ return new GCCounts(FullGCMode.ORPHANS_EMPTYPROPS_KEEP_ONE_USER_PROPS,
+ deletedDocGCCount, deletedPropsCount, deletedInternalPropsCount,
+ deletedPropRevsCount, deletedInternalPropRevsCount,
+ deletedUnmergedBCCount, updatedFullGCDocsCount);
+ }
- @BeforeClass
- public static void before() throws Exception {
- for (Thread t : Thread.getAllStackTraces().keySet()) {
- tbefore.add(t);
- }
+ static GCCounts unmergedBcs() {
+ return new GCCounts(FullGCMode.ORPHANS_EMPTYPROPS_UNMERGED_BC);
}
- @AfterClass
- public static void after() throws Exception {
- for (Thread t : Thread.getAllStackTraces().keySet()) {
- if (!tbefore.contains(t)) {
- System.err.println("potentially leaked thread: " + t);
- }
- }
+ static GCCounts unmergedBcs(int deletedDocGCCount, int deletedPropsCount,
+ int deletedInternalPropsCount, int deletedPropRevsCount,
+ int deletedInternalPropRevsCount, int deletedUnmergedBCCount,
+ int updatedFullGCDocsCount) {
+ return new GCCounts(FullGCMode.ORPHANS_EMPTYPROPS_UNMERGED_BC,
+ deletedDocGCCount, deletedPropsCount, deletedInternalPropsCount,
+ deletedPropRevsCount, deletedInternalPropRevsCount,
+ deletedUnmergedBCCount, updatedFullGCDocsCount);
+ }
+
+ static GCCounts betweenChkp() {
+ return new GCCounts(FullGCMode.ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_NO_UNMERGED_BC);
+ }
+
+ static GCCounts betweenChkp(int deletedDocGCCount, int deletedPropsCount,
+ int deletedInternalPropsCount, int deletedPropRevsCount,
+ int deletedInternalPropRevsCount, int deletedUnmergedBCCount,
+ int updatedFullGCDocsCount) {
+ return new GCCounts(FullGCMode.ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_NO_UNMERGED_BC,
+ deletedDocGCCount, deletedPropsCount, deletedInternalPropsCount,
+ deletedPropRevsCount, deletedInternalPropRevsCount,
+ deletedUnmergedBCCount, updatedFullGCDocsCount);
+ }
+
+ static GCCounts btwnChkpUBC() {
+ return new GCCounts(FullGCMode.ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_WITH_UNMERGED_BC);
+ }
+
+ static GCCounts btwnChkpUBC(int deletedDocGCCount, int deletedPropsCount,
+ int deletedInternalPropsCount, int deletedPropRevsCount,
+ int deletedInternalPropRevsCount, int deletedUnmergedBCCount,
+ int updatedFullGCDocsCount) {
+ return new GCCounts(FullGCMode.ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_WITH_UNMERGED_BC,
+ deletedDocGCCount, deletedPropsCount, deletedInternalPropsCount,
+ deletedPropRevsCount, deletedInternalPropRevsCount,
+ deletedUnmergedBCCount, updatedFullGCDocsCount);
}
@Test
- public void gcIgnoredForCheckpoint() throws Exception {
- long expiryTime = 100, maxAge = 20;
+ public void testBundledPropRevGC() throws Exception {
+ //0. Initialize bundling configs
+ final NodeBuilder builder = store1.getRoot().builder();
+ new InitialContent().initialize(builder);
+ BundlingConfigInitializer.INSTANCE.initialize(builder);
+ merge(store1, builder);
+ store1.runBackgroundOperations();
- Revision cp = Revision.fromString(store1.checkpoint(expiryTime));
+ //1. Create nodes with properties
+ NodeBuilder b1 = store1.getRoot().builder();
+ b1.child("x").setProperty("jcr:primaryType", "nt:file", NAME);
- //Fast forward time to future but before expiry of checkpoint
- clock.waitUntil(cp.getTimestamp() + expiryTime - maxAge);
- VersionGCStats stats = gc.gc(maxAge, TimeUnit.MILLISECONDS);
- assertTrue(stats.ignoredGCDueToCheckPoint);
+ // Add property to node & save
+ for (int i = 0; i < 10; i++) {
+ b1.child("x").child("jcr:content").setProperty("bprop"+i, "t", STRING);
+ b1.child("x").setProperty(META_PROP_PATTERN, of("jcr:content"), STRINGS);
+ b1.child("x").setProperty("prop"+i, "bar", STRING);
+ }
+ String nasty_key1 = "nas.ty_key$%^";
+ String nasty_key2 = "nas.ty_key$%^2";
+ b1.child("x").setProperty(nasty_key1, "v", STRING);
+ b1.child("x").child("jcr:content").setProperty(nasty_key2, "v2", STRING);
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
- //Fast forward time to future such that checkpoint get expired
- clock.waitUntil(clock.getTime() + expiryTime + 1);
- stats = gc.gc(maxAge, TimeUnit.MILLISECONDS);
- assertFalse("GC should be performed", stats.ignoredGCDueToCheckPoint);
+ // make some overwrites for FullGC to cleanup
+ b1 = store1.getRoot().builder();
+ for (int i = 0; i < 6; i++) {
+ b1.child("x").child("jcr:content").setProperty("bprop"+i, "t2", STRING);
+ }
+ for (int i = 0; i < 3; i++) {
+ b1.child("x").setProperty("prop"+i, "bar2", STRING);
+ }
+ b1.child("x").setProperty(nasty_key1, "bv", STRING);
+ b1.child("x").child("jcr:content").setProperty(nasty_key2, "bv2", STRING);
+ store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ // enable the full gc flag
+ writeField(gc, "fullGCEnabled", true, true);
+ long maxAgeHours = 1;
+ long maxAgeMillis = TimeUnit.HOURS.toMillis(maxAgeHours);
+ //1. Go past GC age and check no GC done as nothing deleted
+ clock.waitUntil(getCurrentTimestamp() + maxAgeMillis + 1);
+ VersionGCStats stats = gc(gc, maxAgeHours, HOURS);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
+ keepOneFull(0, 0, 0, 11, 0, 0, 1),
+ keepOneUser(0, 0, 0, 11, 0, 0, 1),
+ unmergedBcs(),
+ betweenChkp(0, 0, 0, 0, 1, 0, 1),
+ btwnChkpUBC(0, 0, 0, 0, 1, 0, 1));
+
+ NodeState x = store1.getRoot().getChildNode("x");
+ assertTrue(x.exists());
+ assertEquals(x.getProperty("prop0").getValue(Type.STRING), "bar2");
+ assertEquals(x.getProperty("prop9").getValue(Type.STRING), "bar");
+ NodeState jcrContent = x.getChildNode("jcr:content");
+ assertTrue(jcrContent.exists());
+ assertEquals(jcrContent.getProperty("bprop0").getValue(Type.STRING), "t2");
+ assertEquals(jcrContent.getProperty("bprop9").getValue(Type.STRING), "t");
+
+ NodeDocument doc = store1.getDocumentStore().find(NODES, "1:/x", -1);
+ assertNotNull(doc);
+ if (VersionGarbageCollector.getFullGcMode() == FullGCMode.ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_WITH_UNMERGED_BC
+ || VersionGarbageCollector.getFullGcMode() == FullGCMode.NONE || VersionGarbageCollector.getFullGcMode() == FullGCMode.GAP_ORPHANS
+ || VersionGarbageCollector.getFullGcMode() == FullGCMode.GAP_ORPHANS_EMPTYPROPS
+ || VersionGarbageCollector.getFullGcMode() == FullGCMode.ALL_ORPHANS_EMPTYPROPS
+ || VersionGarbageCollector.getFullGcMode() == FullGCMode.ORPHANS_EMPTYPROPS_UNMERGED_BC
+ || VersionGarbageCollector.getFullGcMode() == FullGCMode.ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_NO_UNMERGED_BC) {
+ // this mode doesn't currently delete all revisions,
+ // thus would fail below assert.
+ return;
+ }
+ for (Entry<String, Object> e : doc.entrySet()) {
+ Object v = e.getValue();
+ if (v instanceof Map) {
+ @SuppressWarnings("rawtypes")
+ Map m = (Map)v;
+ assertEquals("more than 1 entry for " + e.getKey(), 1, m.size());
+ }
+ }
}
+ // OAK-10370
@Test
- public void testGCDeletedDocument() throws Exception{
- //1. Create nodes
+ public void testGCDeletedPropsWithDryRunMode() throws Exception {
+ //1. Create nodes with properties
NodeBuilder b1 = store1.getRoot().builder();
- b1.child("x").child("y");
- b1.child("z");
+ b1.child("x").setProperty("test", "t", STRING);
+ b1.child("z").setProperty("prop", "foo", STRING);
store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ // enable the full gc flag
+ enableFullGC(gc);
long maxAge = 1; //hours
- long delta = TimeUnit.MINUTES.toMillis(10);
- //1. Go past GC age and check no GC done as nothing deleted
- clock.waitUntil(Revision.getCurrentTimestamp() + maxAge);
- VersionGCStats stats = gc.gc(maxAge, HOURS);
- assertEquals(0, stats.deletedDocGCCount);
+ long delta = MINUTES.toMillis(10);
- //Remove x/y
+ //2. Remove property
NodeBuilder b2 = store1.getRoot().builder();
- b2.child("x").child("y").remove();
+ b2.getChildNode("z").removeProperty("prop");
store1.merge(b2, EmptyHook.INSTANCE, CommitInfo.EMPTY);
store1.runBackgroundOperations();
- //2. Check that a deleted doc is not collected before
- //maxAge
- //Clock cannot move back (it moved forward in #1) so double the maxAge
- clock.waitUntil(clock.getTime() + delta);
- stats = gc.gc(maxAge*2, HOURS);
- assertEquals(0, stats.deletedDocGCCount);
- assertEquals(0, stats.deletedLeafDocGCCount);
-
- //3. Check that deleted doc does get collected post maxAge
+ //3. Check that deleted property does get collected post maxAge
clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
- stats = gc.gc(maxAge*2, HOURS);
- assertEquals(1, stats.deletedDocGCCount);
- assertEquals(1, stats.deletedLeafDocGCCount);
-
- //4. Check that a revived doc (deleted and created again) does not get gc
+ VersionGCStats stats = gc(gc, maxAge*2, HOURS);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 1, 0, 0, 0, 0, 1),
+ allOrphProp(0, 1, 0, 0, 0, 0, 1),
+ keepOneFull(0, 1, 0, 0, 0, 0, 1),
+ keepOneUser(0, 1, 0, 0, 0, 0, 1),
+ unmergedBcs(0, 1, 0, 0, 0, 0, 1),
+ betweenChkp(0, 1, 0, 0, 1, 0, 2),
+ btwnChkpUBC(0, 1, 0, 0, 1, 0, 2));
+ assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
+
+ // 4. Save values of fullGC settings collection fields
+ final String oldestModifiedDocId = stats.oldestModifiedDocId;
+ final long oldestModifiedDocTimeStamp = stats.oldestModifiedDocTimeStamp;
+
+ final Document documentBefore = store1.getDocumentStore().find(SETTINGS, SETTINGS_COLLECTION_ID);
+ assert documentBefore != null;
+ assertEquals(documentBefore.get(SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP), oldestModifiedDocTimeStamp);
+ assertEquals(documentBefore.get(SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP), oldestModifiedDocId);
+
+ //5. Verify that in dryRun mode property does not get gc and fullGC fields remain the same
NodeBuilder b3 = store1.getRoot().builder();
- b3.child("z").remove();
+ b3.child("x").removeProperty("test");
store1.merge(b3, EmptyHook.INSTANCE, CommitInfo.EMPTY);
-
- NodeBuilder b4 = store1.getRoot().builder();
- b4.child("z");
- store1.merge(b4, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
- stats = gc.gc(maxAge*2, HOURS);
- assertEquals(0, stats.deletedDocGCCount);
- assertEquals(0, stats.deletedLeafDocGCCount);
- assertEquals(1, stats.updateResurrectedGCCount);
+ // enabled dryRun mode
+ enableFullGCDryRun(gc);
+ stats = gc(gc, maxAge*2, HOURS);
+
+ final String oldestModifiedDryRunDocId = stats.oldestModifiedDocId;
+ final long oldestModifiedDocDryRunTimeStamp = stats.oldestModifiedDocTimeStamp;
+
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 1, 0, 0, 0, 0, 1),
+ allOrphProp(0, 1, 0, 0, 0, 0, 1),
+ keepOneFull(0, 1, 0, 0, 0, 0, 1),
+ keepOneUser(0, 1, 0, 0, 0, 0, 1),
+ unmergedBcs(0, 1, 0, 0, 0, 0, 1),
+ betweenChkp(0, 1, 0, 0, 1, 0, 2),
+ btwnChkpUBC(0, 1, 0, 0, 1, 0, 2));
+ assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
+ assertTrue(stats.fullGCDryRunMode);
+
+ final Document documentAfter = store1.getDocumentStore().find(SETTINGS, SETTINGS_COLLECTION_ID);
+ assert documentAfter != null;
+ assertEquals(documentAfter.get(SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP), oldestModifiedDocTimeStamp);
+ assertEquals(documentAfter.get(SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP), oldestModifiedDocId);
+
+ assertEquals(documentAfter.get(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_TIMESTAMP_PROP), oldestModifiedDocDryRunTimeStamp);
+ assertEquals(documentAfter.get(SETTINGS_COLLECTION_FULL_GC_DRY_RUN_DOCUMENT_ID_PROP), oldestModifiedDryRunDocId);
}
@Test
- public void gcSplitDocs() throws Exception {
- gcSplitDocsInternal("foo");
+ public void testDeletedPropsAndUnmergedBCWithCollisionWithDryRunMode() throws Exception {
+ // create a node with property.
+ NodeBuilder nb = store1.getRoot().builder();
+ nb.child("bar").setProperty("prop", "value");
+ nb.child("bar").setProperty("x", "y");
+ store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ // remove the property
+ nb = store1.getRoot().builder();
+ nb.child("bar").removeProperty("prop");
+ store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ // create branch commits
+ mergedBranchCommit(store1, b -> b.child("foo").setProperty("p", "prop"));
+ RevisionVector br1 = unmergedBranchCommit(store1, b -> b.child("foo").setProperty("a", "b"));
+ RevisionVector br2 = unmergedBranchCommit(store1, b -> b.child("foo").setProperty("a", "c"));
+ RevisionVector br3 = unmergedBranchCommit(store1, b -> b.child("foo").setProperty("a", "d"));
+ RevisionVector br4 = unmergedBranchCommit(store1, b -> b.child("bar").setProperty("x", "z"));
+ mergedBranchCommit(store1, b -> b.child("foo").removeProperty("p"));
+ store1.runBackgroundOperations();
+
+ // enable the full gc flag
+ enableFullGC(gc);
+ enableFullGCDryRun(gc);
+
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ // clean everything older than one hour
+ VersionGCStats stats = gc(gc, 1, HOURS);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 3, 0, 0, 0, 0, 2),
+ allOrphProp(0, 3, 0, 0, 0, 0, 2),
+ keepOneFull(0, 3, 2, 1,17, 0, 3),
+ keepOneUser(0, 3, 0, 1, 0, 0, 2),
+ unmergedBcs(0, 3, 2, 1,15, 4, 3),
+ betweenChkp(0, 3, 0, 0, 1, 0, 3),
+ btwnChkpUBC(0, 3, 2, 1,16, 4, 3));
+ assertTrue(stats.fullGCDryRunMode);
+
+ assertBranchRevisionNotRemovedFromAllDocuments(store1, br1);
+ assertBranchRevisionNotRemovedFromAllDocuments(store1, br2);
+ assertBranchRevisionNotRemovedFromAllDocuments(store1, br3);
+ assertBranchRevisionNotRemovedFromAllDocuments(store1, br4);
}
+ // OAK-10370 END
+
+ // OAK-10676
@Test
- public void gcLongPathSplitDocs() throws Exception {
- gcSplitDocsInternal(Strings.repeat("sub", 120));
+ public void removePropertyAddedByLateWriteWithoutUnrelatedPath() throws Exception {
+ // create a node with property.
+ assumeTrue(fixture.hasSinglePersistence());
+ NodeBuilder nb = store1.getRoot().builder();
+ nb.child("bar").setProperty("p", "v");
+ store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ // remove the property
+ nb = store1.getRoot().builder();
+ nb.child("bar").removeProperty("p");
+ store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ // unrelated path should be such that the paths and unrelated path shouldn't have common parent
+ // for e.g. if path is /bar & unrelated is /d then there common ancestor is "/" i.e. root.
+ lateWriteAddPropertiesNodes(of("/bar"), null, "p", "v2");
+
+ assertDocumentsExist(of("/bar"));
+ assertPropertyExist("/bar", "p");
+
+ enableFullGC(store1.getVersionGarbageCollector());
+
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ // clean everything older than one hour
+ VersionGCStats stats = gc(store1.getVersionGarbageCollector(), 1, HOURS);
+ assertFalse(store1.getRoot().getChildNode("bar").hasProperty("prop"));
+ assertNotNull(stats);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 1, 0, 0, 0, 0, 1),
+ allOrphProp(0, 1, 0, 0, 0, 0, 1),
+ keepOneFull(0, 1, 0, 0, 0, 0, 1),
+ keepOneUser(0, 1, 0, 0, 0, 0, 1),
+ unmergedBcs(0, 1, 0, 0, 0, 0, 1),
+ betweenChkp(0, 1, 0, 0, 1, 0, 2),
+ btwnChkpUBC(0, 1, 0, 0, 1, 0, 2));
+ assertDocumentsExist(of("/bar"));
+ }
+
+ @Test
+ public void removePropertyAddedByLateWriteWithoutUnrelatedPath_2() throws Exception {
+ // create a node with property.
+ assumeTrue(fixture.hasSinglePersistence());
+ NodeBuilder nb = store1.getRoot().builder();
+ nb.child("foo").child("bar").child("baz").setProperty("prop", "value");
+ store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ // remove the property
+ nb = store1.getRoot().builder();
+ nb.child("foo").child("bar").child("baz").removeProperty("prop");
+ store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ // unrelated path should be such that the paths and unrelated path shouldn't have common parent
+ // for e.g. if path is /bar & unrelated is /d then there common ancestor is "/" i.e. root.
+ lateWriteAddPropertiesNodes(of("/foo/bar/baz"), "/a", "prop", "value2");
+
+ assertDocumentsExist(of("/foo/bar/baz"));
+ assertPropertyExist("/foo/bar/baz", "prop");
+
+ enableFullGC(store1.getVersionGarbageCollector());
+
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ // clean everything older than one hour
+ VersionGCStats stats = gc(store1.getVersionGarbageCollector(), 1, HOURS);
+ assertFalse(store1.getRoot().getChildNode("bar").hasProperty("prop"));
+ assertNotNull(stats);
+ // since we have updated a totally unrelated path i.e. "/a",
+ // we should still be seeing the garbage from late write and
+ // thus it will be collected.
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 1, 0, 0, 0, 0, 1),
+ allOrphProp(0, 1, 0, 0, 0, 0, 1),
+ keepOneFull(0, 1, 0, 0, 0, 0, 1),
+ keepOneUser(0, 1, 0, 0, 0, 0, 1),
+ unmergedBcs(0, 1, 0, 0, 0, 0, 1),
+ betweenChkp(0, 1, 0, 0, 2, 0, 2),
+ btwnChkpUBC(0, 1, 0, 0, 2, 0, 2));
+
+ assertDocumentsExist(of("/foo/bar/baz"));
+ }
+
+ @Test
+ public void removePropertyAddedByLateWriteWithRelatedPath() throws Exception {
+ // create a node with property.
+ assumeTrue(fixture.hasSinglePersistence());
+ NodeBuilder nb = store1.getRoot().builder();
+ nb.child("bar").setProperty("prop", "value");
+ store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ // remove the property
+ nb = store1.getRoot().builder();
+ nb.child("bar").removeProperty("prop");
+ store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ // unrelated path should be such that the paths and unrelated path shouldn't have common parent
+ // for e.g. if path is /bar & unrelated is /d then there common ancestor is "/" i.e. root.
+ lateWriteAddPropertiesNodes(of("/bar"), "/d", "prop", "value2");
+
+ assertDocumentsExist(of("/bar"));
+ assertPropertyExist("/bar", "prop");
+
+ enableFullGC(store1.getVersionGarbageCollector());
+
+ // wait two hours
+ clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
+ // clean everything older than one hour
+ VersionGCStats stats = gc(store1.getVersionGarbageCollector(), 1, HOURS);
+ assertEquals("value2", store1.getRoot().getChildNode("bar").getProperty("prop").getValue(Type.STRING));
+ // deletedPropsCount : we shouldn't be able to remove the property since we have
+ // updated an related path that has lead to an update of common ancestor and
+ // this would make late write visible
+ // deletedPropRevsCount : 2 prop-revs GCed : the original prop=value, plus the
+ // removeProperty(prop) plus 1 _commitRoot entry
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
+ keepOneFull(0, 0, 0, 2, 0, 0, 1),
+ keepOneUser(0, 0, 0, 2, 0, 0, 1),
+ unmergedBcs(),
+ betweenChkp(0, 0, 0, 0, 2, 0, 1),
+ btwnChkpUBC(0, 0, 0, 0, 2, 0, 1));
+ assertDocumentsExist(of("/bar"));
+ }
+
+ @Test
+ public void skipPropertyRemovedByLateWriteWithoutUnrelatedPath() throws Exception {
+ // create a node with property.
+ assumeTrue(fixture.hasSinglePersistence());
+ NodeBuilder nb = store1.getRoot().builder();
+ nb.child("bar").setProperty("p", "value");
+ store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ store1.runBackgroundOperations();
+
+ // unrelated path should be such that the paths and unrelated path shouldn't have common parent
+ // for e.g. if path is /bar & unrelated is /d then there common ancestor is "/" i.e. root.
+ lateWriteRemovePropertiesNodes(of("/bar"), null, false, "p");
+
+ assertDocumentsExist(of("/bar"));
+ assertPropertyNotExist("/bar", "p");
+
+ enableFullGC(store1.getVersionGarbageCollector());
+
+ // wait two hours
... 945 lines suppressed ...