You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by jo...@apache.org on 2024/02/26 23:40:19 UTC
(jackrabbit-oak) 05/05: Some cleanup and work on RevisionCommand (WIP).
This is an automated email from the ASF dual-hosted git repository.
joscorbe pushed a commit to branch old-revisions-cleanup-rebase
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 29d13489ccd9491f9feb11055ce009d2b15e20f4
Author: Jose Cordero <co...@adobe.com>
AuthorDate: Wed Feb 7 14:33:21 2024 +0100
Some cleanup and work on RevisionCommand (WIP).
---
.../oak/console/commands/RevisionCommand.groovy | 19 +-
.../jackrabbit/oak/run/RevisionsCommand.java | 73 +--
.../document/DocumentRevisionCleanupHelper.java | 287 ----------
.../document/NodeDocumentRevisionCleaner.java | 188 +++----
.../DocumentRevisionCleanupHelperTest.java | 620 ---------------------
.../document/NodeDocumentRevisionCleanerTest.java | 298 ++--------
6 files changed, 160 insertions(+), 1325 deletions(-)
diff --git a/oak-run/src/main/groovy/org/apache/jackrabbit/oak/console/commands/RevisionCommand.groovy b/oak-run/src/main/groovy/org/apache/jackrabbit/oak/console/commands/RevisionCommand.groovy
index 8e055d17a9..81ddc2af8d 100644
--- a/oak-run/src/main/groovy/org/apache/jackrabbit/oak/console/commands/RevisionCommand.groovy
+++ b/oak-run/src/main/groovy/org/apache/jackrabbit/oak/console/commands/RevisionCommand.groovy
@@ -22,8 +22,8 @@ package org.apache.jackrabbit.oak.console.commands
import groovy.transform.CompileStatic
import org.apache.jackrabbit.oak.console.ConsoleSession
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore
-import org.apache.jackrabbit.oak.plugins.document.DocumentRevisionCleanupHelper
import org.apache.jackrabbit.oak.plugins.document.DocumentStore
+import org.apache.jackrabbit.oak.plugins.document.NodeDocumentRevisionCleaner
import org.apache.jackrabbit.oak.plugins.document.Revision
import org.apache.jackrabbit.oak.spi.state.NodeStore
import org.codehaus.groovy.tools.shell.CommandSupport
@@ -56,7 +56,7 @@ class RevisionCommand extends CommandSupport {
private static boolean capResults = true
private static boolean showPropertiesUse = false
- private DocumentRevisionCleanupHelper cleanupHelper;
+ private NodeDocumentRevisionCleaner cleanupHelper;
RevisionCommand(Groovysh shell) {
super(shell, COMMAND_NAME, 'rev')
@@ -105,9 +105,9 @@ class RevisionCommand extends CommandSupport {
*/
void list() {
int count = 0
- NavigableMap<Revision, String> allRevisions = cleanupHelper.getAllRevisions();
+ SortedMap<Revision, String> allRevisions = cleanupHelper.getAllRevisions();
- for (Map.Entry<Revision, String> revisionEntry : order == ORDER_ASC ? allRevisions.entrySet() : allRevisions.descendingMap().entrySet()) {
+ for (Map.Entry<Revision, String> revisionEntry : allRevisions.entrySet()) {
Revision revision = revisionEntry.getKey()
String value = revisionEntry.getValue()
io.out.println(revision.toReadableString() + " " + value)
@@ -172,7 +172,7 @@ class RevisionCommand extends CommandSupport {
void cleanup(int numberToCleanup, int clusterToCleanup) {
io.out.println("This will delete the following revisions and the property values permanently:")
SortedMap<Integer, TreeSet<Revision>> candidateRevisionsToClean = cleanupHelper.getCandidateRevisionsToClean()
- SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = cleanupHelper.getPropertiesModifiedByRevision()
+ SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = null;//cleanupHelper.getPropertiesModifiedByRevision()
TreeSet<Revision> revisions = candidateRevisionsToClean.get(clusterToCleanup)
if (revisions != null) {
int count = 0
@@ -223,9 +223,10 @@ class RevisionCommand extends CommandSupport {
DocumentStore documentStore = nodeStore.getDocumentStore()
DocumentNodeStore documentNodeStore = (DocumentNodeStore) nodeStore
- cleanupHelper = new DocumentRevisionCleanupHelper(documentStore, documentNodeStore, session.getWorkingPath())
- cleanupHelper.initializeCleanupProcess()
+ //cleanupHelper = new NodeDocumentRevisionCleaner(documentStore, documentNodeStore, session.get)
+
+ //cleanupHelper.initializeCleanupProcess()
SortedMap<Revision, String> allRevisions = cleanupHelper.getAllRevisions()
TreeMap<Integer, Integer> revisionsByClusterId = allRevisions.keySet().groupBy { it.clusterId }
@@ -233,7 +234,7 @@ class RevisionCommand extends CommandSupport {
[clusterId, revisions.size()]
} as TreeMap<Integer, Integer>
- io.out.println("=== Last Revision by clusterId ===")
+ /*io.out.println("=== Last Revision by clusterId ===")
for (Map.Entry<Integer, Revision> entry : cleanupHelper.getLastRev()) {
io.out.printf(" [%d] -> %s%n", entry.key, entry.value.toReadableString())
}
@@ -243,7 +244,7 @@ class RevisionCommand extends CommandSupport {
for (Map.Entry<Integer, Revision> entry : cleanupHelper.getSweepRev()) {
io.out.printf(" [%d] -> %s%n", entry.key, entry.value.toReadableString())
}
- io.out.println()
+ io.out.println()*/
io.out.println("=== Total Revisions by clusterId ===")
for (Map.Entry<Integer, Integer> entry : revisionsByClusterId) {
diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
index 7c44cefa65..ac72509b26 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
@@ -37,7 +37,6 @@ import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.stream.Collectors;
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.LoggerContext;
@@ -47,13 +46,16 @@ import org.apache.jackrabbit.oak.commons.TimeDurationFormatter;
import org.apache.jackrabbit.oak.plugins.document.ClusterNodeInfoDocument;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder;
-import org.apache.jackrabbit.oak.plugins.document.DocumentRevisionCleanupHelper;
import org.apache.jackrabbit.oak.plugins.document.DocumentStore;
+import org.apache.jackrabbit.oak.plugins.document.DocumentStoreException;
import org.apache.jackrabbit.oak.plugins.document.FormatVersion;
import org.apache.jackrabbit.oak.plugins.document.MissingLastRevSeeker;
+import org.apache.jackrabbit.oak.plugins.document.NodeDocument;
+import org.apache.jackrabbit.oak.plugins.document.NodeDocumentRevisionCleaner;
import org.apache.jackrabbit.oak.plugins.document.Revision;
import org.apache.jackrabbit.oak.plugins.document.RevisionContextWrapper;
import org.apache.jackrabbit.oak.plugins.document.SweepHelper;
+import org.apache.jackrabbit.oak.plugins.document.UpdateOp;
import org.apache.jackrabbit.oak.plugins.document.VersionGCSupport;
import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCInfo;
import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCStats;
@@ -66,7 +68,9 @@ import org.slf4j.LoggerFactory;
import static java.lang.Boolean.TRUE;
import static java.lang.Integer.getInteger;
+import static java.util.Objects.requireNonNull;
import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreHelper.createVersionGC;
import static org.apache.jackrabbit.oak.plugins.document.FormatVersion.versionOf;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getRootDocument;
@@ -496,16 +500,6 @@ public class RevisionsCommand implements Command {
System.err.println("path option is required for " + RevisionsOptions.CMD_CLEANUP + " command");
return;
}
- Integer clusterToCleanup = options.getClusterId();
- if (clusterToCleanup == null) {
- System.err.println("clusterId option is required for " + RevisionsOptions.CMD_CLEANUP + " command");
- return;
- }
- Integer numberToCleanup = options.getNumber();
- if (numberToCleanup == null) {
- System.err.println("number option is required for " + RevisionsOptions.CMD_CLEANUP + " command");
- return;
- }
DocumentNodeStoreBuilder<?> builder = createDocumentMKBuilder(options, closer);
if (builder == null) {
@@ -517,10 +511,14 @@ public class RevisionsCommand implements Command {
builder.setReadOnlyMode();
useMemoryBlobStore(builder);
DocumentNodeStore documentNodeStore = builder.build();
- DocumentRevisionCleanupHelper cleanupHelper = new DocumentRevisionCleanupHelper(documentStore, documentNodeStore, path);
- cleanupHelper.initializeCleanupProcess();
+ String id = org.apache.jackrabbit.oak.plugins.document.util.Utils.getIdFromPath(path);
+ NodeDocument workingDocument = documentStore.find(NODES, id);
+ NodeDocumentRevisionCleaner revisionCleaner = new NodeDocumentRevisionCleaner(documentStore, documentNodeStore, workingDocument);
+
+ final UpdateOp op = new UpdateOp(requireNonNull(id), false);
+ revisionCleaner.initializeCleanupProcess(op);
- SortedMap<Revision, String> allRevisions = cleanupHelper.getAllRevisions();
+ SortedMap<Revision, String> allRevisions = revisionCleaner.getAllRevisions();
SortedMap<Integer, Integer> revisionsByClusterId = new TreeMap<>();
for (Revision revision : allRevisions.keySet()) {
Integer cid = revision.getClusterId();
@@ -531,15 +529,6 @@ public class RevisionsCommand implements Command {
revisionsByClusterId.put(cid, count + 1);
}
- System.out.println("Last Revision by clusterId");
- for (Map.Entry<Integer, Revision> entry : cleanupHelper.getLastRev().entrySet()) {
- System.out.printf(" [%d] -> %s%n", entry.getKey(), entry.getValue().toReadableString());
- }
-
- System.out.println("Sweep Revision by clusterId");
- for (Map.Entry<Integer, Revision> entry : cleanupHelper.getSweepRev().entrySet()) {
- System.out.printf(" [%d] -> %s%n", entry.getKey(), entry.getValue().toReadableString());
- }
System.out.println();
int count = 0;
@@ -553,15 +542,18 @@ public class RevisionsCommand implements Command {
}
}
- for (Map.Entry<Integer, TreeSet<Revision>> entry : cleanupHelper.getCandidateRevisionsToClean().entrySet()) {
+ for (Map.Entry<Integer, TreeSet<Revision>> entry : revisionCleaner.getCandidateRevisionsToClean().entrySet()) {
Integer cid = entry.getKey();
TreeSet<Revision> revisions = entry.getValue();
- int blocked = cleanupHelper.getBlockedRevisionsToKeep().get(cid) != null ? cleanupHelper.getBlockedRevisionsToKeep().get(cid).size() : 0;
+ int blocked = revisionCleaner.getBlockedRevisionsToKeep().get(cid) != null ? revisionCleaner.getBlockedRevisionsToKeep().get(cid).size() : 0;
System.out.printf("ClusterId [%d] has %d Candidates and %d Blocked%n", cid, revisions.size(), blocked);
}
- System.out.println("=== Revisions to be cleaned for clusterId " + clusterToCleanup + " ===");
- TreeSet<Revision> revisionsToClean = cleanupHelper.getCandidateRevisionsToClean().get(clusterToCleanup)
+ System.out.println("=== Operations to be executed ===");
+ System.out.println("Total of operations: " + op.getChanges().size());
+ System.out.println(op.getChanges());
+
+ /*TreeSet<Revision> revisionsToClean = revisionCleaner.getCandidateRevisionsToClean().get(clusterToCleanup)
.stream().limit(numberToCleanup).collect(Collectors.toCollection(TreeSet::new));
if (revisionsToClean.isEmpty()) {
System.out.println("No revisions to clean");
@@ -577,32 +569,17 @@ public class RevisionsCommand implements Command {
break;
}
}
- }
+ }*/
Scanner scanner = new Scanner(System.in);
System.out.println("The revisions will be deleted permanently. Are you sure to proceed? [y/N]");
String confirmation = scanner.nextLine().trim().toLowerCase();
if (confirmation.equals("y") || confirmation.equals("yes")) {
- // Start the cleanup
- count = 0;
- for (Revision revision : cleanupHelper.getCandidateRevisionsToClean().get(clusterToCleanup)) {
- // Remove the revision from _revisions and all the properties
- /*UpdateOp update = new UpdateOp(workingDocument.path.toString(), false)
- update.removeMapEntry("_revisions", revision)
- for (String property : propertiesModifiedByRevision.get(revision)) {
- update.removeMapEntry(property, revision)
- }
- io.out.println("Executing UpdateOp: " + update)
- try {
- documentStore.findAndUpdate(NODES, update)
- } catch (DocumentStoreException ex) {
- io.out.println("Operation failed: " + ex)
- }*/
-
+ try {
count++;
- if (count >= numberToCleanup) {
- break;
- }
+ documentStore.findAndUpdate(NODES, op);
+ } catch (DocumentStoreException ex) {
+ System.out.println("Operation failed: " + ex);
}
System.out.println("-- Executed " + count + " operations --");
}
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentRevisionCleanupHelper.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentRevisionCleanupHelper.java
deleted file mode 100644
index 0613444efe..0000000000
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentRevisionCleanupHelper.java
+++ /dev/null
@@ -1,287 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.jackrabbit.oak.plugins.document;
-
-import org.apache.jackrabbit.oak.plugins.document.util.Utils;
-
-import java.time.Instant;
-import java.time.temporal.ChronoUnit;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
-import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.isDeletedEntry;
-
-public class DocumentRevisionCleanupHelper {
-
- private final DocumentStore documentStore;
- private final DocumentNodeStore documentNodeStore;
- private final NodeDocument rootDoc;
- private final NodeDocument workingDocument;
-
- private SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingPropertyByCluster;
- private SortedMap<String, TreeSet<Revision>> revisionsModifyingProperty;
- private SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision;
- private SortedMap<Integer, TreeSet<Revision>> blockedRevisionsToKeep;
- private SortedMap<Integer, TreeSet<Revision>> candidateRevisionsToClean;
-
- /**
- * Constructor for DocumentRevisionCleanupHelper.
- * @param documentStore The DocumentStore instance. Must be writable to perform cleanup.
- * @param documentNodeStore The DocumentNodeStore instance.
- * @param path The path of the document to clean up.
- */
- public DocumentRevisionCleanupHelper(DocumentStore documentStore, DocumentNodeStore documentNodeStore, String path) {
- this.candidateRevisionsToClean = new TreeMap<>();
- this.blockedRevisionsToKeep = new TreeMap<>();
- this.revisionsModifyingPropertyByCluster = new TreeMap<>();
- this.revisionsModifyingProperty = new TreeMap<>();
- this.propertiesModifiedByRevision = new TreeMap<>(StableRevisionComparator.INSTANCE);
-
- this.rootDoc = documentStore.find(NODES, Utils.getIdFromPath("/"));
- String id = Utils.getIdFromPath(path);
- this.workingDocument = documentStore.find(NODES, id);
- this.documentStore = documentStore;
- this.documentNodeStore = documentNodeStore;
- }
-
- /**
- * Performs the full revision cleanup process for the given document for a clusterId.
- */
- public void initializeCleanupProcess() {
- classifyAndMapRevisionsAndProperties();
- markRevisionsNewerThanThresholdToPreserve(24, ChronoUnit.HOURS);
- markLastRevisionForEachProperty();
- markCheckpointRevisionsToPreserve();
- removeCandidatesInList(blockedRevisionsToKeep);
- }
-
- public int executeCleanupProcess(int numberToCleanup, int clusterToCleanup) {
- return -99;
- }
-
- /**
- * Step 1:
- * This method processes the revisions of the working document, classifying them into two categories:
- * candidate revisions that can be cleaned up and used revisions that should be kept. It also creates maps to
- * track the relationships between revisions and properties modified by them.
- */
- protected void classifyAndMapRevisionsAndProperties() {
- candidateRevisionsToClean = new TreeMap<>();
- blockedRevisionsToKeep = new TreeMap<>();
- revisionsModifyingPropertyByCluster = new TreeMap<>();
- revisionsModifyingProperty = new TreeMap<>();
- propertiesModifiedByRevision = new TreeMap<>(StableRevisionComparator.INSTANCE);
-
- // The first entry in "_deleted" has to be kept, as is when the document was created
- NavigableMap<Revision, String> deletedRevisions = ((NavigableMap<Revision, String>) workingDocument.get("_deleted"));
- if (deletedRevisions != null && !deletedRevisions.isEmpty()) {
- // TODO: This is just a check to ensure the order is the expected
- assert(deletedRevisions.descendingMap().lastKey().getTimestamp() <= deletedRevisions.descendingMap().firstKey().getTimestamp());
-
- Revision createdRevision = deletedRevisions.descendingMap().lastKey();
- addBlockedRevisionToKeep(createdRevision);
- }
-
- SortedMap<Revision, String> documentRevisions = getAllRevisions();
- for (Map.Entry<Revision, String> revisionEntry : documentRevisions.entrySet()) {
- Revision revision = revisionEntry.getKey();
- String revisionValue = revisionEntry.getValue();
-
- // Only check committed revisions (ignore branch commits starting with "c-")
- if ("c".equals(revisionValue)) {
- // Candidate to clean up
- addCandidateRevisionToClean(revision);
- // Store properties usage
- mapPropertiesModifiedByThisRevision(revision);
- }
- }
- }
-
- /**
- * Step 2:
- * This method filters out revisions that are newer than a specified time threshold (specified by amount and unit)
- * from the candidate revisions to be cleaned up, marking them as used revisions that should be kept.
- * @param amount the amount of time
- * @param unit the unit of time
- */
- protected void markRevisionsNewerThanThresholdToPreserve(long amount, ChronoUnit unit) {
- // TODO: Should we add a buffer here? Maybe 1 minute or even 1 hour?
- long thresholdToPreserve = Instant.now().minus(amount, unit).toEpochMilli();
- for (TreeSet<Revision> revisionSet : candidateRevisionsToClean.values()) {
- for (Revision revision : revisionSet) {
- if (revision.getTimestamp() > thresholdToPreserve) {
- addBlockedRevisionToKeep(revision);
- }
- }
- }
- }
-
- /**
- * Step 3:
- * This method processes a set of revisions that modified certain properties and keeps the last revision that
- * modified each property. This means, the current status of the node will be preserved, for each clusterId.
- */
- protected void markLastRevisionForEachProperty() {
- for (SortedMap<Integer, TreeSet<Revision>> revisionsByCluster : revisionsModifyingPropertyByCluster.values()) {
- for (TreeSet<Revision> revisions : revisionsByCluster.values()) {
- Revision lastRevision = revisions.last();
- addBlockedRevisionToKeep(lastRevision);
- }
- }
- }
-
- /**
- * Step 4:
- * Process a set of revisions that modified certain properties and determine which revisions should be
- * kept based on the checkpoints.
- */
- protected void markCheckpointRevisionsToPreserve() {
- SortedMap<Revision, Checkpoints.Info> checkpoints = documentNodeStore.getCheckpoints().getCheckpoints();
- checkpoints.forEach((revision, info) -> {
- // For each checkpoint, keep the last revision that modified a property prior to checkpoint
- revisionsModifyingProperty.forEach((propertyName, revisionsSet) -> {
- // Traverse the revisionVector of the checkpoint and find the last revision that modified the property
- info.getCheckpoint().forEach(revisionToFind -> {
- // If the exact revision exists, keep it. If not, find the previous one that modified that property
- if (revisionsSet.contains(revisionToFind)) {
- addBlockedRevisionToKeep(revisionToFind);
- } else {
- Revision previousRevision = revisionsSet.descendingSet().ceiling(revisionToFind);
- if (previousRevision != null) {
- addBlockedRevisionToKeep(previousRevision);
- }
- }
- });
- });
- });
- }
-
- /**
- * Step 5:
- * Removes for each clusterId the revisions in the Map, that were blocked in the methods above.
- */
- protected void removeCandidatesInList(SortedMap<Integer, TreeSet<Revision>> revisions) {
- revisions.forEach((key, value) -> {
- if (candidateRevisionsToClean.containsKey(key)) {
- candidateRevisionsToClean.get(key).removeAll(value);
- }
- });
- }
-
- /**
- * This method processes a given revision and identify the properties modified by it within the working document.
- * It maintains two data structures, propertiesModifiedByRevision and revisionsModifyingProperty, to store
- * the properties and their associated revisions.
- * @param revision
- */
- private void mapPropertiesModifiedByThisRevision(Revision revision) {
- for (Map.Entry<String, Object> propertyEntry : workingDocument.entrySet()) {
- if (Utils.isPropertyName(propertyEntry.getKey()) || isDeletedEntry(propertyEntry.getKey())) {
- Map<Revision, String> valueMap = (Map) propertyEntry.getValue();
- if (valueMap.containsKey(revision)) {
- propertiesModifiedByRevision.computeIfAbsent(revision, key ->
- new TreeSet<>()).add(propertyEntry.getKey()
- );
-
- revisionsModifyingPropertyByCluster.computeIfAbsent(propertyEntry.getKey(), key ->
- new TreeMap<>()
- ).computeIfAbsent(revision.getClusterId(), key ->
- new TreeSet<>(StableRevisionComparator.INSTANCE)
- ).add(revision);
-
- revisionsModifyingProperty.computeIfAbsent(propertyEntry.getKey(), key ->
- new TreeSet<>(StableRevisionComparator.INSTANCE)
- ).add(revision);
- }
- }
- }
- }
-
- /**
- * Adds a revision to the list of candidates to delete.
- * @param revision
- */
- private void addCandidateRevisionToClean(Revision revision) {
- candidateRevisionsToClean.computeIfAbsent(revision.getClusterId(), key ->
- new TreeSet<>(StableRevisionComparator.INSTANCE)
- ).add(revision);
- }
-
- /**
- * Adds a revision to the list of revisions to keep.
- * @param revision
- */
- private void addBlockedRevisionToKeep(Revision revision) {
- blockedRevisionsToKeep.computeIfAbsent(revision.getClusterId(), key ->
- new TreeSet<>(StableRevisionComparator.INSTANCE)
- ).add(revision);
- }
-
- /**
- * Returns the LastRev map from the root document.
- * @return
- */
- public Map<Integer, Revision> getLastRev() {
- return rootDoc.getLastRev();
- }
-
- /**
- * Returns the SweepRev map from the root document.
- * @return
- */
- public Map<Integer, Revision> getSweepRev() {
- Map<Integer, Revision> map = new HashMap<>();
- Map<Revision, String> valueMap = (SortedMap<Revision, String>) rootDoc.get("_sweepRev");
- for (Map.Entry<Revision, String> e : valueMap.entrySet()) {
- int clusterId = e.getKey().getClusterId();
- Revision rev = Revision.fromString(e.getValue());
- map.put(clusterId, rev);
- }
- return map;
- }
-
- public NavigableMap<Revision, String> getAllRevisions() {
- return (NavigableMap<Revision, String>) workingDocument.get("_revisions");
- }
-
- public SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> getRevisionsModifyingPropertyByCluster() {
- return revisionsModifyingPropertyByCluster;
- }
-
- public SortedMap<String, TreeSet<Revision>> getRevisionsModifyingProperty() {
- return revisionsModifyingProperty;
- }
-
- public SortedMap<Revision, TreeSet<String>> getPropertiesModifiedByRevision() {
- return propertiesModifiedByRevision;
- }
-
- public SortedMap<Integer, TreeSet<Revision>> getBlockedRevisionsToKeep() {
- return blockedRevisionsToKeep;
- }
-
- public SortedMap<Integer, TreeSet<Revision>> getCandidateRevisionsToClean() {
- return candidateRevisionsToClean;
- }
-}
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleaner.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleaner.java
index ed28c957ed..b55db4afb6 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleaner.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleaner.java
@@ -22,150 +22,80 @@ import org.apache.jackrabbit.oak.plugins.document.util.Utils;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
-import java.util.HashMap;
import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.TreeSet;
-import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
-
public class NodeDocumentRevisionCleaner {
private final DocumentStore documentStore;
private final DocumentNodeStore documentNodeStore;
private final NodeDocument workingDocument;
-
- protected RevisionClassifierUtility revisionClassifier;
- protected RevisionCleanerUtility revisionCleaner;
+ private final RevisionPropertiesClassifier revisionClassifier;
+ private final RevisionCleanerUtility revisionCleaner;
/**
- * Constructor for DocumentRevisionCleanupHelper.
+ * Constructor for NodeDocumentRevisionCleaner.
* @param documentStore The DocumentStore instance. Must be writable to perform cleanup.
* @param documentNodeStore The DocumentNodeStore instance.
- * @param path The path of the document to clean up.
+ * @param workingDocument The document to clean up.
*/
- public NodeDocumentRevisionCleaner(DocumentStore documentStore, DocumentNodeStore documentNodeStore, String path) {
- String id = Utils.getIdFromPath(path);
- this.workingDocument = documentStore.find(NODES, id);
+ public NodeDocumentRevisionCleaner(DocumentStore documentStore, DocumentNodeStore documentNodeStore,
+ NodeDocument workingDocument) {
+ this.workingDocument = workingDocument;
this.documentStore = documentStore;
this.documentNodeStore = documentNodeStore;
- revisionClassifier = new RevisionClassifierUtility(workingDocument);
+ revisionClassifier = new RevisionPropertiesClassifier(workingDocument);
revisionCleaner = new RevisionCleanerUtility(revisionClassifier);
}
/**
- * Performs the full revision cleanup process for the given document for a clusterId.
+ * Collects cleanable old revisions for the given document.
*/
- public void initializeCleanupProcess() {
- revisionClassifier.identifyRevisionsToClean();
- revisionCleaner.markRevisionsNewerThanThresholdToPreserve(24, ChronoUnit.HOURS);
- revisionCleaner.markLastRevisionForEachProperty();
- revisionCleaner.markCheckpointRevisionsToPreserve();
+ public void collectOldRevisions(UpdateOp op) {
+ revisionClassifier.classifyRevisionsAndProperties();
+ revisionCleaner.preserveRevisionsNewerThanThreshold(24, ChronoUnit.HOURS);
+ revisionCleaner.preserveLastRevisionForEachProperty();
+ revisionCleaner.preserveRevisionsReferencedByCheckpoints();
revisionCleaner.removeCandidatesInList();
- }
-
- public int executeCleanupProcess(int numberToCleanup, int clusterToCleanup) {
- return -99;
- }
- protected void classifyAndMapRevisionsAndProperties() {
- //identifyRevisionsToClean();
- //BclassifyAndMapRevisionsAndProperties();
-
- revisionClassifier.identifyRevisionsToClean();
- //newClassifyAndMapRevisionsAndProperties();
- }
-
- /*protected void newClassifyAndMapRevisionsAndProperties() {
- candidateRevisionsToClean = new TreeMap<>();
- blockedRevisionsToKeep = new TreeMap<>();
- revisionsModifyingPropertyByCluster = new TreeMap<>();
- revisionsModifyingProperty = new TreeMap<>();
- propertiesModifiedByRevision = new TreeMap<>(StableRevisionComparator.INSTANCE);
-
- // The first entry in "_deleted" has to be kept, as is when the document was created
- //NavigableMap<Revision, String> deletedRevisions = ((NavigableMap<Revision, String>) workingDocument.get("_deleted"));
- SortedMap<Revision, String> deletedRevisions = workingDocument.getLocalDeleted();
- if (deletedRevisions != null && !deletedRevisions.isEmpty()) {
- Revision createdRevision = deletedRevisions.firstKey();
- addBlockedRevisionToKeep(createdRevision);
- }
-
- SortedMap<Revision, String> documentRevisions = getAllRevisions();
- for (Map.Entry<Revision, String> revisionEntry : documentRevisions.entrySet()) {
- Revision revision = revisionEntry.getKey();
- String revisionValue = revisionEntry.getValue();
-
- // Only check committed revisions (ignore branch commits starting with "c-")
- if ("c".equals(revisionValue)) {
- // Candidate to clean up
- addCandidateRevisionToClean(revision);
- // Store properties usage
- mapPropertiesModifiedByThisRevision(revision);
- }
- }
- }*/
-
- /**
- * Step 1:
- * This method processes the revisions of the working document, classifying them into two categories:
- * candidate revisions that can be cleaned up and used revisions that should be kept. It also creates maps to
- * track the relationships between revisions and properties modified by them.
- */
- /*protected void BclassifyAndMapRevisionsAndProperties() {
- candidateRevisionsToClean = new TreeMap<>();
- blockedRevisionsToKeep = new TreeMap<>();
- revisionsModifyingPropertyByCluster = new TreeMap<>();
- revisionsModifyingProperty = new TreeMap<>();
- propertiesModifiedByRevision = new TreeMap<>(StableRevisionComparator.INSTANCE);
-
- // The first entry in "_deleted" has to be kept, as is when the document was created
- //NavigableMap<Revision, String> deletedRevisions = ((NavigableMap<Revision, String>) workingDocument.get("_deleted"));
- NavigableMap<Revision, String> deletedRevisions = ((NavigableMap<Revision, String>) workingDocument.getLocalDeleted());
- if (deletedRevisions != null && !deletedRevisions.isEmpty()) {
- // TODO: This is just a check to ensure the order is the expected
- assert(deletedRevisions.descendingMap().lastKey().getTimestamp() <= deletedRevisions.descendingMap().firstKey().getTimestamp());
-
- Revision createdRevision = deletedRevisions.descendingMap().lastKey();
- addBlockedRevisionToKeep(createdRevision);
- }
+ /*for (Map.Entry<Integer, TreeSet<Revision>> entry : revisionCleaner.candidateRevisionsToClean.entrySet()) {
+ for (Revision revision : entry.getValue()) {
+ System.out.println("Removing revision " + revision);
+ TreeSet<String> properties = revisionClassifier.propertiesModifiedByRevision.get(revision);
+ System.out.println("Properties modified by this revision: " + properties);
+ if (properties != null) {
+ for (String property : properties) {
+ op.removeMapEntry(property, revision);
+ }
+ }
- SortedMap<Revision, String> documentRevisions = getAllRevisions();
- for (Map.Entry<Revision, String> revisionEntry : documentRevisions.entrySet()) {
- Revision revision = revisionEntry.getKey();
- String revisionValue = revisionEntry.getValue();
-
- // Only check committed revisions (ignore branch commits starting with "c-")
- if ("c".equals(revisionValue)) {
- // Candidate to clean up
- addCandidateRevisionToClean(revision);
- // Store properties usage
- mapPropertiesModifiedByThisRevision(revision);
+ op.removeMapEntry("_revisions", revision);
}
- }
- }*/
+ }*/
+ }
- protected class RevisionClassifierUtility {
+ protected class RevisionPropertiesClassifier {
private final NodeDocument workingDocument;
- private final SortedMap<Revision, String> documentRevisions;
private SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingPropertyByCluster;
private SortedMap<String, TreeSet<Revision>> revisionsModifyingProperty;
private SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision;
- RevisionClassifierUtility(NodeDocument workingDocument) {
+ private RevisionPropertiesClassifier(NodeDocument workingDocument) {
this.workingDocument = workingDocument;
- this.documentRevisions = workingDocument.getLocalRevisions();
this.revisionsModifyingPropertyByCluster = new TreeMap<>();
this.revisionsModifyingProperty = new TreeMap<>();
this.propertiesModifiedByRevision = new TreeMap<>(StableRevisionComparator.INSTANCE);
}
- public void identifyRevisionsToClean() {
+ /**
+ * This method processes the revisions of the working document, creating maps to
+ * track the relationships between revisions and modified properties.
+ */
+ protected void classifyRevisionsAndProperties() {
SortedMap<Revision, String> deletedRevisions = workingDocument.getLocalDeleted();
// Always keep the first "_deleted" entry, as is when the document was created
if (!deletedRevisions.isEmpty()) {
@@ -178,17 +108,17 @@ public class NodeDocumentRevisionCleaner {
Revision revision = revisionEntry.getKey();
String revisionValue = revisionEntry.getValue();
- // Only check committed revisions (ignore branch commits starting with "c-")
+ // Only check committed revisions
if (Utils.isCommitted(revisionValue)) {
// Candidate to clean up
revisionCleaner.addCandidateRevisionToClean(revision);
// Store properties usage
- mapPropertiesModifiedByThisRevision(revision);
+ classifyPropertiesModifiedByRevision(revision);
}
}
}
- private void mapPropertiesModifiedByThisRevision(Revision revision) {
+ private void classifyPropertiesModifiedByRevision(Revision revision) {
for (Map.Entry<String, Object> propertyEntry : workingDocument.entrySet()) {
if (Utils.isPropertyName(propertyEntry.getKey()) || NodeDocument.isDeletedEntry(propertyEntry.getKey())) {
Map<Revision, String> valueMap = (Map) propertyEntry.getValue();
@@ -214,17 +144,17 @@ public class NodeDocumentRevisionCleaner {
protected class RevisionCleanerUtility {
- private SortedMap<Integer, TreeSet<Revision>> blockedRevisionsToKeep;
- private SortedMap<Integer, TreeSet<Revision>> candidateRevisionsToClean;
- private final RevisionClassifierUtility revisionClassifier;
+ private final SortedMap<Integer, TreeSet<Revision>> blockedRevisionsToKeep;
+ private final SortedMap<Integer, TreeSet<Revision>> candidateRevisionsToClean;
+ private final RevisionPropertiesClassifier revisionClassifier;
- protected RevisionCleanerUtility(RevisionClassifierUtility revisionClassifier) {
+ private RevisionCleanerUtility(RevisionPropertiesClassifier revisionClassifier) {
this.revisionClassifier = revisionClassifier;
this.candidateRevisionsToClean = new TreeMap<>();
this.blockedRevisionsToKeep = new TreeMap<>();
}
- protected void markLastRevisionForEachProperty() {
+ protected void preserveLastRevisionForEachProperty() {
for (SortedMap<Integer, TreeSet<Revision>> revisionsByCluster : revisionClassifier.revisionsModifyingPropertyByCluster.values()) {
for (TreeSet<Revision> revisions : revisionsByCluster.values()) {
Revision lastRevision = revisions.last();
@@ -233,7 +163,7 @@ public class NodeDocumentRevisionCleaner {
}
}
- protected void markRevisionsNewerThanThresholdToPreserve(long amount, ChronoUnit unit) {
+ protected void preserveRevisionsNewerThanThreshold(long amount, ChronoUnit unit) {
long thresholdToPreserve = Instant.now().minus(amount, unit).toEpochMilli();
for (TreeSet<Revision> revisionSet : candidateRevisionsToClean.values()) {
for (Revision revision : revisionSet) {
@@ -244,7 +174,7 @@ public class NodeDocumentRevisionCleaner {
}
}
- protected void markCheckpointRevisionsToPreserve() {
+ protected void preserveRevisionsReferencedByCheckpoints() {
SortedMap<Revision, Checkpoints.Info> checkpoints = documentNodeStore.getCheckpoints().getCheckpoints();
checkpoints.forEach((revision, info) -> {
// For each checkpoint, keep the last revision that modified a property prior to checkpoint
@@ -294,19 +224,19 @@ public class NodeDocumentRevisionCleaner {
}
}
- public NavigableMap<Revision, String> getAllRevisions() {
- return (NavigableMap<Revision, String>) workingDocument.getLocalRevisions();
+ public SortedMap<Revision, String> getAllRevisions() {
+ return workingDocument.getLocalRevisions();
}
- public SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> getRevisionsModifyingPropertyByCluster() {
+ protected SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> getRevisionsModifyingPropertyByCluster() {
return revisionClassifier.revisionsModifyingPropertyByCluster;
}
- public SortedMap<String, TreeSet<Revision>> getRevisionsModifyingProperty() {
+ protected SortedMap<String, TreeSet<Revision>> getRevisionsModifyingProperty() {
return revisionClassifier.revisionsModifyingProperty;
}
- public SortedMap<Revision, TreeSet<String>> getPropertiesModifiedByRevision() {
+ protected SortedMap<Revision, TreeSet<String>> getPropertiesModifiedByRevision() {
return revisionClassifier.propertiesModifiedByRevision;
}
@@ -317,4 +247,24 @@ public class NodeDocumentRevisionCleaner {
public SortedMap<Integer, TreeSet<Revision>> getCandidateRevisionsToClean() {
return revisionCleaner.candidateRevisionsToClean;
}
+
+ protected void classifyRevisionsAndProperties() {
+ revisionClassifier.classifyRevisionsAndProperties();
+ }
+
+ protected void markLastRevisionForEachProperty() {
+ revisionCleaner.preserveLastRevisionForEachProperty();
+ }
+
+ protected void markRevisionsNewerThanThresholdToPreserve(long amount, ChronoUnit unit) {
+ revisionCleaner.preserveRevisionsNewerThanThreshold(amount, unit);
+ }
+
+ protected void markCheckpointRevisionsToPreserve() {
+ revisionCleaner.preserveRevisionsReferencedByCheckpoints();
+ }
+
+ protected void removeCandidatesInList() {
+ revisionCleaner.removeCandidatesInList();
+ }
}
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentRevisionCleanupHelperTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentRevisionCleanupHelperTest.java
deleted file mode 100644
index e1f6608c9d..0000000000
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentRevisionCleanupHelperTest.java
+++ /dev/null
@@ -1,620 +0,0 @@
-package org.apache.jackrabbit.oak.plugins.document;
-
-import org.apache.jackrabbit.guava.common.collect.Maps;
-import org.apache.jackrabbit.oak.api.CommitFailedException;
-import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore;
-import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.MockitoAnnotations;
-
-import java.io.IOException;
-import java.lang.reflect.Field;
-import java.time.Instant;
-import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
-import static org.apache.jackrabbit.oak.plugins.document.TestUtils.merge;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-public class DocumentRevisionCleanupHelperTest {
-
- @Mock
- DocumentStore documentStore;
-
- @Mock
- DocumentNodeStore documentNodeStore;
-
- @Mock
- Checkpoints checkpoints;
-
- @Mock
- NodeDocument workingDocument;
-
- //NodeDocumentRevisionCleaner NodeDocumentRevisionCleaner;
- DocumentRevisionCleanupHelper documentRevisionCleanupHelper;
-
- @Before
- public void setUp() {
- MockitoAnnotations.initMocks(this);
- workingDocument = Mockito.mock(NodeDocument.class);
- //workingDocument = new NodeDocument(documentStore);
-
- Mockito.when(documentStore.find(Mockito.eq(NODES), Mockito.anyString())).thenReturn(workingDocument);
- //NodeDocumentRevisionCleaner = new NodeDocumentRevisionCleaner(documentStore, documentNodeStore, "/");
- documentRevisionCleanupHelper = new DocumentRevisionCleanupHelper(documentStore, documentNodeStore, "/");
- }
-
- @Test
- public void testMarkCheckpointRevisionsToPreserveOnePropertyOneCluster() throws IOException {
- Revision revisionA = Revision.fromString("r100000000-0-1");
- Revision revisionB = Revision.fromString("r105000000-0-1");
- Revision revisionC = Revision.fromString("r110000000-0-1");
- Revision revisionD = Revision.fromString("r115000000-0-1");
- Revision revisionE = Revision.fromString("r120000000-0-1");
- Revision revisionF = Revision.fromString("r125000000-0-1");
-
- Revision checkpoint1 = Revision.fromString("r109000000-0-1");
- Revision checkpoint2 = Revision.fromString("r119000000-0-1");
-
- String jsonProperties = "{" +
- "'prop1': {'" + revisionA + "': 'value1', '" + revisionB + "': 'value2', '" + revisionC + "': 'value3', '" + revisionD + "': 'value4', '" + revisionE + "': 'value5', '" + revisionF + "': 'value6'}, " +
- "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c', '" + revisionC + "': 'c', '" + revisionD + "': 'c', '" + revisionE + "': 'c', '" + revisionF + "': 'c'}" +
- "}";
- String jsonCheckpoints = "{" +
- "'" + checkpoint1 + "': {'expires':'200000000','rv':'r109000000-0-1'}," +
- "'" + checkpoint2 + "': {'expires':'200000000','rv':'r119000000-0-1'}" +
- "}";
-
-
- prepareDocumentMock(jsonProperties);
- prepareCheckpointsMock(jsonCheckpoints);
-
- documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties();
- documentRevisionCleanupHelper.markCheckpointRevisionsToPreserve();
- documentRevisionCleanupHelper.removeCandidatesInList(documentRevisionCleanupHelper.getBlockedRevisionsToKeep());
-
- // The revisions blocked should be:
- // - r105000000-0-1 (blocked by checkpoint r109000000-0-1)
- // - r115000000-0-1 (blocked by checkpoint r119000000-0-1)
- assertEquals(Set.of(revisionB, revisionD), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1));
- assertEquals(Set.of(revisionA, revisionC, revisionE, revisionF), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(1));
- }
-
- @Test
- public void testRecentRevisionsArePreserved() throws IOException {
- StringBuilder jsonPropBuilder = new StringBuilder("'prop1': {");
- StringBuilder jsonRevisionsBuilder = new StringBuilder("'_revisions': {");
-
- Instant currentTime = Instant.now();
- List<Revision> revisions = new ArrayList<>();
- for (int i = 29; i >= 0; i--) {
- long timestamp = currentTime.minus(i, ChronoUnit.HOURS).toEpochMilli();
- Revision revision = new Revision(timestamp, 0, 1);
- revisions.add(revision);
-
- jsonPropBuilder.append("'").append(revision).append("': 'value").append(i).append("', ");
- jsonRevisionsBuilder.append("'").append(revision).append("': 'c', ");
- }
-
- // 23 hours and 59 minutes ago -> Should be preserved
- long timestamp = currentTime.minus(23, ChronoUnit.HOURS).minus(59, ChronoUnit.MINUTES).toEpochMilli();
- Revision revision = new Revision(timestamp, 0, 1);
- revisions.add(revision); // revisions[30]
- jsonPropBuilder.append("'").append(revision).append("': 'value").append(30).append("', ");
- jsonRevisionsBuilder.append("'").append(revision).append("': 'c', ");
-
- // 24 hours and 1 minute ago -> Should be candidate to clean
- timestamp = currentTime.minus(24, ChronoUnit.HOURS).minus(1, ChronoUnit.MINUTES).toEpochMilli();
- revision = new Revision(timestamp, 0, 1);
- revisions.add(revision); // revisions[31]
- jsonPropBuilder.append("'").append(revision).append("': 'value").append(30).append("'");
- jsonRevisionsBuilder.append("'").append(revision).append("': 'c'");
-
- jsonPropBuilder.append("}");
- jsonRevisionsBuilder.append("}");
- StringBuilder jsonBuilder = new StringBuilder("{");
- jsonBuilder.append(jsonPropBuilder).append(", ").append(jsonRevisionsBuilder).append("}");
-
- // revisions.forEach(rev -> System.out.println(rev.toReadableString()));
-
- prepareDocumentMock(jsonBuilder.toString());
-
- documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties();
- documentRevisionCleanupHelper.markRevisionsNewerThanThresholdToPreserve(24, ChronoUnit.HOURS);
- documentRevisionCleanupHelper.removeCandidatesInList(documentRevisionCleanupHelper.getBlockedRevisionsToKeep());
-
- // The candidate revisions should be the 6 oldest ones (current time -29, -28, -27, -26, -25, -24)
- // and the one created 24 hours and 1 minute ago
- Set<Revision> expectedCandidateRevisions = Stream.concat(
- revisions.subList(0, 6).stream(), Stream.of(revisions.get(31))
- ).collect(Collectors.toSet());
-
- // Blocked revisions are all the 24 revisions created in the last 24 hours, and the one created 23 hours and 59 minutes ago
- Set<Revision> expectedBlockedRevisions = revisions.subList(6, 31).stream().collect(Collectors.toSet());
-
- assertEquals(expectedCandidateRevisions, documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(1));
- assertEquals(expectedBlockedRevisions, documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1));
- }
-
- @Test
- public void testInitializeCleanupProcessMultipleClusters() throws IOException {
- List<Revision> revs = new ArrayList<>();
-
- // Some initial revisions (0-2)
- revs.add(Revision.fromString("r100000000-0-1"));
- revs.add(Revision.fromString("r100000001-0-2"));
- revs.add(Revision.fromString("r100000002-0-3"));
-
- // Blocked by first checkpoint r109000000 (3-5)
- revs.add(Revision.fromString("r106000003-0-1"));
- revs.add(Revision.fromString("r107000004-0-2"));
- revs.add(Revision.fromString("r108000005-0-3"));
-
- // Some unblocked revisions in middle (6-10)
- revs.add(Revision.fromString("r110000006-0-3"));
- revs.add(Revision.fromString("r110000006-1-3"));
- revs.add(Revision.fromString("r114000007-0-2"));
- revs.add(Revision.fromString("r115000008-0-1"));
- revs.add(Revision.fromString("r118000009-0-1"));
-
- // Blocked by second checkpoint r118000000 (11-13)
- revs.add(Revision.fromString("r118000009-1-1"));
- revs.add(Revision.fromString("r118000010-0-2"));
- revs.add(Revision.fromString("r118000011-0-3"));
-
- // Some more unblocked revisions (14-19)
- revs.add(Revision.fromString("r120000012-0-1"));
- revs.add(Revision.fromString("r122000013-0-2"));
- revs.add(Revision.fromString("r122000013-1-2"));
- revs.add(Revision.fromString("r123000014-0-2"));
- revs.add(Revision.fromString("r125000015-0-3"));
- revs.add(Revision.fromString("r130000016-0-1"));
-
- // Last revision (20-22)
- revs.add(Revision.fromString("r130000016-1-1"));
- revs.add(Revision.fromString("r130000017-0-2"));
- revs.add(Revision.fromString("r130000018-0-3"));
-
- // Checkpoint revisions
- Revision checkpoint1 = Revision.fromString("r109000000-0-1");
- Revision checkpoint2 = Revision.fromString("r119000000-0-1");
-
- StringBuilder jsonPropBuilder = new StringBuilder("'prop1': {");
- StringBuilder jsonRevisionsBuilder = new StringBuilder("'_revisions': {");
-
- for (int i = 0; i < revs.size(); i++) {
- jsonPropBuilder.append("'").append(revs.get(i)).append("': 'value").append(i).append("'");
- jsonRevisionsBuilder.append("'").append(revs.get(i)).append("': 'c'");
- if (i < revs.size() - 1) {
- jsonPropBuilder.append(", ");
- jsonRevisionsBuilder.append(", ");
- }
- }
-
- jsonPropBuilder.append("}");
- jsonRevisionsBuilder.append("}");
- StringBuilder jsonBuilder = new StringBuilder("{");
- jsonBuilder.append(jsonPropBuilder).append(", ").append(jsonRevisionsBuilder).append("}");
-
- String jsonCheckpoints = "{" +
- "'" + checkpoint1 + "': {'expires':'200000000','rv':'r106500003-0-1,r107500004-0-2,r108500005-0-3'}," +
- "'" + checkpoint2 + "': {'expires':'200000000','rv':'r118000009-1-1,r118000010-0-2,r118000011-0-3'}" +
- "}";
-
- prepareDocumentMock(jsonBuilder.toString());
- prepareCheckpointsMock(jsonCheckpoints);
-
- documentRevisionCleanupHelper.initializeCleanupProcess();
-
- // The revisions blocked should be:
- // - r106000003-0-3, r118000004-0-2, r108000005-0-3 (referenced by checkpoint 1)
- // - r118000009-1-1, r118000010-0-2, r118000011-0-3 (referenced by checkpoint 2)
- // - r130000016-1-1, r130000017-0-2, r130000018-0-3 (last revision)
- assertEquals(Set.of(revs.get(3), revs.get(11), revs.get(20)), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1));
- assertEquals(Set.of(revs.get(4), revs.get(12), revs.get(21)), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(2));
- assertEquals(Set.of(revs.get(5), revs.get(13), revs.get(22)), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(3));
-
- // Rest of revisions are candidates to clean
- assertEquals(Set.of(revs.get(0), revs.get(9), revs.get(10), revs.get(14), revs.get(19)), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(1));
- assertEquals(Set.of(revs.get(1), revs.get(8), revs.get(15), revs.get(16), revs.get(17)), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(2));
- assertEquals(Set.of(revs.get(2), revs.get(6), revs.get(7), revs.get(18)), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(3));
-
- assertTrue(Collections.disjoint(documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(1)));
- assertTrue(Collections.disjoint(documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(2), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(2)));
- assertTrue(Collections.disjoint(documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(3), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(3)));
- }
-
- @Test
- public void testCheckpointedRevisionFallback() throws IOException {
- List<Revision> revs = new ArrayList<>();
-
- revs.add(Revision.fromString("r100000000-0-1"));
- revs.add(Revision.fromString("r100000001-0-2"));
- revs.add(Revision.fromString("r100000002-0-3"));
-
- revs.add(Revision.fromString("r101100003-0-1"));
- revs.add(Revision.fromString("r101200004-0-2"));
- revs.add(Revision.fromString("r101300005-0-3"));
-
- revs.add(Revision.fromString("r102000006-0-1"));
- revs.add(Revision.fromString("r102000007-0-3"));
-
- revs.add(Revision.fromString("r103000008-0-1"));
- revs.add(Revision.fromString("r103000009-0-2"));
- revs.add(Revision.fromString("r103000010-0-3"));
-
- // Checkpoint
- Revision checkpoint1 = Revision.fromString("r102000000-0-1");
-
- StringBuilder jsonPropBuilder = new StringBuilder("'prop1': {");
- StringBuilder jsonRevisionsBuilder = new StringBuilder("'_revisions': {");
-
- for (int i = 0; i < revs.size(); i++) {
- jsonPropBuilder.append("'").append(revs.get(i)).append("': 'value").append(i).append("'");
- jsonRevisionsBuilder.append("'").append(revs.get(i)).append("': 'c'");
- if (i < revs.size() - 1) {
- jsonPropBuilder.append(", ");
- jsonRevisionsBuilder.append(", ");
- }
- }
-
- jsonPropBuilder.append("}");
- jsonRevisionsBuilder.append("}");
- StringBuilder jsonBuilder = new StringBuilder("{");
- jsonBuilder.append(jsonPropBuilder).append(", ").append(jsonRevisionsBuilder).append("}");
-
- String jsonCheckpoints = "{" +
- "'" + checkpoint1 + "': {'expires':'200000000','rv':'r102000007-0-1,r102000000-0-2,r102000008-0-3'}" +
- "}";
-
- prepareDocumentMock(jsonBuilder.toString());
- prepareCheckpointsMock(jsonCheckpoints);
-
- documentRevisionCleanupHelper.initializeCleanupProcess();
-
- // The revisions blocked should be:
- // - r103000008-0-1, r103000009-0-2, r103000010-0-3 (last revisions)
- // - r102000006-0-1, r101300005-0-3, r102000007-0-3 (referenced by checkpoint for clusters 1, 2 and 3 respectively)
- assertEquals(Set.of(revs.get(6), revs.get(8)), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1));
- assertEquals(Set.of(revs.get(9)), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(2));
- assertEquals(Set.of(revs.get(5), revs.get(7), revs.get(10)), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(3));
-
- // Rest of revisions are candidates to clean
- assertEquals(Set.of(revs.get(0), revs.get(3)), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(1));
- assertEquals(Set.of(revs.get(1), revs.get(4)), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(2));
- assertEquals(Set.of(revs.get(2)), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(3));
- }
-
- @Test
- public void testLastRevisionIsBlocked() throws IOException {
- Revision revisionA = new Revision(111111111L, 0, 1);
- Revision revisionB = new Revision(222222222L, 0, 1);
- Revision revisionC = new Revision(333333333L, 0, 1);
-
- String jsonProperties = "{" +
- "'prop1': {'" + revisionA + "': 'value1', '" + revisionB + "': 'value2', '" + revisionC + "': 'value3'}, " +
- "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c'}" +
- "}";
- prepareDocumentMock(jsonProperties);
-
- documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties();
- documentRevisionCleanupHelper.markLastRevisionForEachProperty();
-
- assertFalse(documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1).contains(revisionA));
- assertFalse(documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1).contains(revisionB));
- assertEquals(Set.of(revisionC), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1));
- assertEquals(Set.of(revisionA, revisionB, revisionC), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(1));
- }
-
- @Test
- public void testFirstDeletedRevisionIsBlocked() throws Exception {
- Revision revisionA = new Revision(111111111L, 0, 1);
- Revision revisionB = new Revision(222222222L, 0, 1);
- Revision revisionC = new Revision(333333333L, 0, 1);
-
- String jsonProperties = "{" +
- "'_deleted': {'" + revisionA + "': 'false', '" + revisionB + "': 'true', '" + revisionC + "': 'false'}," +
- "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c'}" +
- "}";
- prepareDocumentMock(jsonProperties);
-
- documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties();
-
- assertTrue(documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1).contains(revisionA));
- assertEquals(Set.of(revisionA, revisionB, revisionC), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(1));
- }
-
- /*@Test
- public void testFirstDeletedRevisionIsBlocked() throws Exception {
- Revision revisionA = new Revision(111111111L, 0, 1);
- Revision revisionB = new Revision(222222222L, 0, 1);
- Revision revisionC = new Revision(333333333L, 0, 1);
-
- String jsonProperties = "{" +
- "'_deleted': {'" + revisionA + "': 'false', '" + revisionB + "': 'true', '" + revisionC + "': 'false'}," +
- "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c'}" +
- "}";
- //prepareDocumentMock(jsonProperties);
-
- insertJsonDataToNodeDocument(workingDocument, jsonProperties);
-
- NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties();
-
- assertTrue(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA));
- assertEquals(Set.of(revisionA, revisionB, revisionC), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
- }*/
-
- /*private void insertJsonDataToNodeDocument(NodeDocument document, String jsonProperties) throws Exception {
- String json = jsonProperties.replace("'", "\"");
-
- ObjectMapper mapper = new ObjectMapper();
- // TypeReference helps in specifying the complex type (Map<String, Object>)
- Map<String, Object> parsedData = mapper.readValue(json, new TypeReference<Map<String, Object>>() {});
-
- // Since the data field is a Map<String, Object>, we need to ensure the inner maps are also correctly typed
- for (Map.Entry<String, Object> entry : parsedData.entrySet()) {
- if (entry.getValue() instanceof Map) {
- Map<?, ?> innerMap = (Map<?, ?>) entry.getValue();
- Map<String, Object> correctedInnerMap = new TreeMap<>();
- for (Map.Entry<?, ?> innerEntry : innerMap.entrySet()) {
- correctedInnerMap.put(innerEntry.getKey().toString(), innerEntry.getValue());
- }
- parsedData.put(entry.getKey(), correctedInnerMap);
- }
- }
-
- document.data = parsedData; // Directly setting the protected field
- }*/
-
- /*@Test
- public void testtest() throws CommitFailedException, NoSuchFieldException, IllegalAccessException {
- DocumentStore store = new MemoryDocumentStore();
- DocumentNodeStore ns = new DocumentMK.Builder().setDocumentStore(store).setAsyncDelay(0).getNodeStore();
-
- // add properties
- for (int i = 0; i < 10; i++) {
- NodeBuilder nb = ns.getRoot().builder();
-
- nb.child("x").setProperty("p"+i, i);
- merge(ns, nb);
- }
-
- NodeDocument nodeDocument = store.find(NODES, "1:/x");
- System.out.println(nodeDocument.getPropertyNames());
-
- // Mocked data
- Map<String, Object> data = Maps.newHashMap();
-
-
- NodeDocument document = new NodeDocument(store, System.currentTimeMillis());
- document.data.put("foo", "bar");
- System.out.println(document.getPropertyNames());
-
- document.data = Maps.newHashMap();
-
-
- System.out.println(document.getPropertyNames());
- }*/
-
- @Test
- public void testClassifyAndMapRevisionsAndPropertiesWithDeleted() throws IOException {
- Revision revisionA = new Revision(111111111L, 0, 1);
- Revision revisionB = new Revision(222222222L, 0, 1);
- Revision revisionC = new Revision(333333333L, 0, 1);
-
- String jsonProperties = "{" +
- "'prop1': {'" + revisionA + "': 'value1', '" + revisionB + "': 'value2', '" + revisionC + "': 'value3'}, " +
- "'prop2': {'" + revisionB + "': 'value4'}, " +
- "'_deleted': {'" + revisionA + "': 'false'}," +
- "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c'}" +
- "}";
- prepareDocumentMock(jsonProperties);
-
- documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties();
-
- SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = documentRevisionCleanupHelper.getPropertiesModifiedByRevision();
- assertEquals(Set.of("prop1", "_deleted"), propertiesModifiedByRevision.get(revisionA));
- assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB));
- assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC));
-
- SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = documentRevisionCleanupHelper.getRevisionsModifyingPropertyByCluster();
- assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("prop1").get(1));
- assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop2").get(1));
- assertEquals(Set.of(revisionA), revisionsModifyingProperty.get("_deleted").get(1));
- }
-
- @Test
- public void testClassifyAndMapRevisionsMultipleDeleted() throws IOException {
- Revision revisionA = new Revision(111111111L, 0, 1);
- Revision revisionB = new Revision(222222222L, 0, 1);
- Revision revisionC = new Revision(333333333L, 0, 1);
-
- String jsonProperties = "{" +
- "'_deleted': {'" + revisionA + "': 'false', '" + revisionB + "': 'true', '" + revisionC + "': 'false'}," +
- "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c'}" +
- "}";
- prepareDocumentMock(jsonProperties);
-
- documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties();
-
- SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = documentRevisionCleanupHelper.getPropertiesModifiedByRevision();
- assertEquals(Set.of("_deleted"), propertiesModifiedByRevision.get(revisionA));
- assertEquals(Set.of("_deleted"), propertiesModifiedByRevision.get(revisionB));
- assertEquals(Set.of("_deleted"), propertiesModifiedByRevision.get(revisionC));
-
- SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = documentRevisionCleanupHelper.getRevisionsModifyingPropertyByCluster();
- assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("_deleted").get(1));
- }
-
- @Test
- public void testClassifyAndMapRevisionsAndPropertiesWithoutDeleted() throws IOException {
- Revision revisionA = new Revision(111111111L, 0, 1);
- Revision revisionB = new Revision(222222222L, 0, 1);
- Revision revisionC = new Revision(333333333L, 0, 1);
-
- String jsonProperties = "{" +
- "'prop1': {'" + revisionA + "': 'value1', '" + revisionB + "': 'value2', '" + revisionC + "': 'value3'}, " +
- "'prop2': {'" + revisionB + "': 'value4'}, " +
- "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c'}" +
- "}";
- prepareDocumentMock(jsonProperties);
-
- documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties();
-
- SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = documentRevisionCleanupHelper.getPropertiesModifiedByRevision();
- assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionA));
- assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB));
- assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC));
-
- SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = documentRevisionCleanupHelper.getRevisionsModifyingPropertyByCluster();
- assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("prop1").get(1));
- assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop2").get(1));
- }
-
- @Test
- public void testClassifyAndMapRevisionsAndPropertiesNotCommitted() throws IOException {
- Revision revisionA = new Revision(111111111L, 0, 1);
- Revision revisionB = new Revision(222222222L, 0, 1);
- Revision revisionC = new Revision(333333333L, 0, 1);
- Revision revisionD = new Revision(444444444L, 0, 1);
-
- String jsonProperties = "{" +
- "'prop1': {'" + revisionA + "': 'value1', '" + revisionB + "': 'value2', '" + revisionC + "': 'value3', '" + revisionD + "': 'value5'}, " +
- "'prop2': {'" + revisionB + "': 'value4', '" + revisionD + "': 'value5'}, " +
- "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c', '" + revisionD + "': 'nc'}" +
- "}";
- prepareDocumentMock(jsonProperties);
-
- documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties();
-
- // Modifications done in revisionD should be ignored, as it is not committed
- SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = documentRevisionCleanupHelper.getPropertiesModifiedByRevision();
- assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionA));
- assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB));
- assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC));
- assertNull(propertiesModifiedByRevision.get(revisionD));
-
- SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = documentRevisionCleanupHelper.getRevisionsModifyingPropertyByCluster();
- assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("prop1").get(1));
- assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop2").get(1));
- }
-
- @Test
- public void testClassifyAndMapRevisionsAndPropertiesDifferentClusters() throws IOException {
- Revision revisionA = new Revision(111111111L, 0, 1);
- Revision revisionB = new Revision(222222222L, 0, 2);
- Revision revisionC = new Revision(333333333L, 0, 3);
- Revision revisionD = new Revision(444444444L, 0, 1);
-
- String jsonProperties = "{" +
- "'prop1': {'" + revisionA + "': 'value1', '" + revisionB + "': 'value2', '" + revisionC + "': 'value3', '" + revisionD + "': 'value5'}, " +
- "'prop2': {'" + revisionB + "': 'value4', '" + revisionD + "': 'value5'}, " +
- "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c', '" + revisionD + "': 'c'}" +
- "}";
- prepareDocumentMock(jsonProperties);
-
- documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties();
-
- SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = documentRevisionCleanupHelper.getPropertiesModifiedByRevision();
- assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionA));
- assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB));
- assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC));
- assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionD));
-
- SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = documentRevisionCleanupHelper.getRevisionsModifyingPropertyByCluster();
- assertEquals(Set.of(revisionA, revisionD), revisionsModifyingProperty.get("prop1").get(1));
- assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop1").get(2));
- assertEquals(Set.of(revisionC), revisionsModifyingProperty.get("prop1").get(3));
- assertEquals(Set.of(revisionD), revisionsModifyingProperty.get("prop2").get(1));
- assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop2").get(2));
- assertNull(revisionsModifyingProperty.get("prop2").get(3));
- }
-
- private void prepareDocumentMock(String jsonProperties) throws IOException {
- String json = jsonProperties.replace("'", "\"");
-
- ObjectMapper objectMapper = new ObjectMapper();
- Map<String, Map<String, String>> data = objectMapper.readValue(json, new TypeReference<>() {});
-
- SortedMap<String, Object> entries = new TreeMap<>();
- SortedMap<Revision, String> allRevisions = new TreeMap<>(StableRevisionComparator.INSTANCE);
- for (Map.Entry<String, Map<String, String>> entry : data.entrySet()) {
- String property = entry.getKey();
- Map<String, String> revisions = entry.getValue();
-
- SortedMap<Revision, String> sortedRevisions = new TreeMap<>(StableRevisionComparator.INSTANCE);
- for (Map.Entry<String, String> revisionEntry : revisions.entrySet()) {
- String revisionStr = revisionEntry.getKey();
- String value = revisionEntry.getValue();
-
- String[] parts = revisionStr.split("-");
- // The timestamp part of the revision string (first part) is parsed as hexadecimal (radix 16)
- long timestamp = Long.parseLong(parts[0].substring(1), 16);
- int counter = Integer.parseInt(parts[1]);
- int clusterId = Integer.parseInt(parts[2]);
-
- Revision revision = new Revision(timestamp, counter, clusterId);
- sortedRevisions.put(revision, value);
-
- // Add all revisions to the "_revisions" list
- allRevisions.put(revision, value);
- }
- entries.put(property, sortedRevisions);
- }
-
- Mockito.when(workingDocument.entrySet()).thenReturn(entries.entrySet());
- Mockito.when(workingDocument.get("_deleted")).thenReturn(entries.get("_deleted"));
- Mockito.when(workingDocument.get("_revisions")).thenReturn(allRevisions);
- }
-
- private void prepareCheckpointsMock(String jsonCheckpoints) throws IOException {
- String json = jsonCheckpoints.replace("'", "\"");
-
- ObjectMapper objectMapper = new ObjectMapper();
- Map<String, Map<String, Object>> data = objectMapper.readValue(json, new TypeReference<>() {});
-
- SortedMap<Revision, Checkpoints.Info> checkpoints = new TreeMap<>(StableRevisionComparator.REVERSE);
- for (Map.Entry<String, Map<String, Object>> entry : data.entrySet()) {
- String checkpointStr = entry.getKey();
- Map<String, Object> checkpointData = entry.getValue();
-
- String[] parts = checkpointStr.split("-");
- long timestamp = Long.parseLong(parts[0].substring(1), 16);
- int counter = Integer.parseInt(parts[1]);
- int clusterId = Integer.parseInt(parts[2]);
-
- Revision checkpoint = new Revision(timestamp, counter, clusterId);
- String checkpointDataJson = objectMapper.writeValueAsString(checkpointData);
-
- Checkpoints.Info info = Checkpoints.Info.fromString(checkpointDataJson);
- checkpoints.put(checkpoint, info);
- }
-
- Mockito.when(documentNodeStore.getCheckpoints()).thenReturn(this.checkpoints);
- Mockito.when(documentNodeStore.getCheckpoints().getCheckpoints()).thenReturn(checkpoints);
-
- /*Document cdoc = SETTINGS.newDocument(documentStore);
- Mockito.when(documentNodeStore.getCheckpoints()).thenReturn(this.checkpoints);
- Mockito.when(documentStore.find(Collection.SETTINGS, "checkpoint", 0)).thenReturn(cdoc);*/
- }
-}
\ No newline at end of file
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleanerTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleanerTest.java
index 4cd9731c21..ab0880ea37 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleanerTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleanerTest.java
@@ -2,7 +2,6 @@ package org.apache.jackrabbit.oak.plugins.document;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mock;
@@ -40,25 +39,18 @@ public class NodeDocumentRevisionCleanerTest {
@Mock
Checkpoints checkpoints;
- //@Mock
NodeDocument workingDocument;
- NodeDocumentRevisionCleaner NodeDocumentRevisionCleaner;
+ NodeDocumentRevisionCleaner nodeDocumentRevisionCleaner;
@Before
public void setUp() {
MockitoAnnotations.initMocks(this);
- //workingDocument = Mockito.mock(NodeDocument.class);
-
- /*documentStore = new MemoryDocumentStore();
- documentNodeStore = new DocumentMK.Builder()
- .setDocumentStore(documentStore).setAsyncDelay(0).setClusterId(1).build();*/
-
workingDocument = new NodeDocument(documentStore);
Mockito.when(documentStore.find(Mockito.eq(NODES), Mockito.anyString())).thenReturn(workingDocument);
Mockito.when(documentNodeStore.getBranches()).thenReturn(new UnmergedBranches());
- NodeDocumentRevisionCleaner = new NodeDocumentRevisionCleaner(documentStore, documentNodeStore, "/");
+ nodeDocumentRevisionCleaner = new NodeDocumentRevisionCleaner(documentStore, documentNodeStore, workingDocument);
}
@Test
@@ -86,15 +78,15 @@ public class NodeDocumentRevisionCleanerTest {
prepareDocumentMock(jsonProperties);
prepareCheckpointsMock(jsonCheckpoints);
- NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties();
- NodeDocumentRevisionCleaner.revisionCleaner.markCheckpointRevisionsToPreserve();
- NodeDocumentRevisionCleaner.revisionCleaner.removeCandidatesInList();
+ nodeDocumentRevisionCleaner.classifyRevisionsAndProperties();
+ nodeDocumentRevisionCleaner.markCheckpointRevisionsToPreserve();
+ nodeDocumentRevisionCleaner.removeCandidatesInList();
// The revisions blocked should be:
// - r105000000-0-1 (blocked by checkpoint r109000000-0-1)
// - r115000000-0-1 (blocked by checkpoint r119000000-0-1)
- assertEquals(Set.of(revisionB, revisionD), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1));
- assertEquals(Set.of(revisionA, revisionC, revisionE, revisionF), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
+ assertEquals(Set.of(revisionB, revisionD), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1));
+ assertEquals(Set.of(revisionA, revisionC, revisionE, revisionF), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
}
@Test
@@ -136,9 +128,9 @@ public class NodeDocumentRevisionCleanerTest {
prepareDocumentMock(jsonBuilder.toString());
- NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties();
- NodeDocumentRevisionCleaner.revisionCleaner.markRevisionsNewerThanThresholdToPreserve(24, ChronoUnit.HOURS);
- NodeDocumentRevisionCleaner.revisionCleaner.removeCandidatesInList();
+ nodeDocumentRevisionCleaner.classifyRevisionsAndProperties();
+ nodeDocumentRevisionCleaner.markRevisionsNewerThanThresholdToPreserve(24, ChronoUnit.HOURS);
+ nodeDocumentRevisionCleaner.removeCandidatesInList();
// The candidate revisions should be the 6 oldest ones (current time -29, -28, -27, -26, -25, -24)
// and the one created 24 hours and 1 minute ago
@@ -149,8 +141,8 @@ public class NodeDocumentRevisionCleanerTest {
// Blocked revisions are all the 24 revisions created in the last 24 hours, and the one created 23 hours and 59 minutes ago
Set<Revision> expectedBlockedRevisions = revisions.subList(6, 31).stream().collect(Collectors.toSet());
- assertEquals(expectedCandidateRevisions, NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
- assertEquals(expectedBlockedRevisions, NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1));
+ assertEquals(expectedCandidateRevisions, nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
+ assertEquals(expectedBlockedRevisions, nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1));
}
@Test
@@ -221,24 +213,24 @@ public class NodeDocumentRevisionCleanerTest {
prepareDocumentMock(jsonBuilder.toString());
prepareCheckpointsMock(jsonCheckpoints);
- NodeDocumentRevisionCleaner.initializeCleanupProcess();
+ nodeDocumentRevisionCleaner.collectOldRevisions(null);
// The revisions blocked should be:
// - r106000003-0-3, r118000004-0-2, r108000005-0-3 (referenced by checkpoint 1)
// - r118000009-1-1, r118000010-0-2, r118000011-0-3 (referenced by checkpoint 2)
// - r130000016-1-1, r130000017-0-2, r130000018-0-3 (last revision)
- assertEquals(Set.of(revs.get(3), revs.get(11), revs.get(20)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1));
- assertEquals(Set.of(revs.get(4), revs.get(12), revs.get(21)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2));
- assertEquals(Set.of(revs.get(5), revs.get(13), revs.get(22)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3));
+ assertEquals(Set.of(revs.get(3), revs.get(11), revs.get(20)), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1));
+ assertEquals(Set.of(revs.get(4), revs.get(12), revs.get(21)), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2));
+ assertEquals(Set.of(revs.get(5), revs.get(13), revs.get(22)), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3));
// Rest of revisions are candidates to clean
- assertEquals(Set.of(revs.get(0), revs.get(9), revs.get(10), revs.get(14), revs.get(19)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
- assertEquals(Set.of(revs.get(1), revs.get(8), revs.get(15), revs.get(16), revs.get(17)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2));
- assertEquals(Set.of(revs.get(2), revs.get(6), revs.get(7), revs.get(18)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3));
+ assertEquals(Set.of(revs.get(0), revs.get(9), revs.get(10), revs.get(14), revs.get(19)), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
+ assertEquals(Set.of(revs.get(1), revs.get(8), revs.get(15), revs.get(16), revs.get(17)), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2));
+ assertEquals(Set.of(revs.get(2), revs.get(6), revs.get(7), revs.get(18)), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3));
- assertTrue(Collections.disjoint(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)));
- assertTrue(Collections.disjoint(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2)));
- assertTrue(Collections.disjoint(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3)));
+ assertTrue(Collections.disjoint(nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)));
+ assertTrue(Collections.disjoint(nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2)));
+ assertTrue(Collections.disjoint(nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3)));
}
@Test
@@ -287,120 +279,21 @@ public class NodeDocumentRevisionCleanerTest {
prepareDocumentMock(jsonBuilder.toString());
prepareCheckpointsMock(jsonCheckpoints);
- NodeDocumentRevisionCleaner.initializeCleanupProcess();
+ nodeDocumentRevisionCleaner.collectOldRevisions(null);
// The revisions blocked should be:
// - r103000008-0-1, r103000009-0-2, r103000010-0-3 (last revisions)
// - r102000006-0-1, r101300005-0-3, r102000007-0-3 (referenced by checkpoint for clusters 1, 2 and 3 respectively)
- assertEquals(Set.of(revs.get(6), revs.get(8)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1));
- assertEquals(Set.of(revs.get(9)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2));
- assertEquals(Set.of(revs.get(5), revs.get(7), revs.get(10)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3));
+ assertEquals(Set.of(revs.get(6), revs.get(8)), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1));
+ assertEquals(Set.of(revs.get(9)), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2));
+ assertEquals(Set.of(revs.get(5), revs.get(7), revs.get(10)), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3));
// Rest of revisions are candidates to clean
- assertEquals(Set.of(revs.get(0), revs.get(3)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
- assertEquals(Set.of(revs.get(1), revs.get(4)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2));
- assertEquals(Set.of(revs.get(2)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3));
+ assertEquals(Set.of(revs.get(0), revs.get(3)), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
+ assertEquals(Set.of(revs.get(1), revs.get(4)), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2));
+ assertEquals(Set.of(revs.get(2)), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3));
}
- // New test copy
- /*@Test
- public void testInitializeCleanupProcessMultipleClustersNew() throws IOException {
- List<Revision> revs = new ArrayList<>();
-
- // Some initial revisions (0-2)
- revs.add(Revision.fromString("r100000000-0-1"));
- revs.add(Revision.fromString("r100000001-0-2"));
- revs.add(Revision.fromString("r100000002-0-3"));
-
- // Blocked by first checkpoint r109000000 (3-5)
- revs.add(Revision.fromString("r106000003-0-1"));
- revs.add(Revision.fromString("r107000004-0-2"));
- revs.add(Revision.fromString("r108000005-0-3"));
-
- // Some unblocked revisions in middle (6-10)
- revs.add(Revision.fromString("r110000006-0-3"));
- revs.add(Revision.fromString("r110000006-1-3"));
- revs.add(Revision.fromString("r114000007-0-2"));
- revs.add(Revision.fromString("r115000008-0-1"));
- revs.add(Revision.fromString("r118000009-0-1"));
-
- // Blocked by second checkpoint r118000000 (11-13)
- revs.add(Revision.fromString("r118000009-1-1"));
- revs.add(Revision.fromString("r118000010-0-2"));
- revs.add(Revision.fromString("r118000011-0-3"));
-
- // Some more unblocked revisions (14-19)
- revs.add(Revision.fromString("r120000012-0-1"));
- revs.add(Revision.fromString("r122000013-0-2"));
- revs.add(Revision.fromString("r122000013-1-2"));
- revs.add(Revision.fromString("r123000014-0-2"));
- revs.add(Revision.fromString("r125000015-0-3"));
- revs.add(Revision.fromString("r130000016-0-1"));
-
- // Last revision (20-22)
- revs.add(Revision.fromString("r130000016-1-1"));
- revs.add(Revision.fromString("r130000017-0-2"));
- revs.add(Revision.fromString("r130000018-0-3"));
-
- // Checkpoint revisions
- Revision checkpoint1 = Revision.fromString("r109000000-0-1");
- Revision checkpoint2 = Revision.fromString("r119000000-0-1");
-
- StringBuilder jsonPropBuilder = new StringBuilder("'prop1': {");
- StringBuilder jsonRevisionsBuilder = new StringBuilder("'_revisions': {");
-
- for (int i = 0; i < revs.size(); i++) {
- jsonPropBuilder.append("'").append(revs.get(i)).append("': 'value").append(i).append("'");
- jsonRevisionsBuilder.append("'").append(revs.get(i)).append("': 'c'");
- if (i < revs.size() - 1) {
- jsonPropBuilder.append(", ");
- jsonRevisionsBuilder.append(", ");
- }
- }
-
- jsonPropBuilder.append("}");
- jsonRevisionsBuilder.append("}");
- StringBuilder jsonBuilder = new StringBuilder("{");
- jsonBuilder.append(jsonPropBuilder).append(", ").append(jsonRevisionsBuilder).append(", '_deleted': {'r100000000-0-1': 'false'}").append("}");
-
- String jsonCheckpoints = "{" +
- "'" + checkpoint1 + "': {'expires':'200000000','rv':'r106500003-0-1,r107500004-0-2,r108500005-0-3'}," +
- "'" + checkpoint2 + "': {'expires':'200000000','rv':'r118000009-1-1,r118000010-0-2,r118000011-0-3'}" +
- "}";
-
- prepareDocumentMock(jsonBuilder.toString());
- prepareCheckpointsMock(jsonCheckpoints);
-
- NodeDocumentRevisionCleaner.initializeCleanupProcess();
-
- RevisionVector readRevision1 = documentNodeStore.getCheckpoints().getCheckpoints().get(checkpoint1).getCheckpoint();
- RevisionVector readRevision2 = documentNodeStore.getCheckpoints().getCheckpoints().get(checkpoint2).getCheckpoint();
-
- //workingDocument.getLiveRevision();
- DocumentNodeState state1 = workingDocument.getNodeAtRevision(documentNodeStore, readRevision1,null);
- DocumentNodeState state2 = workingDocument.getNodeAtRevision(documentNodeStore, readRevision2,null);
-
- System.out.println(state1);
- System.out.println(state2);
-
- // The revisions blocked should be:
- // - r106000003-0-3, r118000004-0-2, r108000005-0-3 (referenced by checkpoint 1)
- // - r118000009-1-1, r118000010-0-2, r118000011-0-3 (referenced by checkpoint 2)
- // - r130000016-1-1, r130000017-0-2, r130000018-0-3 (last revision)
- /*assertEquals(Set.of(revs.get(3), revs.get(11), revs.get(20)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1));
- assertEquals(Set.of(revs.get(4), revs.get(12), revs.get(21)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2));
- assertEquals(Set.of(revs.get(5), revs.get(13), revs.get(22)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3));
-
- // Rest of revisions are candidates to clean
- assertEquals(Set.of(revs.get(0), revs.get(9), revs.get(10), revs.get(14), revs.get(19)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
- assertEquals(Set.of(revs.get(1), revs.get(8), revs.get(15), revs.get(16), revs.get(17)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2));
- assertEquals(Set.of(revs.get(2), revs.get(6), revs.get(7), revs.get(18)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3));
-
- assertTrue(Collections.disjoint(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)));
- assertTrue(Collections.disjoint(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2)));
- assertTrue(Collections.disjoint(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3)));*/
- //}
-
@Test
public void testLastRevisionIsBlocked() throws IOException {
Revision revisionA = new Revision(111111111L, 0, 1);
@@ -413,13 +306,13 @@ public class NodeDocumentRevisionCleanerTest {
"}";
prepareDocumentMock(jsonProperties);
- NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties();
- NodeDocumentRevisionCleaner.revisionCleaner.markLastRevisionForEachProperty();
+ nodeDocumentRevisionCleaner.classifyRevisionsAndProperties();
+ nodeDocumentRevisionCleaner.markLastRevisionForEachProperty();
- assertFalse(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA));
- assertFalse(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionB));
- assertEquals(Set.of(revisionC), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1));
- assertEquals(Set.of(revisionA, revisionB, revisionC), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
+ assertFalse(nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA));
+ assertFalse(nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionB));
+ assertEquals(Set.of(revisionC), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1));
+ assertEquals(Set.of(revisionA, revisionB, revisionC), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
}
@Test
@@ -434,10 +327,10 @@ public class NodeDocumentRevisionCleanerTest {
"}";
prepareDocumentMock(jsonProperties);
- NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties();
+ nodeDocumentRevisionCleaner.classifyRevisionsAndProperties();
- assertTrue(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA));
- assertEquals(Set.of(revisionA, revisionB, revisionC), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
+ assertTrue(nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA));
+ assertEquals(Set.of(revisionA, revisionB, revisionC), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
}
@Test
@@ -452,84 +345,12 @@ public class NodeDocumentRevisionCleanerTest {
"}";
prepareDocumentMock(jsonProperties);
- NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties();
+ nodeDocumentRevisionCleaner.classifyRevisionsAndProperties();
- assertTrue(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA));
- assertEquals(Set.of(revisionA, revisionB, revisionC), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
+ assertTrue(nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA));
+ assertEquals(Set.of(revisionA, revisionB, revisionC), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
}
- /*@Test
- public void testFirstDeletedRevisionIsBlocked() throws Exception {
- Revision revisionA = new Revision(111111111L, 0, 1);
- Revision revisionB = new Revision(222222222L, 0, 1);
- Revision revisionC = new Revision(333333333L, 0, 1);
-
- String jsonProperties = "{" +
- "'_deleted': {'" + revisionA + "': 'false', '" + revisionB + "': 'true', '" + revisionC + "': 'false'}," +
- "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c'}" +
- "}";
- //prepareDocumentMock(jsonProperties);
-
- insertJsonDataToNodeDocument(workingDocument, jsonProperties);
-
- NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties();
-
- assertTrue(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA));
- assertEquals(Set.of(revisionA, revisionB, revisionC), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1));
- }*/
-
- /*private void insertJsonDataToNodeDocument(NodeDocument document, String jsonProperties) throws Exception {
- String json = jsonProperties.replace("'", "\"");
-
- ObjectMapper mapper = new ObjectMapper();
- // TypeReference helps in specifying the complex type (Map<String, Object>)
- Map<String, Object> parsedData = mapper.readValue(json, new TypeReference<Map<String, Object>>() {});
-
- // Since the data field is a Map<String, Object>, we need to ensure the inner maps are also correctly typed
- for (Map.Entry<String, Object> entry : parsedData.entrySet()) {
- if (entry.getValue() instanceof Map) {
- Map<?, ?> innerMap = (Map<?, ?>) entry.getValue();
- Map<String, Object> correctedInnerMap = new TreeMap<>();
- for (Map.Entry<?, ?> innerEntry : innerMap.entrySet()) {
- correctedInnerMap.put(innerEntry.getKey().toString(), innerEntry.getValue());
- }
- parsedData.put(entry.getKey(), correctedInnerMap);
- }
- }
-
- document.data = parsedData; // Directly setting the protected field
- }*/
-
- /*@Test
- public void testtest() throws CommitFailedException, NoSuchFieldException, IllegalAccessException {
- DocumentStore store = new MemoryDocumentStore();
- DocumentNodeStore ns = new DocumentMK.Builder().setDocumentStore(store).setAsyncDelay(0).getNodeStore();
-
- // add properties
- for (int i = 0; i < 10; i++) {
- NodeBuilder nb = ns.getRoot().builder();
-
- nb.child("x").setProperty("p"+i, i);
- merge(ns, nb);
- }
-
- NodeDocument nodeDocument = store.find(NODES, "1:/x");
- System.out.println(nodeDocument.getPropertyNames());
-
- // Mocked data
- Map<String, Object> data = Maps.newHashMap();
-
-
- NodeDocument document = new NodeDocument(store, System.currentTimeMillis());
- document.data.put("foo", "bar");
- System.out.println(document.getPropertyNames());
-
- document.data = Maps.newHashMap();
-
-
- System.out.println(document.getPropertyNames());
- }*/
-
@Test
public void testClassifyAndMapRevisionsAndPropertiesWithDeleted() throws IOException {
Revision revisionA = new Revision(111111111L, 0, 1);
@@ -544,14 +365,14 @@ public class NodeDocumentRevisionCleanerTest {
"}";
prepareDocumentMock(jsonProperties);
- NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties();
+ nodeDocumentRevisionCleaner.classifyRevisionsAndProperties();
- SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = NodeDocumentRevisionCleaner.getPropertiesModifiedByRevision();
+ SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = nodeDocumentRevisionCleaner.getPropertiesModifiedByRevision();
assertEquals(Set.of("prop1", "_deleted"), propertiesModifiedByRevision.get(revisionA));
assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB));
assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC));
- SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = NodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster();
+ SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = nodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster();
assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("prop1").get(1));
assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop2").get(1));
assertEquals(Set.of(revisionA), revisionsModifyingProperty.get("_deleted").get(1));
@@ -569,14 +390,14 @@ public class NodeDocumentRevisionCleanerTest {
"}";
prepareDocumentMock(jsonProperties);
- NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties();
+ nodeDocumentRevisionCleaner.classifyRevisionsAndProperties();
- SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = NodeDocumentRevisionCleaner.getPropertiesModifiedByRevision();
+ SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = nodeDocumentRevisionCleaner.getPropertiesModifiedByRevision();
assertEquals(Set.of("_deleted"), propertiesModifiedByRevision.get(revisionA));
assertEquals(Set.of("_deleted"), propertiesModifiedByRevision.get(revisionB));
assertEquals(Set.of("_deleted"), propertiesModifiedByRevision.get(revisionC));
- SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = NodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster();
+ SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = nodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster();
assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("_deleted").get(1));
}
@@ -593,14 +414,14 @@ public class NodeDocumentRevisionCleanerTest {
"}";
prepareDocumentMock(jsonProperties);
- NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties();
+ nodeDocumentRevisionCleaner.classifyRevisionsAndProperties();
- SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = NodeDocumentRevisionCleaner.getPropertiesModifiedByRevision();
+ SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = nodeDocumentRevisionCleaner.getPropertiesModifiedByRevision();
assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionA));
assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB));
assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC));
- SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = NodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster();
+ SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = nodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster();
assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("prop1").get(1));
assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop2").get(1));
}
@@ -619,16 +440,16 @@ public class NodeDocumentRevisionCleanerTest {
"}";
prepareDocumentMock(jsonProperties);
- NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties();
+ nodeDocumentRevisionCleaner.classifyRevisionsAndProperties();
// Modifications done in revisionD should be ignored, as it is not committed
- SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = NodeDocumentRevisionCleaner.getPropertiesModifiedByRevision();
+ SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = nodeDocumentRevisionCleaner.getPropertiesModifiedByRevision();
assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionA));
assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB));
assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC));
assertNull(propertiesModifiedByRevision.get(revisionD));
- SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = NodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster();
+ SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = nodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster();
assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("prop1").get(1));
assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop2").get(1));
}
@@ -647,15 +468,15 @@ public class NodeDocumentRevisionCleanerTest {
"}";
prepareDocumentMock(jsonProperties);
- NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties();
+ nodeDocumentRevisionCleaner.classifyRevisionsAndProperties();
- SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = NodeDocumentRevisionCleaner.getPropertiesModifiedByRevision();
+ SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = nodeDocumentRevisionCleaner.getPropertiesModifiedByRevision();
assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionA));
assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB));
assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC));
assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionD));
- SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = NodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster();
+ SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = nodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster();
assertEquals(Set.of(revisionA, revisionD), revisionsModifyingProperty.get("prop1").get(1));
assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop1").get(2));
assertEquals(Set.of(revisionC), revisionsModifyingProperty.get("prop1").get(3));
@@ -696,9 +517,6 @@ public class NodeDocumentRevisionCleanerTest {
entries.put(property, sortedRevisions);
}
- //System.out.println("ENTRIES::" + entries);
- //System.out.println("DATA::" + data);
-
workingDocument.data = entries;
}
@@ -727,9 +545,5 @@ public class NodeDocumentRevisionCleanerTest {
Mockito.when(documentNodeStore.getCheckpoints()).thenReturn(this.checkpoints);
Mockito.when(documentNodeStore.getCheckpoints().getCheckpoints()).thenReturn(checkpoints);
-
- /*Document cdoc = SETTINGS.newDocument(documentStore);
- Mockito.when(documentNodeStore.getCheckpoints()).thenReturn(this.checkpoints);
- Mockito.when(documentStore.find(Collection.SETTINGS, "checkpoint", 0)).thenReturn(cdoc);*/
}
}
\ No newline at end of file