You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2017/10/12 06:47:05 UTC
svn commit: r1811912 - in /jackrabbit/oak/trunk/oak-lucene/src:
main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/
test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/
Author: chetanm
Date: Thu Oct 12 06:47:04 2017
New Revision: 1811912
URL: http://svn.apache.org/viewvc?rev=1811912&view=rev
Log:
OAK-6787 - Delete property index entries recursively in batches to avoid large transaction
Modify RecursiveDelete to support batch delete for multiple root paths
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDelete.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDeleteTest.java
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java?rev=1811912&r1=1811911&r2=1811912&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java Thu Oct 12 06:47:04 2017
@@ -225,12 +225,10 @@ public class PropertyIndexCleaner implem
}
if (recursiveDelete) {
- for (String path : bucketPaths) {
- RecursiveDelete rd = new RecursiveDelete(nodeStore, createCommitHook(),
- PropertyIndexCleaner::createCommitInfo, path);
- rd.run();
- stats.numOfNodesDeleted += rd.getNumRemoved();
- }
+ RecursiveDelete rd = new RecursiveDelete(nodeStore, createCommitHook(),
+ PropertyIndexCleaner::createCommitInfo);
+ rd.run(bucketPaths);
+ stats.numOfNodesDeleted += rd.getNumRemoved();
} else {
NodeState root = nodeStore.getRoot();
NodeBuilder builder = root.builder();
@@ -308,8 +306,9 @@ public class PropertyIndexCleaner implem
@Override
public String toString() {
- return String.format("Removed %d index buckets, %d unique index entries " +
- "from indexes %s", purgedBucketCount, uniqueIndexEntryRemovalCount, purgedIndexPaths);
+ String nodeCountMsg = numOfNodesDeleted > 0 ? String.format("(%d nodes)", numOfNodesDeleted) : "";
+ return String.format("Removed %d index buckets %s, %d unique index entries " +
+ "from indexes %s", purgedBucketCount, nodeCountMsg, uniqueIndexEntryRemovalCount, purgedIndexPaths);
}
}
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDelete.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDelete.java?rev=1811912&r1=1811911&r2=1811912&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDelete.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDelete.java Thu Oct 12 06:47:04 2017
@@ -22,6 +22,7 @@ package org.apache.jackrabbit.oak.plugin
import java.util.function.Supplier;
import com.google.common.base.Stopwatch;
+import com.google.common.collect.Iterables;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.spi.commit.CommitHook;
@@ -39,21 +40,36 @@ public class RecursiveDelete {
private final NodeStore nodeStore;
private final CommitHook commitHook;
private final Supplier<CommitInfo> commitInfo;
- private final String path;
private int batchSize = 1024;
private int numRemoved = 0;
private int mergeCount;
private NodeBuilder builder;
public RecursiveDelete(NodeStore nodeStore, CommitHook commitHook,
- Supplier<CommitInfo> commitInfo, String path) {
+ Supplier<CommitInfo> commitInfo) {
this.nodeStore = nodeStore;
this.commitHook = commitHook;
this.commitInfo = commitInfo;
- this.path = path;
}
- public void run() throws CommitFailedException {
+ public void run(Iterable<String> paths) throws CommitFailedException {
+ Stopwatch w = Stopwatch.createStarted();
+ NodeState root = nodeStore.getRoot();
+ builder = root.builder();
+ int currentSize = 0;
+ for (String path : paths) {
+ NodeState node = NodeStateUtils.getNode(root, path);
+ currentSize = delete(node, path);
+ save(path, currentSize, false);
+ }
+
+ String pathDetails = Iterables.toString(paths);
+ save(pathDetails, currentSize, true);
+ log.debug("Removed subtree under [{}] with {} child nodes " +
+ "in {} ({} saves)", pathDetails, numRemoved, w, mergeCount);
+ }
+
+ public void run(String path) throws CommitFailedException {
NodeState root = nodeStore.getRoot();
builder = root.builder();
NodeState node = NodeStateUtils.getNode(root, path);
@@ -99,9 +115,9 @@ public class RecursiveDelete {
return currentSize;
}
- private boolean save(String path, int currentSize, boolean force) throws CommitFailedException {
+ private boolean save(String pathDetails, int currentSize, boolean force) throws CommitFailedException {
if (currentSize >= batchSize || force) {
- log.debug("Deleting {} nodes on {} ({} removed so far)", currentSize, path, numRemoved);
+ log.debug("Deleting {} nodes under {} ({} removed so far)", currentSize, pathDetails, numRemoved);
nodeStore.merge(builder, commitHook, commitInfo.get());
builder = nodeStore.getRoot().builder();
mergeCount++;
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDeleteTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDeleteTest.java?rev=1811912&r1=1811911&r2=1811912&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDeleteTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDeleteTest.java Thu Oct 12 06:47:04 2017
@@ -45,6 +45,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import static java.util.Arrays.asList;
import static org.apache.jackrabbit.oak.spi.state.NodeStateUtils.getNode;
import static org.junit.Assert.*;
@@ -80,9 +81,9 @@ public class RecursiveDeleteTest {
int actualCount = createSubtree(10000);
assertEquals(actualCount, getSubtreeCount(getNode(nodeStore.getRoot(), testNodePath)));
- RecursiveDelete rd = new RecursiveDelete(nodeStore, EmptyHook.INSTANCE, () -> CommitInfo.EMPTY, testNodePath);
+ RecursiveDelete rd = new RecursiveDelete(nodeStore, EmptyHook.INSTANCE, () -> CommitInfo.EMPTY);
rd.setBatchSize(100);
- rd.run();
+ rd.run(testNodePath);
assertEquals(actualCount, rd.getNumRemoved());
assertFalse(getNode(nodeStore.getRoot(), testNodePath).exists());
@@ -91,6 +92,28 @@ public class RecursiveDeleteTest {
System.out.println(actualCount);
}
+ @Test
+ public void multiplePaths() throws Exception{
+ int count = 121;
+ NodeBuilder nb = nodeStore.getRoot().builder();
+ nb.child("c");
+ for (int i = 0; i < count; i++) {
+ nb.child("a").child("c"+i);
+ nb.child("b").child("c"+i);
+ }
+ nodeStore.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ RecursiveDelete rd = new RecursiveDelete(nodeStore, EmptyHook.INSTANCE, () -> CommitInfo.EMPTY);
+ rd.setBatchSize(50);
+ rd.run(asList("/a", "/b"));
+
+ assertEquals(5, rd.getMergeCount());
+ assertEquals(2 * count + 2, rd.getNumRemoved());
+ assertFalse(getNode(nodeStore.getRoot(), "/a").exists());
+ assertFalse(getNode(nodeStore.getRoot(), "/b").exists());
+ assertTrue(getNode(nodeStore.getRoot(), "/c").exists());
+ }
+
private int createSubtree(int maxNodesCount) throws CommitFailedException {
NodeBuilder builder = nodeStore.getRoot().builder();
NodeBuilder child = TestUtil.child(builder, testNodePath);