You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2017/10/10 15:17:07 UTC
svn commit: r1811710 - in /jackrabbit/oak/trunk/oak-lucene/src:
main/java/org/apache/jackrabbit/oak/plugins/index/lucene/
main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/
test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/
Author: chetanm
Date: Tue Oct 10 15:17:07 2017
New Revision: 1811710
URL: http://svn.apache.org/viewvc?rev=1811710&view=rev
Log:
OAK-6787 - Delete property index entries recursively in batches to avoid large transaction
Added:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDelete.java (with props)
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDeleteTest.java (with props)
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleanerTest.java
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java?rev=1811710&r1=1811709&r2=1811710&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java Tue Oct 10 15:17:07 2017
@@ -78,6 +78,7 @@ import org.apache.jackrabbit.oak.spi.gc.
import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider;
import org.apache.jackrabbit.oak.spi.query.QueryIndex;
import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+import org.apache.jackrabbit.oak.spi.state.Clusterable;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.apache.jackrabbit.oak.spi.whiteboard.Registration;
import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard;
@@ -795,6 +796,12 @@ public class LuceneIndexProviderService
}
cleaner = new PropertyIndexCleaner(nodeStore, indexPathService, asyncIndexInfoService, statisticsProvider);
+
+ //Proxy check for DocumentNodeStore
+ if (nodeStore instanceof Clusterable) {
+ cleaner.setRecursiveDelete(true);
+ log.info("PropertyIndexCleaner configured to perform recursive delete");
+ }
oakRegs.add(scheduleWithFixedDelay(whiteboard, cleaner,
ImmutableMap.of("scheduler.name", PropertyIndexCleaner.class.getName()),
cleanerInterval, true, true));
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java?rev=1811710&r1=1811709&r2=1811710&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java Tue Oct 10 15:17:07 2017
@@ -75,6 +75,7 @@ public class PropertyIndexCleaner implem
private final TimerStats cleanupTime;
private final MeterStats noopMeter;
+ private boolean recursiveDelete;
public PropertyIndexCleaner(NodeStore nodeStore, IndexPathService indexPathService,
AsyncIndexInfoService asyncIndexInfoService,
@@ -144,6 +145,15 @@ public class PropertyIndexCleaner implem
uniqueIndexCleaner = new UniqueIndexCleaner(unit, time);
}
+
+ public boolean isRecursiveDelete() {
+ return recursiveDelete;
+ }
+
+ public void setRecursiveDelete(boolean recursiveDelete) {
+ this.recursiveDelete = recursiveDelete;
+ }
+
List<String> getSyncIndexPaths() {
List<String> indexPaths = new ArrayList<>();
NodeState root = nodeStore.getRoot();
@@ -214,17 +224,25 @@ public class PropertyIndexCleaner implem
return;
}
- NodeState root = nodeStore.getRoot();
- NodeBuilder builder = root.builder();
+ if (recursiveDelete) {
+ for (String path : bucketPaths) {
+ RecursiveDelete rd = new RecursiveDelete(nodeStore, createCommitHook(),
+ PropertyIndexCleaner::createCommitInfo, path);
+ rd.run();
+ stats.numOfNodesDeleted += rd.getNumRemoved();
+ }
+ } else {
+ NodeState root = nodeStore.getRoot();
+ NodeBuilder builder = root.builder();
- for (String path : bucketPaths) {
- NodeBuilder bucket = child(builder, path);
- //TODO Recursive delete to avoid large transaction
- bucket.remove();
- }
+ for (String path : bucketPaths) {
+ NodeBuilder bucket = child(builder, path);
+ bucket.remove();
+ }
+ merge(builder);
+ }
stats.purgedBucketCount = bucketPaths.size();
- merge(builder);
}
private void purgeOldUniqueIndexEntries(Map<String, Long> asyncInfo, CleanupStats stats) throws CommitFailedException {
@@ -249,14 +267,18 @@ public class PropertyIndexCleaner implem
private void merge(NodeBuilder builder) throws CommitFailedException {
//TODO Configure validator
- CompositeHook hooks = new CompositeHook(
- ResetCommitAttributeHook.INSTANCE,
- new ConflictHook(new AnnotatingConflictHandler()),
- new EditorHook(CompositeEditorProvider.compose(singletonList(new ConflictValidatorProvider())))
- );
+ CompositeHook hooks = createCommitHook();
nodeStore.merge(builder, hooks, createCommitInfo());
}
+ private CompositeHook createCommitHook() {
+ return new CompositeHook(
+ ResetCommitAttributeHook.INSTANCE,
+ new ConflictHook(new AnnotatingConflictHandler()),
+ new EditorHook(CompositeEditorProvider.compose(singletonList(new ConflictValidatorProvider())))
+ );
+ }
+
private static CommitInfo createCommitInfo() {
Map<String, Object> info = ImmutableMap.of(CommitContext.NAME, new SimpleCommitContext());
return new CommitInfo(CommitInfo.OAK_UNKNOWN, CommitInfo.OAK_UNKNOWN, info);
@@ -282,6 +304,7 @@ public class PropertyIndexCleaner implem
public int purgedBucketCount;
public Set<String> purgedIndexPaths = new HashSet<>();
public boolean cleanupPerformed;
+ public int numOfNodesDeleted;
@Override
public String toString() {
Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDelete.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDelete.java?rev=1811710&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDelete.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDelete.java Tue Oct 10 15:17:07 2017
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene.property;
+
+import java.util.function.Supplier;
+
+import com.google.common.base.Stopwatch;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.spi.commit.CommitHook;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RecursiveDelete {
+ private final Logger log = LoggerFactory.getLogger(getClass());
+ private final NodeStore nodeStore;
+ private final CommitHook commitHook;
+ private final Supplier<CommitInfo> commitInfo;
+ private final String path;
+ private int batchSize = 1024;
+ private int numRemoved = 0;
+ private int mergeCount;
+ private NodeBuilder builder;
+
+ public RecursiveDelete(NodeStore nodeStore, CommitHook commitHook,
+ Supplier<CommitInfo> commitInfo, String path) {
+ this.nodeStore = nodeStore;
+ this.commitHook = commitHook;
+ this.commitInfo = commitInfo;
+ this.path = path;
+ }
+
+ public void run() throws CommitFailedException {
+ NodeState root = nodeStore.getRoot();
+ builder = root.builder();
+ NodeState node = NodeStateUtils.getNode(root, path);
+ Stopwatch w = Stopwatch.createStarted();
+
+ int currentSize = delete(node, path);
+ save(path, currentSize, true);
+
+ log.debug("Removed subtree under [{}] with {} child nodes " +
+ "in {} ({} saves)", path, numRemoved, w, mergeCount);
+ }
+
+ public int getNumRemoved() {
+ return numRemoved;
+ }
+
+ public int getMergeCount() {
+ return mergeCount;
+ }
+
+ public void setBatchSize(int batchSize) {
+ this.batchSize = batchSize;
+ }
+
+ private int delete(NodeState node, String path) throws CommitFailedException {
+ int currentSize = deleteChildNodes(node, path);
+ child(builder, path).remove();
+ numRemoved++;
+ return currentSize + 1;
+ }
+
+ private int deleteChildNodes(NodeState node, String path) throws CommitFailedException {
+ int currentSize = 0;
+ for (ChildNodeEntry cne : node.getChildNodeEntries()) {
+ String name = cne.getName();
+ String childPath = PathUtils.concat(path, name);
+
+ currentSize += delete(cne.getNodeState(), childPath);
+ if (save(childPath, currentSize, false)) {
+ currentSize = 0;
+ }
+ }
+ return currentSize;
+ }
+
+ private boolean save(String path, int currentSize, boolean force) throws CommitFailedException {
+ if (currentSize >= batchSize || force) {
+ log.debug("Deleting {} nodes on {} ({} removed so far)", currentSize, path, numRemoved);
+ nodeStore.merge(builder, commitHook, commitInfo.get());
+ builder = nodeStore.getRoot().builder();
+ mergeCount++;
+ return true;
+ }
+ return false;
+ }
+
+ private static NodeBuilder child(NodeBuilder nb, String path) {
+ for (String name : PathUtils.elements(path)) {
+ nb = nb.getChildNode(name);
+ }
+ return nb;
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDelete.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleanerTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleanerTest.java?rev=1811710&r1=1811709&r2=1811710&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleanerTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleanerTest.java Tue Oct 10 15:17:07 2017
@@ -251,6 +251,39 @@ public class PropertyIndexCleanerTest {
assertCleanUpPerformed(cleaner.performCleanup(false), false);
}
+ @Test
+ public void recursiveDelete() throws Exception{
+ IndexDefinitionBuilder defnb = new IndexDefinitionBuilder();
+ defnb.indexRule("nt:base").property("foo").propertyIndex().sync();
+ String indexPath = "/oak:index/foo";
+ addIndex(indexPath, defnb);
+
+ PropertyIndexCleaner cleaner =
+ new PropertyIndexCleaner(nodeStore, () -> asList("/oak:index/uuid", indexPath), asyncService, StatisticsProvider.NOOP);
+ cleaner.setRecursiveDelete(true);
+
+ NodeBuilder builder = nodeStore.getRoot().builder();
+ PropertyIndexUpdateCallback cb = newCallback(builder, indexPath);
+ propertyUpdated(cb, indexPath, "/a", "foo", "bar");
+ merge(builder);
+
+ assertThat(query(indexPath, "foo", "bar"), containsInAnyOrder("/a"));
+
+ //------------------------ Run 1
+ asyncService.addInfo("async", 1000);
+ CleanupStats stats = cleaner.performCleanup(false);
+ assertCleanUpPerformed(stats, true);
+
+ asyncService.addInfo("async", 2000);
+ stats = cleaner.performCleanup(false);
+
+ //1 - for bucket
+ //1 - for indexed value 'bar'
+ //1 - for indexed path 'a'
+ assertEquals(3, stats.numOfNodesDeleted);
+
+ }
+
private void assertCleanUpPerformed(CleanupStats stats, boolean expected) {
assertEquals(expected, stats.cleanupPerformed);
}
Added: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDeleteTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDeleteTest.java?rev=1811710&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDeleteTest.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDeleteTest.java Tue Oct 10 15:17:07 2017
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene.property;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.TreeTraverser;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.commons.FixturesHelper;
+import org.apache.jackrabbit.oak.fixture.DocumentMemoryFixture;
+import org.apache.jackrabbit.oak.fixture.MemoryFixture;
+import org.apache.jackrabbit.oak.fixture.NodeStoreFixture;
+import org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.junit.After;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static org.apache.jackrabbit.oak.spi.state.NodeStateUtils.getNode;
+import static org.junit.Assert.*;
+
+@RunWith(Parameterized.class)
+public class RecursiveDeleteTest {
+ private final NodeStoreFixture fixture;
+ private final NodeStore nodeStore;
+ private String testNodePath = "/content/testNode";
+ private Random rnd = new Random();
+ private int maxBucketSize = 100;
+ private int maxDepth = 4;
+
+ public RecursiveDeleteTest(NodeStoreFixture fixture) {
+ this.nodeStore = fixture.createNodeStore();
+ this.fixture = fixture;
+ }
+
+ @After
+ public void tearDown(){
+ fixture.dispose(nodeStore);
+ }
+
+ @Parameterized.Parameters(name="{0}")
+ public static Collection<Object[]> fixtures() {
+ Collection<Object[]> result = new ArrayList<Object[]>();
+ result.add(new Object[]{new MemoryFixture()});
+ result.add(new Object[]{new DocumentMemoryFixture()});
+ return result;
+ }
+
+ @Test
+ public void recursiveDelete() throws Exception{
+ int actualCount = createSubtree(10000);
+ assertEquals(actualCount, getSubtreeCount(getNode(nodeStore.getRoot(), testNodePath)));
+
+ RecursiveDelete rd = new RecursiveDelete(nodeStore, EmptyHook.INSTANCE, () -> CommitInfo.EMPTY, testNodePath);
+ rd.setBatchSize(100);
+ rd.run();
+
+ assertEquals(actualCount, rd.getNumRemoved());
+ assertFalse(getNode(nodeStore.getRoot(), testNodePath).exists());
+
+ System.out.println(rd.getMergeCount());
+ System.out.println(actualCount);
+ }
+
+ private int createSubtree(int maxNodesCount) throws CommitFailedException {
+ NodeBuilder builder = nodeStore.getRoot().builder();
+ NodeBuilder child = TestUtil.child(builder, testNodePath);
+ AtomicInteger maxNodes = new AtomicInteger(maxNodesCount);
+ int actualCount = createChildren(child, maxNodes, 0);
+ nodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ return actualCount + 1;
+ }
+
+ private int createChildren(NodeBuilder child, AtomicInteger maxNodes, int depth) {
+ if (maxNodes.get() <= 0 || depth > maxDepth) {
+ return 0;
+ }
+
+ int totalCount = 0;
+ int childCount = rnd.nextInt(maxBucketSize);
+ if (childCount == 0) {
+ childCount = 1;
+ }
+
+ List<NodeBuilder> children = new ArrayList<>();
+ for (int i = 0; i < childCount && maxNodes.get() > 0; i++){
+ maxNodes.decrementAndGet();
+ totalCount++;
+ children.add(child.child("c"+i));
+
+ }
+
+ for (NodeBuilder c : children) {
+ totalCount += createChildren(c, maxNodes, depth + 1);
+ }
+
+ return totalCount;
+ }
+
+ private int getSubtreeCount(NodeState state){
+ TreeTraverser<NodeState> t = new TreeTraverser<NodeState>() {
+ @Override
+ public Iterable<NodeState> children(NodeState root) {
+ return Iterables.transform(root.getChildNodeEntries(), ChildNodeEntry::getNodeState);
+ }
+ };
+ return t.preOrderTraversal(state).size();
+ }
+
+}
\ No newline at end of file
Propchange: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDeleteTest.java
------------------------------------------------------------------------------
svn:eol-style = native