You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by th...@apache.org on 2020/05/20 14:17:31 UTC
svn commit: r1877956 - in
/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene:
LuceneIndexMBean.java LuceneIndexMBeanImpl.java
property/PropertyIndexCleaner.java property/RecursiveDelete.java
Author: thomasm
Date: Wed May 20 14:17:31 2020
New Revision: 1877956
URL: http://svn.apache.org/viewvc?rev=1877956&view=rev
Log:
OAK-9065 JMX feature to remove large disabled indexes (recursive delete)
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBean.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDelete.java
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBean.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBean.java?rev=1877956&r1=1877955&r2=1877956&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBean.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBean.java Wed May 20 14:17:31 2020
@@ -125,6 +125,20 @@ public interface LuceneIndexMBean {
@Description("Performs any possible cleanup of the hybrid property indexes")
String performPropertyIndexCleanup() throws CommitFailedException;
+ @Description("Performs cleanup of property indexes")
+ String performPropertyIndexCleanup(
+ @Name("paths")
+ @Description("The list of paths (comma separated). Must be hidden nodes")
+ String paths,
+ @Name("batchSize")
+ @Description("The batch size, e.g. 1024")
+ int batchSize,
+ @Name("sleepPerBatch")
+ @Description("The number of milliseconds to sleep per batch")
+ int sleepPerBatch,
+ @Name("maxRemoveCount")
+ @Description("The maximum number of nodes to remove per path, e.g. 1000000")
+ int maxRemoveCount) throws CommitFailedException;
@Description("Fetches hybrid property index info as json for index at given path")
String getHybridIndexInfo(@Name("indexPath") String indexPath);
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java?rev=1877956&r1=1877955&r2=1877956&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java Wed May 20 14:17:31 2020
@@ -19,10 +19,20 @@
package org.apache.jackrabbit.oak.plugins.index.lucene;
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.TermFactory.newAncestorTerm;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.directory.DirectoryUtils.dirSize;
+import static org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.INDEX_DEFINITION_NODE;
+
import java.io.File;
import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
import javax.management.openmbean.CompositeDataSupport;
import javax.management.openmbean.CompositeType;
@@ -33,10 +43,6 @@ import javax.management.openmbean.Tabula
import javax.management.openmbean.TabularDataSupport;
import javax.management.openmbean.TabularType;
-import com.google.common.base.Stopwatch;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Sets;
-import com.google.common.collect.TreeTraverser;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.api.jmx.Name;
import org.apache.jackrabbit.oak.commons.PathUtils;
@@ -45,15 +51,15 @@ import org.apache.jackrabbit.oak.commons
import org.apache.jackrabbit.oak.json.JsopDiff;
import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
import org.apache.jackrabbit.oak.plugins.index.IndexPathService;
-import org.apache.jackrabbit.oak.plugins.index.search.BadIndexTracker.BadIndexInfo;
-import org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexInfo;
-import org.apache.jackrabbit.oak.plugins.index.lucene.property.PropertyIndexCleaner;
-import org.apache.jackrabbit.oak.plugins.index.lucene.util.PathStoredFieldVisitor;
import org.apache.jackrabbit.oak.plugins.index.lucene.directory.IndexConsistencyChecker;
import org.apache.jackrabbit.oak.plugins.index.lucene.directory.IndexConsistencyChecker.Level;
import org.apache.jackrabbit.oak.plugins.index.lucene.directory.IndexConsistencyChecker.Result;
+import org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexInfo;
+import org.apache.jackrabbit.oak.plugins.index.lucene.property.PropertyIndexCleaner;
import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReader;
+import org.apache.jackrabbit.oak.plugins.index.lucene.util.PathStoredFieldVisitor;
import org.apache.jackrabbit.oak.plugins.index.search.BadIndexTracker;
+import org.apache.jackrabbit.oak.plugins.index.search.BadIndexTracker.BadIndexInfo;
import org.apache.jackrabbit.oak.plugins.index.search.FieldNames;
import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants;
import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
@@ -84,12 +90,10 @@ import org.jetbrains.annotations.Nullabl
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
-import static org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.INDEX_DEFINITION_NODE;
-import static org.apache.jackrabbit.oak.plugins.index.lucene.TermFactory.newAncestorTerm;
-import static org.apache.jackrabbit.oak.plugins.index.lucene.directory.DirectoryUtils.dirSize;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.common.collect.TreeTraverser;
public class LuceneIndexMBeanImpl extends AnnotatedStandardMBean implements LuceneIndexMBean {
@@ -366,6 +370,16 @@ public class LuceneIndexMBeanImpl extend
}
@Override
+ public String performPropertyIndexCleanup(String paths, int batchSize, int sleepPerBatch, int maxRemoveCount) throws CommitFailedException {
+ String result = "PropertyIndexCleaner not enabled";
+ if (propertyIndexCleaner != null) {
+ result = "Removed nodes: " + propertyIndexCleaner.performCleanup(paths, batchSize, sleepPerBatch, maxRemoveCount);
+ }
+ log.info("Explicit cleanup run done with result {}", result);
+ return result;
+ }
+
+ @Override
public String performPropertyIndexCleanup() throws CommitFailedException {
String result = "PropertyIndexCleaner not enabled";
if (propertyIndexCleaner != null) {
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java?rev=1877956&r1=1877955&r2=1877956&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java Wed May 20 14:17:31 2020
@@ -19,6 +19,15 @@
package org.apache.jackrabbit.oak.plugins.index.lucene.property;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Collections.singletonList;
+import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.TYPE_LUCENE;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.PROPERTY_INDEX;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.simplePropertyIndex;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.uniquePropertyIndex;
+import static org.apache.jackrabbit.oak.spi.state.NodeStateUtils.getNode;
+
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -28,8 +37,6 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
-import com.google.common.base.Stopwatch;
-import com.google.common.collect.ImmutableMap;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.plugins.commit.AnnotatingConflictHandler;
@@ -48,6 +55,7 @@ import org.apache.jackrabbit.oak.spi.com
import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.apache.jackrabbit.oak.stats.MeterStats;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
@@ -56,14 +64,8 @@ import org.apache.jackrabbit.oak.stats.T
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static com.google.common.base.Preconditions.checkNotNull;
-import static java.util.Collections.singletonList;
-import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
-import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.TYPE_LUCENE;
-import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.PROPERTY_INDEX;
-import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.simplePropertyIndex;
-import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.uniquePropertyIndex;
-import static org.apache.jackrabbit.oak.spi.state.NodeStateUtils.getNode;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.ImmutableMap;
public class PropertyIndexCleaner implements Runnable{
private final Logger log = LoggerFactory.getLogger(getClass());
@@ -98,6 +100,35 @@ public class PropertyIndexCleaner implem
}
/**
+ * Perform some cleanup.
+ *
+ * @param paths the list of paths (comma separated)
+ * @param batchSize the bach size
+ * @param sleepPerBatch the number of milliseconds to sleep per batch
+ * @param maxRemoveCount the maximum number of nodes to remove per path
+ * @return the number of nodes removed
+ */
+ public int performCleanup(String paths, int batchSize, int sleepPerBatch, int maxRemoveCount) throws CommitFailedException {
+ String[] list = paths.split(",");
+ int numOfNodesDeleted = 0;
+ for(String s : list) {
+ log.info("Cleanup of {}", s);
+ if (!NodeStateUtils.isHidden(PathUtils.getName(s))) {
+ log.warn("Not a hidden node: {}", s);
+ continue;
+ }
+ RecursiveDelete rd = new RecursiveDelete(nodeStore, createCommitHook(),
+ PropertyIndexCleaner::createCommitInfo);
+ rd.setBatchSize(batchSize);
+ rd.setSleepPerBatch(sleepPerBatch);
+ rd.setMaxRemoveCount(maxRemoveCount);
+ rd.run(singletonList(s));
+ numOfNodesDeleted += rd.getNumRemoved();
+ }
+ return numOfNodesDeleted;
+ }
+
+ /**
* Performs the cleanup run
*
* @param forceCleanup if true then clean up would attempted even if no change
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDelete.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDelete.java?rev=1877956&r1=1877955&r2=1877956&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDelete.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/RecursiveDelete.java Wed May 20 14:17:31 2020
@@ -41,7 +41,9 @@ public class RecursiveDelete {
private final CommitHook commitHook;
private final Supplier<CommitInfo> commitInfo;
private int batchSize = 1024;
- private int numRemoved = 0;
+ private int sleepPerBatch;
+ private int maxRemoveCount = Integer.MAX_VALUE;
+ private int numRemoved;
private int mergeCount;
private NodeBuilder builder;
@@ -96,6 +98,9 @@ public class RecursiveDelete {
private int delete(NodeState node, String path) throws CommitFailedException {
int currentSize = deleteChildNodes(node, path);
+ if (numRemoved >= maxRemoveCount) {
+ return currentSize;
+ }
child(builder, path).remove();
numRemoved++;
return currentSize + 1;
@@ -111,6 +116,9 @@ public class RecursiveDelete {
if (save(childPath, currentSize, false)) {
currentSize = 0;
}
+ if (numRemoved >= maxRemoveCount) {
+ break;
+ }
}
return currentSize;
}
@@ -121,6 +129,13 @@ public class RecursiveDelete {
nodeStore.merge(builder, commitHook, commitInfo.get());
builder = nodeStore.getRoot().builder();
mergeCount++;
+ if (sleepPerBatch > 0) {
+ try {
+ Thread.sleep(sleepPerBatch);
+ } catch (InterruptedException e) {
+ // ignore
+ }
+ }
return true;
}
return false;
@@ -132,4 +147,21 @@ public class RecursiveDelete {
}
return nb;
}
+
+ public int getMaxRemoveCount() {
+ return maxRemoveCount;
+ }
+
+ public void setMaxRemoveCount(int maxRemoveCount) {
+ this.maxRemoveCount = maxRemoveCount;
+ }
+
+ public int getSleepPerBatch() {
+ return sleepPerBatch;
+ }
+
+ public void setSleepPerBatch(int sleepPerBatch) {
+ this.sleepPerBatch = sleepPerBatch;
+ }
+
}