You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2017/05/24 17:03:10 UTC
svn commit: r1796082 - in /jackrabbit/oak/trunk/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/index/
main/java/org/apache/jackrabbit/oak/plugins/index/progress/
test/java/org/apache/jackrabbit/oak/plugins/index/progress/
Author: chetanm
Date: Wed May 24 17:03:10 2017
New Revision: 1796082
URL: http://svn.apache.org/viewvc?rev=1796082&view=rev
Log:
OAK-5970 - (Re-)Indexing: estimate progress / ETA
Handle include and exclude of paths
Added:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/progress/NodeCounterMBeanEstimatorTest.java (with props)
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/IndexingProgressReporter.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/NodeCountEstimator.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/NodeCounterMBeanEstimator.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java?rev=1796082&r1=1796081&r2=1796082&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java Wed May 24 17:03:10 2017
@@ -800,8 +800,7 @@ public class AsyncIndexUpdate implements
indexUpdate.setTraversalRateEstimator(new MetricRateEstimator(name, registry));
}
- NodeCounter nodeCounter = new NodeCounter(store);
- NodeCounterMBeanEstimator estimator = new NodeCounterMBeanEstimator(nodeCounter);
+ NodeCounterMBeanEstimator estimator = new NodeCounterMBeanEstimator(store);
indexUpdate.setNodeCountEstimator(estimator);
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/IndexingProgressReporter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/IndexingProgressReporter.java?rev=1796082&r1=1796081&r2=1796082&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/IndexingProgressReporter.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/IndexingProgressReporter.java Wed May 24 17:03:10 2017
@@ -69,7 +69,7 @@ public class IndexingProgressReporter im
* @param path
*/
public void reindexingTraversalStart(String path) {
- estimatedCount = nodeCountEstimator.getEstimatedNodeCount(path);
+ estimatedCount = nodeCountEstimator.getEstimatedNodeCount(path, getReindexedIndexPaths());
if (estimatedCount >= 0) {
log.info("Estimated node count to be traversed for reindexing under {} is [{}]", path, estimatedCount);
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/NodeCountEstimator.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/NodeCountEstimator.java?rev=1796082&r1=1796081&r2=1796082&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/NodeCountEstimator.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/NodeCountEstimator.java Wed May 24 17:03:10 2017
@@ -19,13 +19,15 @@
package org.apache.jackrabbit.oak.plugins.index.progress;
+import java.util.Set;
+
public interface NodeCountEstimator {
- NodeCountEstimator NOOP = path -> -1;
+ NodeCountEstimator NOOP = (basePath, indexPaths) -> -1;
/**
* Provides an estimate of the sub tree node count at given path
* @param path path under which count is requested
* @return estimated count or -1 if unknown
*/
- long getEstimatedNodeCount(String path);
+ long getEstimatedNodeCount(String basePath, Set<String> indexPaths);
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/NodeCounterMBeanEstimator.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/NodeCounterMBeanEstimator.java?rev=1796082&r1=1796081&r2=1796082&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/NodeCounterMBeanEstimator.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/NodeCounterMBeanEstimator.java Wed May 24 17:03:10 2017
@@ -19,17 +19,83 @@
package org.apache.jackrabbit.oak.plugins.index.progress;
-import org.apache.jackrabbit.oak.plugins.index.counter.jmx.NodeCounterMBean;
+import java.util.HashSet;
+import java.util.Set;
+
+import com.google.common.collect.Iterables;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.plugins.index.counter.jmx.NodeCounter;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+
+import static org.apache.jackrabbit.oak.plugins.index.PathFilter.PROP_EXCLUDED_PATHS;
+import static org.apache.jackrabbit.oak.plugins.index.PathFilter.PROP_INCLUDED_PATHS;
public class NodeCounterMBeanEstimator implements NodeCountEstimator {
- private final NodeCounterMBean counterMBean;
+ private final NodeCounter counter;
+ private final NodeStore nodeStore;
+
+ public NodeCounterMBeanEstimator(NodeStore nodeStore) {
+ this(nodeStore, new NodeCounter(nodeStore));
+ }
- public NodeCounterMBeanEstimator(NodeCounterMBean counterMBean) {
- this.counterMBean = counterMBean;
+ NodeCounterMBeanEstimator(NodeStore nodeStore, NodeCounter nodeCounter) {
+ this.nodeStore = nodeStore;
+ this.counter = nodeCounter;
}
@Override
- public long getEstimatedNodeCount(String path) {
- return counterMBean.getEstimatedNodeCount(path);
+ public long getEstimatedNodeCount(String basePath, Set<String> indexPaths) {
+ PathPerimeter pp = new PathPerimeter();
+ pp.compute(basePath, indexPaths);
+
+ if (pp.includeAll) {
+ return counter.getEstimatedNodeCount(basePath);
+ } else {
+ long totalCount = 0;
+ for (String path : pp.includes) {
+ long estimate = counter.getEstimatedNodeCount(path);
+ if (estimate > 0) {
+ totalCount += estimate;
+ }
+ }
+ for (String path : pp.excludes) {
+ long estimate = counter.getEstimatedNodeCount(path);
+ if (estimate > 0) {
+ totalCount -= estimate;
+ }
+ }
+ return totalCount;
+ }
+ }
+
+ private class PathPerimeter {
+ final Set<String> includes = new HashSet<>();
+ final Set<String> excludes = new HashSet<>();
+ boolean includeAll;
+
+ public void compute(String basePath, Set<String> indexPaths) {
+ NodeState root = nodeStore.getRoot();
+ for (String indexPath : indexPaths) {
+ NodeState idxState = NodeStateUtils.getNode(root, indexPath);
+
+ //No include exclude specified so include all
+ if (!idxState.hasProperty(PROP_INCLUDED_PATHS)
+ && !idxState.hasProperty(PROP_EXCLUDED_PATHS)) {
+ includeAll = true;
+ return;
+ }
+
+ Iterables.addAll(includes, idxState.getStrings(PROP_INCLUDED_PATHS));
+ Iterables.addAll(excludes, idxState.getStrings(PROP_EXCLUDED_PATHS));
+ }
+
+ if (includes.isEmpty()) {
+ includes.add(basePath);
+ }
+
+ PathUtils.unifyInExcludes(includes, excludes);
+ }
}
}
Added: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/progress/NodeCounterMBeanEstimatorTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/progress/NodeCounterMBeanEstimatorTest.java?rev=1796082&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/progress/NodeCounterMBeanEstimatorTest.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/progress/NodeCounterMBeanEstimatorTest.java Wed May 24 17:03:10 2017
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.progress;
+
+import java.util.Map;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.plugins.index.counter.jmx.NodeCounter;
+import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.junit.Test;
+
+import static com.google.common.collect.ImmutableSet.of;
+import static java.util.Arrays.asList;
+import static org.apache.jackrabbit.oak.plugins.index.PathFilter.PROP_EXCLUDED_PATHS;
+import static org.apache.jackrabbit.oak.plugins.index.PathFilter.PROP_INCLUDED_PATHS;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class NodeCounterMBeanEstimatorTest {
+
+ private final NodeStore store = new MemoryNodeStore();
+ private NodeCounter counter = mock(NodeCounter.class);
+
+ @Test
+ public void estimateCount() throws Exception {
+ NodeBuilder builder = store.getRoot().builder();
+
+ builder.child("idx-a");
+
+ builder.child("idx-b").setProperty(PROP_INCLUDED_PATHS, asList("/content"), Type.STRINGS);
+ builder.child("idx-b").setProperty(PROP_EXCLUDED_PATHS, asList("/content/old"), Type.STRINGS);
+
+ builder.child("idx-c").setProperty(PROP_INCLUDED_PATHS, asList("/libs"), Type.STRINGS);
+
+ builder.child("idx-d").setProperty(PROP_EXCLUDED_PATHS, asList("/libs"), Type.STRINGS);
+
+
+ builder.child("content").child("idx-e").setProperty(PROP_EXCLUDED_PATHS, asList("/content/old"), Type.STRINGS);
+ builder.child("content").child("idx-f");
+
+ store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ Map<String, Integer> counts = ImmutableMap.of(
+ "/", 100,
+ "/content", 50,
+ "/content/old", 10,
+ "/libs", 30
+ );
+
+ when(counter.getEstimatedNodeCount(anyString())).then((invocation -> {
+ String path = (String) invocation.getArguments()[0];
+ if (counts.containsKey(path)) {
+ return counts.get(path);
+ }
+ return -1;
+ }));
+
+ NodeCountEstimator estimator = new NodeCounterMBeanEstimator(store, counter);
+
+ assertEquals(100, estimator.getEstimatedNodeCount("/", of("/idx-a")));
+ assertEquals(100, estimator.getEstimatedNodeCount("/", of("/idx-a", "/idx-b")));
+ assertEquals(40, estimator.getEstimatedNodeCount("/", of("/idx-b")));
+ assertEquals(70, estimator.getEstimatedNodeCount("/", of("/idx-b", "/idx-c")));
+
+ assertEquals(50, estimator.getEstimatedNodeCount("/content", of("/content/idx-f")));
+ assertEquals(40, estimator.getEstimatedNodeCount("/content", of("/content/idx-e")));
+ }
+
+}
\ No newline at end of file
Propchange: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/progress/NodeCounterMBeanEstimatorTest.java
------------------------------------------------------------------------------
svn:eol-style = native