You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2017/05/24 11:09:29 UTC
svn commit: r1796042 - in
/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index:
./ progress/
Author: chetanm
Date: Wed May 24 11:09:29 2017
New Revision: 1796042
URL: http://svn.apache.org/viewvc?rev=1796042&view=rev
Log:
OAK-5970 - (Re-)Indexing: estimate progress / ETA
Estimate rate of traversal. Two estimators are provided
* Simple - This uses a simple mean rate of traversal
* Metric - This uses 5 min rate of traversal which is more accurate
Added:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/MetricRateEstimator.java (with props)
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/SimpleRateEstimator.java (with props)
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/TraversalRateEstimator.java (with props)
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdate.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/IndexingProgressReporter.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdate.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdate.java?rev=1796042&r1=1796041&r2=1796042&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdate.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdate.java Wed May 24 11:09:29 2017
@@ -52,6 +52,7 @@ import org.apache.jackrabbit.oak.api.Typ
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.plugins.index.NodeTraversalCallback.PathSource;
import org.apache.jackrabbit.oak.plugins.index.progress.IndexingProgressReporter;
+import org.apache.jackrabbit.oak.plugins.index.progress.TraversalRateEstimator;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.Editor;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
@@ -187,6 +188,10 @@ public class IndexUpdate implements Edit
return rootState.progressReporter.getUpdatedIndexPaths();
}
+ public void setTraversalRateEstimator(TraversalRateEstimator estimator){
+ rootState.progressReporter.setTraversalRateEstimator(estimator);
+ }
+
public String getIndexingStats(){
return rootState.getIndexingStats();
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/IndexingProgressReporter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/IndexingProgressReporter.java?rev=1796042&r1=1796041&r2=1796042&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/IndexingProgressReporter.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/IndexingProgressReporter.java Wed May 24 11:09:29 2017
@@ -46,6 +46,7 @@ public class IndexingProgressReporter im
private final Map<String, IndexUpdateState> indexUpdateStates = new HashMap<>();
private long traversalCount;
private String messagePrefix = INDEX_MSG;
+ private TraversalRateEstimator traversalRateEstimator = new SimpleRateEstimator();
public IndexingProgressReporter(IndexUpdateCallback updateCallback,
NodeTraversalCallback traversalCallback) {
@@ -76,9 +77,12 @@ public class IndexingProgressReporter im
public void traversedNode(PathSource pathSource) throws CommitFailedException {
traversalCount++;
if (++traversalCount % 10000 == 0) {
- log.info("{} Traversed #{} {}", messagePrefix, traversalCount, pathSource.getPath());
+ double rate = traversalRateEstimator.getNodesTraversedPerSecond();
+ String formattedRate = String.format("%1.2f nodes/s, %1.2f nodes/hr", rate, rate * 3600);
+ log.info("{} Traversed #{} {} [{}]", messagePrefix, traversalCount, pathSource.getPath(), formattedRate);
}
traversalCallback.traversedNode(pathSource);
+ traversalRateEstimator.traversedNode();
}
/**
@@ -146,6 +150,10 @@ public class IndexingProgressReporter im
return indexUpdateStates.values().stream().anyMatch(st -> st.updateCount > 0);
}
+ public void setTraversalRateEstimator(TraversalRateEstimator traversalRate) {
+ this.traversalRateEstimator = traversalRate;
+ }
+
private String getReport() {
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
Added: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/MetricRateEstimator.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/MetricRateEstimator.java?rev=1796042&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/MetricRateEstimator.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/MetricRateEstimator.java Wed May 24 11:09:29 2017
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.progress;
+
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistry;
+
+public class MetricRateEstimator implements TraversalRateEstimator {
+ private final Meter meter;
+
+ public MetricRateEstimator(String name, MetricRegistry registry) {
+ this.meter = registry.meter("indexer-" + name);
+ }
+
+ @Override
+ public void traversedNode() {
+ meter.mark();
+ }
+
+ @Override
+ public double getNodesTraversedPerSecond() {
+ return meter.getFiveMinuteRate();
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/MetricRateEstimator.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/SimpleRateEstimator.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/SimpleRateEstimator.java?rev=1796042&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/SimpleRateEstimator.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/SimpleRateEstimator.java Wed May 24 11:09:29 2017
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.progress;
+
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.base.Stopwatch;
+
+public class SimpleRateEstimator implements TraversalRateEstimator {
+ private final Stopwatch w = Stopwatch.createStarted();
+ private long count;
+
+ @Override
+ public void traversedNode() {
+ count++;
+ }
+
+ @Override
+ public double getNodesTraversedPerSecond() {
+ return (double) count / w.elapsed(TimeUnit.SECONDS);
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/SimpleRateEstimator.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/TraversalRateEstimator.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/TraversalRateEstimator.java?rev=1796042&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/TraversalRateEstimator.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/TraversalRateEstimator.java Wed May 24 11:09:29 2017
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.progress;
+
+public interface TraversalRateEstimator {
+
+ void traversedNode();
+
+ double getNodesTraversedPerSecond();
+
+}
Propchange: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/TraversalRateEstimator.java
------------------------------------------------------------------------------
svn:eol-style = native