You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2017/01/04 07:29:55 UTC
lucene-solr:branch_6x: SOLR-9896: Instrument and collect metrics from
query, update, core admin and core load thread pools
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x f65dc0618 -> 084f7a060
SOLR-9896: Instrument and collect metrics from query, update, core admin and core load thread pools
(cherry picked from commit 3c963967242aed73a906b7bc17c26a4b8b07083c)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/084f7a06
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/084f7a06
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/084f7a06
Branch: refs/heads/branch_6x
Commit: 084f7a0604da846c3ccc2c45d560d93403c883e5
Parents: f65dc06
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Wed Jan 4 12:59:47 2017 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Wed Jan 4 12:59:47 2017 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 4 +-
.../org/apache/solr/core/CoreContainer.java | 15 +++++--
.../solr/handler/admin/CoreAdminHandler.java | 13 +++++-
.../component/HttpShardHandlerFactory.java | 45 +++++++++++---------
.../apache/solr/update/UpdateShardHandler.java | 9 +++-
.../org/apache/solr/util/stats/MetricUtils.java | 9 ++++
6 files changed, 69 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/084f7a06/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index df4b911..346aeba 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -121,7 +121,7 @@ New Features
* SOLR-9884: Add version to segments handler output (Steven Bower via Erick Erickson)
* SOLR-9725: Substitute properties into JdbcDataSource configuration ( Jamie Jackson, Yuri Sashevsky via Mikhail Khludnev)
-
+
* SOLR-4735: Improve metrics reporting. This uses the dropwizard metric library, adding an internal API
for registering and reporting metrics from Solr components. Several new metrics and an improved JMX
reporter have been added (Alan Woodward, Jeff Wartes, Christine Poerschke, Kelvin Wong, shalin, ab)
@@ -154,6 +154,8 @@ New Features
* SOLR-9877: Use instrumented http client and connection pool. (shalin)
+* SOLR-9896: Instrument and collect metrics from query, update, core admin and core load thread pools. (shalin)
+
Optimizations
----------------------
* SOLR-9704: Facet Module / JSON Facet API: Optimize blockChildren facets that have
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/084f7a06/solr/core/src/java/org/apache/solr/core/CoreContainer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 7294679..4e87d98 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -72,6 +72,7 @@ import org.apache.solr.security.SecurityPluginHolder;
import org.apache.solr.update.SolrCoreState;
import org.apache.solr.update.UpdateShardHandler;
import org.apache.solr.util.DefaultSolrThreadFactory;
+import org.apache.solr.util.stats.MetricUtils;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -440,6 +441,11 @@ public class CoreContainer {
metricManager = new SolrMetricManager();
+ coreContainerWorkExecutor = MetricUtils.instrumentedExecutorService(
+ coreContainerWorkExecutor,
+ metricManager.registry(SolrMetricManager.getRegistryName(SolrInfoMBean.Group.node)),
+ SolrMetricManager.mkName("coreContainerWorkExecutor", "threadPool"));
+
shardHandlerFactory = ShardHandlerFactory.newInstance(cfg.getShardHandlerFactoryPluginInfo(), loader);
if (shardHandlerFactory instanceof SolrMetricProducer) {
SolrMetricProducer metricProducer = (SolrMetricProducer) shardHandlerFactory;
@@ -502,9 +508,12 @@ public class CoreContainer {
unloadedCores, true, "unloaded", "cores");
// setup executor to load cores in parallel
- ExecutorService coreLoadExecutor = ExecutorUtil.newMDCAwareFixedThreadPool(
- cfg.getCoreLoadThreadCount(isZooKeeperAware()),
- new DefaultSolrThreadFactory("coreLoadExecutor") );
+ ExecutorService coreLoadExecutor = MetricUtils.instrumentedExecutorService(
+ ExecutorUtil.newMDCAwareFixedThreadPool(
+ cfg.getCoreLoadThreadCount(isZooKeeperAware()),
+ new DefaultSolrThreadFactory("coreLoadExecutor")),
+ metricManager.registry(SolrMetricManager.getRegistryName(SolrInfoMBean.Group.node)),
+ SolrMetricManager.mkName("coreLoadExecutor", "threadPool"));
final List<Future<SolrCore>> futures = new ArrayList<>();
try {
List<CoreDescriptor> cds = coresLocator.discover(this);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/084f7a06/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
index 7b0ecfb..458b7a5 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
@@ -18,6 +18,7 @@ package org.apache.solr.handler.admin;
import java.io.File;
import java.lang.invoke.MethodHandles;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
@@ -42,11 +43,13 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.security.AuthorizationContext;
import org.apache.solr.security.PermissionNameProvider;
import org.apache.solr.util.DefaultSolrThreadFactory;
+import org.apache.solr.util.stats.MetricUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
@@ -65,7 +68,7 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNa
protected final CoreContainer coreContainer;
protected final Map<String, Map<String, TaskObject>> requestStatusMap;
- protected final ExecutorService parallelExecutor = ExecutorUtil.newMDCAwareFixedThreadPool(50,
+ protected ExecutorService parallelExecutor = ExecutorUtil.newMDCAwareFixedThreadPool(50,
new DefaultSolrThreadFactory("parallelCoreAdminExecutor"));
protected static int MAX_TRACKED_REQUESTS = 100;
@@ -111,6 +114,14 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNa
"it is a special Handler configured directly by the RequestDispatcher");
}
+ @Override
+ public Collection<String> initializeMetrics(SolrMetricManager manager, String registryName, String scope) {
+ Collection<String> metrics = super.initializeMetrics(manager, registryName, scope);
+ parallelExecutor = MetricUtils.instrumentedExecutorService(parallelExecutor, manager.registry(registryName),
+ SolrMetricManager.mkName("parallelCoreAdminExecutor", getCategory().name(),scope, "threadPool"));
+ return metrics;
+ }
+
/**
* The instance of CoreContainer this handler handles. This should be the CoreContainer instance that created this
* handler.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/084f7a06/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
index ff50cbc..1f1b49d 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
@@ -15,6 +15,24 @@
* limitations under the License.
*/
package org.apache.solr.handler.component;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.TimeUnit;
+
import org.apache.commons.lang.StringUtils;
import org.apache.http.client.HttpClient;
import org.apache.http.impl.client.CloseableHttpClient;
@@ -41,32 +59,16 @@ import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer;
+import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.update.UpdateShardHandler;
import org.apache.solr.update.UpdateShardHandlerConfig;
-import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.util.DefaultSolrThreadFactory;
import org.apache.solr.util.stats.InstrumentedHttpClient;
import org.apache.solr.util.stats.InstrumentedPoolingClientConnectionManager;
+import org.apache.solr.util.stats.MetricUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.apache.solr.util.plugin.PluginInfoInitialized, SolrMetricProducer {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -78,7 +80,7 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
//
// Consider CallerRuns policy and a lower max threads to throttle
// requests at some point (or should we simply return failure?)
- private ThreadPoolExecutor commExecutor = new ExecutorUtil.MDCAwareThreadPoolExecutor(
+ private ExecutorService commExecutor = new ExecutorUtil.MDCAwareThreadPoolExecutor(
0,
Integer.MAX_VALUE,
5, TimeUnit.SECONDS, // terminate idle threads after 5 sec
@@ -234,7 +236,7 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
configurer.configure((DefaultHttpClient)this.defaultClient, getClientParams());
}
- protected ThreadPoolExecutor getThreadPoolExecutor(){
+ protected ExecutorService getThreadPoolExecutor() {
return this.commExecutor;
}
@@ -424,6 +426,9 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
SolrMetricProducer solrMetricProducer = (SolrMetricProducer) defaultClient;
metricNames.addAll(solrMetricProducer.initializeMetrics(manager, registry, scope));
}
+ commExecutor = MetricUtils.instrumentedExecutorService(commExecutor,
+ manager.registry(registry),
+ SolrMetricManager.mkName("httpShardExecutor", scope, "threadPool"));
return metricNames;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/084f7a06/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java b/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
index a1725b3..77ce3f9 100644
--- a/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
+++ b/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
@@ -25,6 +25,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
+import com.codahale.metrics.InstrumentedExecutorService;
import org.apache.http.client.HttpClient;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.impl.client.CloseableHttpClient;
@@ -129,12 +130,18 @@ public class UpdateShardHandler implements SolrMetricProducer {
SolrMetricProducer solrMetricProducer = (SolrMetricProducer) client;
metricNames.addAll(solrMetricProducer.initializeMetrics(manager, registry, scope));
}
+ updateExecutor = new InstrumentedExecutorService(updateExecutor,
+ manager.registry(registry),
+ SolrMetricManager.mkName("updateExecutor", scope, "threadPool"));
+ recoveryExecutor = new InstrumentedExecutorService(recoveryExecutor,
+ manager.registry(registry),
+ SolrMetricManager.mkName("recoveryExecutor", scope, "threadPool"));
return metricNames;
}
@Override
public String getDescription() {
- return "Metrics tracked by UpdateShardHandler for distributed updates and recovery";
+ return "Metrics tracked by UpdateShardHandler related to distributed updates and recovery";
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/084f7a06/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java
index e2f7e9e..4a83c86 100644
--- a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java
+++ b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java
@@ -19,11 +19,13 @@ package org.apache.solr.util.stats;
import java.util.List;
import java.util.Map;
import java.util.SortedSet;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Histogram;
+import com.codahale.metrics.InstrumentedExecutorService;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Metric;
import com.codahale.metrics.MetricFilter;
@@ -184,4 +186,11 @@ public class MetricUtils {
response.add("count", counter.getCount());
return response;
}
+
+ /**
+ * Returns an instrumented wrapper over the given executor service.
+ */
+ public static ExecutorService instrumentedExecutorService(ExecutorService delegate, MetricRegistry metricRegistry, String scope) {
+ return new InstrumentedExecutorService(delegate, metricRegistry, scope);
+ }
}