You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ds...@apache.org on 2022/03/12 18:31:54 UTC
[solr] branch branch_9_0 updated: SOLR-14401: Track distrib/shard metrics differently (#657)
This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch branch_9_0
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9_0 by this push:
new 8c622db SOLR-14401: Track distrib/shard metrics differently (#657)
8c622db is described below
commit 8c622dbaa901f4b2ea3ffa88e6a7ded93bad367b
Author: David Smiley <ds...@salesforce.com>
AuthorDate: Thu Mar 10 17:41:26 2022 -0500
SOLR-14401: Track distrib/shard metrics differently (#657)
* only do for SearchHandler, not all request handlers (less metrics overall)
* track all the same details at the shard level as request (more detailed metrics)
* use [shard] suffix; do away with .distrib. and .local.
* don't limit this to SolrCloud
Prometheus Exporter & Grafana config:
* remove select ".distrib."; this is the default semantic
* remove ".local." additions because these are already expressed via separate request handlers with a suffix
* time_seconds_total computed differently; looks suspicious
* extract an "internal" Prometheus label from the handler; has values "shard" or "false". Updated Grafana to use this to match former logic.
Misc:
* prometheus gradle: fix "run" task
* fix README link
Co-authored-by: Houston Putman <ho...@apache.org>
---
solr/CHANGES.txt | 7 ++
.../apache/solr/handler/RequestHandlerBase.java | 120 +++++++++------------
.../solr/handler/component/SearchHandler.java | 47 +++++++-
.../java/org/apache/solr/util/SolrPluginUtils.java | 32 ++----
.../solr/handler/admin/MetricsHandlerTest.java | 13 +--
solr/prometheus-exporter/README.md | 2 +-
solr/prometheus-exporter/build.gradle | 1 +
.../conf/grafana-solr-dashboard.json | 20 ++--
.../conf/solr-exporter-config.xml | 66 ++++--------
.../pages/performance-statistics-reference.adoc | 16 +--
.../pages/major-changes-in-solr-9.adoc | 6 ++
11 files changed, 161 insertions(+), 169 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 04df4c8..c7992e3 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -250,6 +250,13 @@ when told to. The admin UI now tells it to. (Nazerke Seidan, David Smiley)
* SOLR-15982: Add end time value to backup response, standardize backup response key names and date formats (Artem Abeleshev, Christine Poerschke, Houston Putman)
+* SOLR-14401: Metrics: Only SearchHandler and subclasses have "local" metrics now.
+ It's now tracked as if it's another handler with a "[shard]" suffix, e.g. "/select[shard]".
+ There are no longer ".distrib." named metrics; all metrics are assumed to be such except
+ "[shard]". The default Prometheus exporter config splits that component to a new label
+ named "internal". The sample Grafana dashboard now filters to include or exclude this.
+ (David Smiley)
+
* SOLR-16088: De-couple Http2SolrClient and ContentStreamBase from org.apache.http (janhoy)
Build
diff --git a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
index ddc481e..300230e 100644
--- a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
+++ b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
@@ -24,21 +24,18 @@ import com.codahale.metrics.Timer;
import com.google.common.collect.ImmutableList;
import java.lang.invoke.MethodHandles;
import java.util.Collection;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
import org.apache.solr.api.Api;
import org.apache.solr.api.ApiBag;
import org.apache.solr.api.ApiSupport;
import org.apache.solr.common.SolrException;
-import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SuppressForbidden;
+import org.apache.solr.core.MetricsConfig;
import org.apache.solr.core.PluginBag;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrInfoBean;
-import org.apache.solr.metrics.MetricsMap;
+import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
@@ -64,28 +61,14 @@ public abstract class RequestHandlerBase
protected SolrParams invariants;
protected boolean httpCaching = true;
- // Statistics
- private Meter numErrors = new Meter();
- private Meter numServerErrors = new Meter();
- private Meter numClientErrors = new Meter();
- private Meter numTimeouts = new Meter();
- private Counter requests = new Counter();
- private final Map<String, Counter> shardPurposes = new ConcurrentHashMap<>();
- private Timer requestTimes = new Timer();
- private Timer distribRequestTimes = new Timer();
- private Timer localRequestTimes = new Timer();
- private Counter totalTime = new Counter();
- private Counter distribTotalTime = new Counter();
- private Counter localTotalTime = new Counter();
-
+ protected SolrMetricsContext solrMetricsContext;
+ protected HandlerMetrics metrics = HandlerMetrics.NO_OP;
private final long handlerStart;
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private PluginInfo pluginInfo;
- protected SolrMetricsContext solrMetricsContext;
-
@SuppressForbidden(reason = "Need currentTimeMillis, used only for stats output")
public RequestHandlerBase() {
handlerStart = System.currentTimeMillis();
@@ -159,28 +142,40 @@ public abstract class RequestHandlerBase
@Override
public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
this.solrMetricsContext = parentContext.getChildContext(this);
- numErrors = solrMetricsContext.meter("errors", getCategory().toString(), scope);
- numServerErrors = solrMetricsContext.meter("serverErrors", getCategory().toString(), scope);
- numClientErrors = solrMetricsContext.meter("clientErrors", getCategory().toString(), scope);
- numTimeouts = solrMetricsContext.meter("timeouts", getCategory().toString(), scope);
- requests = solrMetricsContext.counter("requests", getCategory().toString(), scope);
- MetricsMap metricsMap =
- new MetricsMap(map -> shardPurposes.forEach((k, v) -> map.putNoEx(k, v.getCount())));
- solrMetricsContext.gauge(metricsMap, true, "shardRequests", getCategory().toString(), scope);
- requestTimes = solrMetricsContext.timer("requestTimes", getCategory().toString(), scope);
- distribRequestTimes =
- solrMetricsContext.timer("requestTimes", getCategory().toString(), scope, "distrib");
- localRequestTimes =
- solrMetricsContext.timer("requestTimes", getCategory().toString(), scope, "local");
- totalTime = solrMetricsContext.counter("totalTime", getCategory().toString(), scope);
- distribTotalTime =
- solrMetricsContext.counter("totalTime", getCategory().toString(), scope, "distrib");
- localTotalTime =
- solrMetricsContext.counter("totalTime", getCategory().toString(), scope, "local");
+ metrics = new HandlerMetrics(solrMetricsContext, getCategory().toString(), scope);
solrMetricsContext.gauge(
() -> handlerStart, true, "handlerStart", getCategory().toString(), scope);
}
+ /** Metrics for this handler. */
+ public static class HandlerMetrics {
+ public static final HandlerMetrics NO_OP =
+ new HandlerMetrics(
+ new SolrMetricsContext(
+ new SolrMetricManager(
+ null, new MetricsConfig.MetricsConfigBuilder().setEnabled(false).build()),
+ "NO_OP",
+ "NO_OP"));
+
+ private final Meter numErrors;
+ private final Meter numServerErrors;
+ private final Meter numClientErrors;
+ private final Meter numTimeouts;
+ private final Counter requests;
+ private final Timer requestTimes;
+ private final Counter totalTime;
+
+ public HandlerMetrics(SolrMetricsContext solrMetricsContext, String... metricPath) {
+ numErrors = solrMetricsContext.meter("errors", metricPath);
+ numServerErrors = solrMetricsContext.meter("serverErrors", metricPath);
+ numClientErrors = solrMetricsContext.meter("clientErrors", metricPath);
+ numTimeouts = solrMetricsContext.meter("timeouts", metricPath);
+ requests = solrMetricsContext.counter("requests", metricPath);
+ requestTimes = solrMetricsContext.timer("requestTimes", metricPath);
+ totalTime = solrMetricsContext.counter("totalTime", metricPath);
+ }
+ }
+
public static SolrParams getSolrParamsFromNamedList(NamedList<?> args, String key) {
Object o = args.get(key);
if (o != null && o instanceof NamedList) {
@@ -198,28 +193,10 @@ public abstract class RequestHandlerBase
@Override
public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) {
- requests.inc();
- // requests are distributed by default when ZK is in use, unless indicated otherwise
- boolean distrib =
- req.getParams()
- .getBool(
- CommonParams.DISTRIB,
- req.getCore() != null
- ? req.getCore().getCoreContainer().isZooKeeperAware()
- : false);
- if (req.getParams().getBool(ShardParams.IS_SHARD, false)) {
- shardPurposes.computeIfAbsent("total", name -> new Counter()).inc();
- int purpose = req.getParams().getInt(ShardParams.SHARDS_PURPOSE, 0);
- if (purpose != 0) {
- String[] names = SolrPluginUtils.getRequestPurposeNames(purpose);
- for (String n : names) {
- shardPurposes.computeIfAbsent(n, name -> new Counter()).inc();
- }
- }
- }
- Timer.Context timer = requestTimes.time();
- @SuppressWarnings("resource")
- Timer.Context dTimer = distrib ? distribRequestTimes.time() : localRequestTimes.time();
+ HandlerMetrics metrics = getMetricsForThisRequest(req);
+ metrics.requests.inc();
+
+ Timer.Context timer = metrics.requestTimes.time();
try {
TestInjection.injectLeaderTragedy(req.getCore());
if (pluginInfo != null && pluginInfo.attributes.containsKey(USEPARAM))
@@ -233,7 +210,7 @@ public abstract class RequestHandlerBase
if (header != null) {
if (Boolean.TRUE.equals(
header.getBooleanArg(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY))) {
- numTimeouts.mark();
+ metrics.numTimeouts.mark();
rsp.setHttpCaching(false);
}
}
@@ -271,25 +248,24 @@ public abstract class RequestHandlerBase
if (incrementErrors) {
SolrException.log(log, e);
- numErrors.mark();
+ metrics.numErrors.mark();
if (isServerError) {
- numServerErrors.mark();
+ metrics.numServerErrors.mark();
} else {
- numClientErrors.mark();
+ metrics.numClientErrors.mark();
}
}
} finally {
- dTimer.stop();
long elapsed = timer.stop();
- totalTime.inc(elapsed);
- if (distrib) {
- distribTotalTime.inc(elapsed);
- } else {
- localTotalTime.inc(elapsed);
- }
+ metrics.totalTime.inc(elapsed);
}
}
+ /** The metrics to be used for this request. */
+ protected HandlerMetrics getMetricsForThisRequest(SolrQueryRequest req) {
+ return this.metrics;
+ }
+
//////////////////////// SolrInfoMBeans methods //////////////////////
@Override
diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
index 1423787..fca952f 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
@@ -16,12 +16,23 @@
*/
package org.apache.solr.handler.component;
-import static org.apache.solr.common.params.CommonParams.*;
+import static org.apache.solr.common.params.CommonParams.DISTRIB;
+import static org.apache.solr.common.params.CommonParams.FAILURE;
+import static org.apache.solr.common.params.CommonParams.PATH;
+import static org.apache.solr.common.params.CommonParams.STATUS;
+import com.codahale.metrics.Counter;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.lang.invoke.MethodHandles;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.lang3.StringUtils;
import org.apache.lucene.index.ExitableDirectoryReader;
@@ -41,6 +52,8 @@ import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.metrics.MetricsMap;
+import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.pkg.PackageAPI;
import org.apache.solr.pkg.PackageListeners;
import org.apache.solr.pkg.PackageLoader;
@@ -69,11 +82,16 @@ public class SearchHandler extends RequestHandlerBase
static final String INIT_FIRST_COMPONENTS = "first-components";
static final String INIT_LAST_COMPONENTS = "last-components";
+ protected static final String SHARD_HANDLER_SUFFIX = "[shard]";
+
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
/** A counter to ensure that no RID is equal, even if they fall in the same millisecond */
private static final AtomicLong ridCounter = new AtomicLong();
+ private HandlerMetrics metricsShard = HandlerMetrics.NO_OP;
+ private final Map<String, Counter> shardPurposes = new ConcurrentHashMap<>();
+
protected volatile List<SearchComponent> components;
private ShardHandlerFactory shardHandlerFactory;
private PluginInfo shfInfo;
@@ -106,6 +124,25 @@ public class SearchHandler extends RequestHandlerBase
}
@Override
+ public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
+ super.initializeMetrics(parentContext, scope);
+ metricsShard =
+ new HandlerMetrics( // will register various metrics in the context
+ solrMetricsContext, getCategory().toString(), scope + SHARD_HANDLER_SUFFIX);
+ solrMetricsContext.gauge(
+ new MetricsMap(map -> shardPurposes.forEach((k, v) -> map.putNoEx(k, v.getCount()))),
+ true,
+ "purposes",
+ getCategory().toString(),
+ scope + SHARD_HANDLER_SUFFIX);
+ }
+
+ @Override
+ protected HandlerMetrics getMetricsForThisRequest(SolrQueryRequest req) {
+ return req.getParams().getBool(ShardParams.IS_SHARD, false) ? this.metricsShard : this.metrics;
+ }
+
+ @Override
public PermissionNameProvider.Name getPermissionName(AuthorizationContext ctx) {
return PermissionNameProvider.Name.READ_PERM;
}
@@ -290,6 +327,12 @@ public class SearchHandler extends RequestHandlerBase
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
+ if (req.getParams().getBool(ShardParams.IS_SHARD, false)) {
+ int purpose = req.getParams().getInt(ShardParams.SHARDS_PURPOSE, 0);
+ SolrPluginUtils.forEachRequestPurpose(
+ purpose, n -> shardPurposes.computeIfAbsent(n, name -> new Counter()).inc());
+ }
+
List<SearchComponent> components = getComponents();
ResponseBuilder rb = newResponseBuilder(req, rsp, components);
if (rb.requestInfo != null) {
diff --git a/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java b/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java
index 7b70833..88d3044 100644
--- a/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java
+++ b/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java
@@ -39,6 +39,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
+import java.util.function.Consumer;
import java.util.regex.Pattern;
import org.apache.lucene.document.Document;
import org.apache.lucene.search.BooleanClause;
@@ -1023,39 +1024,26 @@ public class SolrPluginUtils {
return UNKNOWN_VALUE;
}
- private static final String[] purposeUnknown = new String[] {UNKNOWN_VALUE};
-
/**
* Given the integer purpose of a request generates a readable value corresponding the request
* purposes (there can be more than one on a single request). If there is a purpose parameter
- * present that's not known this method will return a 1-element array containing {@value
- * #UNKNOWN_VALUE}
+ * present that's not known this method will invoke the consumer with {@value #UNKNOWN_VALUE}.
*
* @param reqPurpose Numeric request purpose
- * @return an array of purpose names.
+ * @param consumer recipient of a string
*/
- public static String[] getRequestPurposeNames(Integer reqPurpose) {
+ public static void forEachRequestPurpose(Integer reqPurpose, Consumer<String> consumer) {
+ boolean sendUnknown = true;
if (reqPurpose != null) {
- int valid = 0;
for (Map.Entry<Integer, String> entry : purposes.entrySet()) {
if ((reqPurpose & entry.getKey()) != 0) {
- valid++;
- }
- }
- if (valid == 0) {
- return purposeUnknown;
- } else {
- String[] result = new String[valid];
- int i = 0;
- for (Map.Entry<Integer, String> entry : purposes.entrySet()) {
- if ((reqPurpose & entry.getKey()) != 0) {
- result[i] = entry.getValue();
- i++;
- }
+ consumer.accept(entry.getValue());
+ sendUnknown = false;
}
- return result;
}
}
- return purposeUnknown;
+ if (sendUnknown) {
+ consumer.accept(UNKNOWN_VALUE);
+ }
}
}
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
index d8d60b6..1289af9 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
@@ -20,7 +20,6 @@ package org.apache.solr.handler.admin;
import com.codahale.metrics.Counter;
import java.util.Arrays;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.MapWriter;
@@ -640,7 +639,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
// test multiple expressions producing overlapping metrics - should be no dupes
- // this key matches also sub-metrics of /select, eg. /select.distrib, /select.local, ...
+ // this key matches also sub-metrics of /select, eg. /select[shard], ...
String key3 = "solr\\.core\\..*:.*/select.*\\.requestTimes:count";
resp = new SolrQueryResponse();
// ORDER OF PARAMS MATTERS HERE! see the refguide
@@ -662,19 +661,15 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
// for requestTimes only the full set of values from the first expr should be present
assertNotNull(val);
SimpleOrderedMap<Object> values = (SimpleOrderedMap<Object>) val;
- assertEquals(values.jsonStr(), 4, values.size());
- List<Object> multipleVals = values.getAll("QUERY./select.requestTimes");
- assertEquals(multipleVals.toString(), 1, multipleVals.size());
- v = values.get("QUERY./select.local.requestTimes");
+ assertEquals(values.jsonStr(), 3, values.size());
+ v = values.get("QUERY./select.requestTimes");
assertTrue(v instanceof MapWriter);
((MapWriter) v).toMap(map);
- assertEquals(map.toString(), 1, map.size());
assertTrue(map.toString(), map.containsKey("count"));
map.clear();
- v = values.get("QUERY./select.distrib.requestTimes");
+ v = values.get("QUERY./select[shard].requestTimes");
assertTrue(v instanceof MapWriter);
((MapWriter) v).toMap(map);
- assertEquals(map.toString(), 1, map.size());
assertTrue(map.toString(), map.containsKey("count"));
}
diff --git a/solr/prometheus-exporter/README.md b/solr/prometheus-exporter/README.md
index 45e1240..cda5c4e 100644
--- a/solr/prometheus-exporter/README.md
+++ b/solr/prometheus-exporter/README.md
@@ -6,7 +6,7 @@ Apache Solr Prometheus Exporter (solr-exporter) provides a way for you to expose
# Getting Started With Solr Prometheus Exporter
For information on how to get started with solr-exporter please see:
- * [Solr Reference Guide's section on Monitoring Solr with Prometheus and Grafana](https://solr.apache.org/guide/monitoring-with-prometheus-and-grafana.html)
+ * [Solr Reference Guide's section on Monitoring Solr with Prometheus and Grafana](https://solr.apache.org/guide/monitoring-solr-with-prometheus-and-grafana.html)
# Docker
diff --git a/solr/prometheus-exporter/build.gradle b/solr/prometheus-exporter/build.gradle
index 739a1ac..1f8faf0 100644
--- a/solr/prometheus-exporter/build.gradle
+++ b/solr/prometheus-exporter/build.gradle
@@ -61,6 +61,7 @@ task run(type: JavaExec) {
main = project.ext.mainClass
classpath = sourceSets.main.runtimeClasspath
systemProperties = ["log4j.configurationFile":"file:conf/log4j2.xml"]
+ args = ["-f", "conf/solr-exporter-config.xml"]
}
jar {
diff --git a/solr/prometheus-exporter/conf/grafana-solr-dashboard.json b/solr/prometheus-exporter/conf/grafana-solr-dashboard.json
index 75065a9..172a6e5 100644
--- a/solr/prometheus-exporter/conf/grafana-solr-dashboard.json
+++ b/solr/prometheus-exporter/conf/grafana-solr-dashboard.json
@@ -775,7 +775,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "sum by (collection) (solr_metrics_core_query_1minRate{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"})",
+ "expr": "sum by (collection) (solr_metrics_core_query_1minRate{internal=\"false\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"})",
"interval": "",
"legendFormat": "{{collection}}",
"refId": "A"
@@ -886,7 +886,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "sum by (base_url) (solr_metrics_core_query_1minRate{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"})",
+ "expr": "sum by (base_url) (solr_metrics_core_query_1minRate{internal=\"false\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"})",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{base_url}}",
@@ -994,7 +994,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_core_query_p95_ms{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
+ "expr": "solr_metrics_core_query_p95_ms{internal=\"false\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -1095,7 +1095,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_core_query_5minRate{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
+ "expr": "solr_metrics_core_query_5minRate{internal=\"false\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -1196,7 +1196,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_core_query_p99_ms{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
+ "expr": "solr_metrics_core_query_p99_ms{internal=\"false\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -1297,7 +1297,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_core_query_1minRate{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
+ "expr": "solr_metrics_core_query_1minRate{internal=\"false\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
"format": "time_series",
"instant": false,
"interval": "",
@@ -1399,7 +1399,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_core_query_p75_ms{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
+ "expr": "solr_metrics_core_query_p75_ms{internal=\"false\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -1501,7 +1501,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_core_query_local_count{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}[1m])",
+ "expr": "increase(solr_metrics_core_query_local_count{internal!=\"false\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}[1m])",
"format": "time_series",
"instant": false,
"interval": "",
@@ -1603,7 +1603,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_core_query_local_1minRate{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
+ "expr": "solr_metrics_core_query_local_1minRate{internal!=\"false\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
"format": "time_series",
"instant": false,
"interval": "",
@@ -1705,7 +1705,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_core_query_local_p95_ms{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
+ "expr": "solr_metrics_core_query_local_p95_ms{internal!=\"false\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 3,
diff --git a/solr/prometheus-exporter/conf/solr-exporter-config.xml b/solr/prometheus-exporter/conf/solr-exporter-config.xml
index bf8e491..8e03b9f 100644
--- a/solr/prometheus-exporter/conf/solr-exporter-config.xml
+++ b/solr/prometheus-exporter/conf/solr-exporter-config.xml
@@ -44,7 +44,7 @@
If a template reference omits the metric, then the unique suffix is used, for instance:
- $jq:core-query(1minRate, endswith(".distrib.requestTimes"))
+ $jq:core-query(1minRate, endswith(".requestTimes"))
Creates a GAUGE metric (default type) named "solr_metrics_core_query_1minRate" using the 1minRate value from the selected JSON object.
@@ -62,7 +62,7 @@
(if $parent_key_item_len == 5 then ($collection + "_" + $shard + "_" + $replica) else $core end) as $core |
$parent.value | to_entries | .[] | {KEYSELECTOR} | select (.value | type == "object") as $object |
$object.key | split(".")[0] as $category |
- $object.key | split(".")[1] as $handler |
+ $object.key | split(".")[1] | rtrimstr("]") | split("[") | .[0] as $handler | .[1] // "false" as $internal |
select($category | startswith("QUERY")) |
select($handler | startswith("/")) |
{METRIC} as $value |
@@ -71,8 +71,8 @@
name: "solr_metrics_core_query_{UNIQUE}",
type: "{TYPE}",
help: "See: https://solr.apache.org/guide/performance-statistics-reference.html",
- label_names: ["category", "searchHandler", "core"],
- label_values: [$category, $handler, $core],
+ label_names: ["category", "searchHandler", "internal", "core"],
+ label_values: [$category, $handler, $internal, $core],
value: $value
}
else
@@ -80,8 +80,8 @@
name: "solr_metrics_core_query_{UNIQUE}",
type: "{TYPE}",
help: "See: https://solr.apache.org/guide/performance-statistics-reference.html",
- label_names: ["category", "searchHandler", "core", "collection", "shard", "replica"],
- label_values: [$category, $handler, $core, $collection, $shard, $replica],
+ label_names: ["category", "searchHandler", "internal", "core", "collection", "shard", "replica"],
+ label_values: [$category, $handler, $internal, $core, $collection, $shard, $replica],
value: $value
}
end
@@ -97,7 +97,7 @@
(if $parent_key_item_len == 5 then ($collection + "_" + $shard + "_" + $replica) else $core end) as $core |
$parent.value | to_entries | .[] | {KEYSELECTOR} as $object |
$object.key | split(".")[0] as $category |
- $object.key | split(".")[1] as $handler |
+ $object.key | split(".")[1] | rtrimstr("]") | split("[") | .[0] as $handler | .[1] // "false" as $internal |
select($handler | startswith("/")) |
{METRIC} as $value |
if $parent_key_item_len == 3 then
@@ -105,8 +105,8 @@
name: "solr_metrics_core_{UNIQUE}",
type: "{TYPE}",
help: "See following URL: https://solr.apache.org/guide/metrics-reporting.html",
- label_names: ["category", "handler", "core"],
- label_values: [$category, $handler, $core],
+ label_names: ["category", "handler", "internal", "core"],
+ label_values: [$category, $handler, $internal, $core],
value: $value
}
else
@@ -114,8 +114,8 @@
name: "solr_metrics_core_{UNIQUE}",
type: "{TYPE}",
help: "See following URL: https://solr.apache.org/guide/metrics-reporting.html",
- label_names: ["category", "handler", "core", "collection", "shard", "replica"],
- label_values: [$category, $handler, $core, $collection, $shard, $replica],
+ label_names: ["category", "handler", "internal", "core", "collection", "shard", "replica"],
+ label_values: [$category, $handler, $internal, $core, $collection, $shard, $replica],
value: $value
}
end
@@ -496,7 +496,7 @@
$jq:node(errors_total, select(.key | endswith(".errors")), count)
</str>
<str>
- $jq:node(requests_total, select(.key | endswith(".local.requestTimes")), count)
+ $jq:node(requests_total, select(.key | endswith(".requestTimes")), count)
</str>
<str>
$jq:node(server_errors_total, select(.key | endswith(".serverErrors")), count)
@@ -505,7 +505,7 @@
$jq:node(timeouts_total, select(.key | endswith(".timeouts")), count)
</str>
<str>
- $jq:node(time_seconds_total, select(.key | endswith(".local.totalTime")), ($object.value / 1000))
+ $jq:node(time_seconds_total, select(.key | endswith(".totalTime")), ($object.value / 1000))
</str>
<str>
.metrics["solr.node"] | to_entries | .[] | select(.key | startswith("CONTAINER.cores.")) as $object |
@@ -574,51 +574,25 @@
$jq:core-query(client_errors_1minRate, select(.key | endswith(".clientErrors")), 1minRate)
</str>
<str>
- $jq:core-query(1minRate, select(.key | endswith(".distrib.requestTimes")), 1minRate)
- </str>
- <str>
- $jq:core-query(5minRate, select(.key | endswith(".distrib.requestTimes")), 5minRate)
- </str>
- <str>
- $jq:core-query(median_ms, select(.key | endswith(".distrib.requestTimes")), median_ms)
- </str>
- <str>
- $jq:core-query(p75_ms, select(.key | endswith(".distrib.requestTimes")), p75_ms)
- </str>
- <str>
- $jq:core-query(p95_ms, select(.key | endswith(".distrib.requestTimes")), p95_ms)
- </str>
- <str>
- $jq:core-query(p99_ms, select(.key | endswith(".distrib.requestTimes")), p99_ms)
- </str>
- <str>
- $jq:core-query(mean_rate, select(.key | endswith(".distrib.requestTimes")), meanRate)
- </str>
-
- <!-- Local (non-distrib) query metrics -->
- <str>
- $jq:core-query(local_1minRate, select(.key | endswith(".local.requestTimes")), 1minRate)
- </str>
- <str>
- $jq:core-query(local_5minRate, select(.key | endswith(".local.requestTimes")), 5minRate)
+ $jq:core-query(1minRate, select(.key | endswith(".requestTimes")), 1minRate)
</str>
<str>
- $jq:core-query(local_median_ms, select(.key | endswith(".local.requestTimes")), median_ms)
+ $jq:core-query(5minRate, select(.key | endswith(".requestTimes")), 5minRate)
</str>
<str>
- $jq:core-query(local_p75_ms, select(.key | endswith(".local.requestTimes")), p75_ms)
+ $jq:core-query(median_ms, select(.key | endswith(".requestTimes")), median_ms)
</str>
<str>
- $jq:core-query(local_p95_ms, select(.key | endswith(".local.requestTimes")), p95_ms)
+ $jq:core-query(p75_ms, select(.key | endswith(".requestTimes")), p75_ms)
</str>
<str>
- $jq:core-query(local_p99_ms, select(.key | endswith(".local.requestTimes")), p99_ms)
+ $jq:core-query(p95_ms, select(.key | endswith(".requestTimes")), p95_ms)
</str>
<str>
- $jq:core-query(local_mean_rate, select(.key | endswith(".local.requestTimes")), meanRate)
+ $jq:core-query(p99_ms, select(.key | endswith(".requestTimes")), p99_ms)
</str>
<str>
- $jq:core-query(local_count, select(.key | endswith(".local.requestTimes")), count, COUNTER)
+ $jq:core-query(mean_rate, select(.key | endswith(".requestTimes")), meanRate)
</str>
<!-- core metrics other than query -->
diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/performance-statistics-reference.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/performance-statistics-reference.adoc
index 9f2f7f8..8d6b78b 100644
--- a/solr/solr-ref-guide/modules/deployment-guide/pages/performance-statistics-reference.adoc
+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/performance-statistics-reference.adoc
@@ -100,15 +100,17 @@ The table below shows the metric names and attributes to request:
`UPDATE./update.handlerStart` |Epoch time when the handler was registered.
|===
-*Distributed vs. Local Request Times*
+*Differentiating Internal Requests*
-Processing of a single distributed request in SolrCloud usually requires making several requests to other nodes and other replicas.
-The common statistics listed above lump these timings together, even though they are very different in nature, thus making it difficult to measure the latency of distributed and local requests separately.
-Solr 8.4 introduced additional statistics that help to do this.
+Processing of a single request in SolrCloud for a large collection requires making additional requests to other replicas, often on other nodes.
+The internal requests look much the same on the surface (same handler), but they are performing a portion of the over-arching task.
+Differentiating these requests is really important!
+Solr tracks its metrics on these handlers with a different handler name when the request is contributing to some other request:
-These metrics are structured the same as `requestTimes` and `totalTime` metrics above but they use different full names, e.g., `QUERY./select.distrib.requestTimes` and `QUERY./select.local.requestTimes`.
-The metrics under the `distrib` path correspond to the time it takes for a (potentially) distributed request to complete all remote calls plus any local processing, and return the result to the caller.
-The metrics under the `local` path correspond to the time it takes for a local call (non-distributed, i.e., being processed only by the Solr core where the handler operates) to complete.
+* Queries: `/select` query's internal requests will be tracked as `/select[shard]`. Technically, this occurs on `SearchHandler` and its subclasses.
+* _(More can be instrumented some day)_
+
+Solr's Prometheus exporter configuration extracts this suffix on the handler to a label named "internal". When configuring Grafana or other metrics tools, be sure to filter these metrics in or out depending on what is being analyzed.
== Update Handler
diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
index b2015b5..296a5b8 100644
--- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
+++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
@@ -138,6 +138,12 @@ changes, however the module needs to be installed - see the section xref:query-g
* SOLR-15097: JWTAuthPlugin has been moved to a module. Users need to add the module to classpath. The plugin has also
changed package name to `org.apache.solr.security.jwt`, but can still be loaded as shortform `class="solr.JWTAuthPlugin"`.
+* SOLR-14401: Metrics: Only SearchHandler and subclasses have "local" metrics now.
+It's now tracked as if it's another handler with a "[shard]" suffix, e.g. "/select[shard]".
+There are no longer ".distrib." named metrics; all metrics are assumed to be such except
+"[shard]". The default Prometheus exporter config splits that component to a new label
+named "internal". The sample Grafana dashboard now filters to include or exclude this.
+
== New Features & Enhancements
* Replica placement plugins