You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2021/08/03 13:33:44 UTC
[lucene-solr] branch branch_8x updated: SOLR-15564: Improve
filtering expressions in /admin/metrics.
This is an automated email from the ASF dual-hosted git repository.
ab pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8x by this push:
new 41b12e9 SOLR-15564: Improve filtering expressions in /admin/metrics.
41b12e9 is described below
commit 41b12e9cd2f950440a7fbca99c903cd0ea29d1b2
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Tue Aug 3 15:33:00 2021 +0200
SOLR-15564: Improve filtering expressions in /admin/metrics.
---
solr/CHANGES.txt | 2 +
.../apache/solr/handler/admin/MetricsHandler.java | 97 +++++++++++++++++++++-
.../solr/handler/admin/MetricsHandlerTest.java | 73 ++++++++++++++++
solr/solr-ref-guide/src/metrics-reporting.adoc | 21 ++++-
4 files changed, 188 insertions(+), 5 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index e16d15f..94ec159 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -43,6 +43,8 @@ Improvements
* SOLR-15570: Include fields declared in the schema in table metadata (SQL) even if they are empty (Timothy Potter)
+* SOLR-15564: Improve filtering expressions in /admin/metrics. (ab)
+
Optimizations
---------------------
* SOLR-15433: Replace transient core cache LRU by Caffeine cache. (Bruno Roustant)
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
index d3b5cb9..5202371 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
@@ -21,9 +21,11 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.TreeSet;
import java.util.function.BiConsumer;
import java.util.function.Predicate;
import java.util.regex.Pattern;
@@ -66,11 +68,12 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
public static final String REGISTRY_PARAM = "registry";
public static final String GROUP_PARAM = "group";
public static final String KEY_PARAM = "key";
+ public static final String EXPR_PARAM = "expr";
public static final String TYPE_PARAM = "type";
public static final String ALL = "all";
- private static final Pattern KEY_REGEX = Pattern.compile("(?<!" + Pattern.quote("\\") + ")" + Pattern.quote(":"));
+ private static final Pattern KEY_SPLIT_REGEX = Pattern.compile("(?<!" + Pattern.quote("\\") + ")" + Pattern.quote(":"));
private final CoreContainer cc;
private final Map<String, String> injectedSysProps = CommonTestInjection.injectAdditionalProps();
private final boolean enabled;
@@ -121,6 +124,11 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
handleKeyRequest(keys, consumer);
return;
}
+ String[] exprs = params.getParams(EXPR_PARAM);
+ if (exprs != null && exprs.length > 0) {
+ handleExprRequest(exprs, consumer);
+ return;
+ }
MetricFilter mustMatchFilter = parseMustMatchFilter(params);
Predicate<CharSequence> propertyFilter = parsePropertyFilter(params);
List<MetricType> metricTypes = parseMetricTypes(params);
@@ -140,6 +148,87 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
consumer.accept("metrics", response);
}
+ private static class MetricsExpr {
+ Pattern registryRegex;
+ MetricFilter metricFilter;
+ Predicate<CharSequence> propertyFilter;
+ }
+
+ private void handleExprRequest(String[] exprs, BiConsumer<String, Object> consumer) {
+ SimpleOrderedMap<Object> result = new SimpleOrderedMap<>();
+ SimpleOrderedMap<Object> errors = new SimpleOrderedMap<>();
+ List<MetricsExpr> metricsExprs = new ArrayList<>();
+
+ for (String key : exprs) {
+ if (key == null || key.isEmpty()) {
+ continue;
+ }
+ String[] parts = KEY_SPLIT_REGEX.split(key);
+ if (parts.length < 2 || parts.length > 3) {
+ errors.add(key, "at least two and at most three colon-separated parts must be provided");
+ continue;
+ }
+ MetricsExpr me = new MetricsExpr();
+ me.registryRegex = Pattern.compile(unescape(parts[0]));
+ me.metricFilter = new SolrMetricManager.RegexFilter(unescape(parts[1]));
+ String propertyPart = parts.length > 2 ? unescape(parts[2]) : null;
+ if (propertyPart == null) {
+ me.propertyFilter = name -> true;
+ } else {
+ me.propertyFilter = new Predicate<CharSequence>() {
+ final Pattern pattern = Pattern.compile(propertyPart);
+ @Override
+ public boolean test(CharSequence charSequence) {
+ return pattern.matcher(charSequence).matches();
+ }
+ };
+ }
+ metricsExprs.add(me);
+ }
+ // find matching registries first, to avoid scanning non-matching registries
+ Set<String> matchingRegistries = new TreeSet<>();
+ metricsExprs.forEach(me -> {
+ metricManager.registryNames().forEach(name -> {
+ if (me.registryRegex.matcher(name).matches()) {
+ matchingRegistries.add(name);
+ }
+ });
+ });
+ for (String registryName : matchingRegistries) {
+ MetricRegistry registry = metricManager.registry(registryName);
+ for (MetricsExpr me : metricsExprs) {
+ @SuppressWarnings("unchecked")
+ SimpleOrderedMap<Object> perRegistryResult = (SimpleOrderedMap<Object>) result.get(registryName);
+ final SimpleOrderedMap<Object> perRegistryTemp = new SimpleOrderedMap<>();
+ // skip processing if not a matching registry
+ if (!me.registryRegex.matcher(registryName).matches()) {
+ continue;
+ }
+ MetricUtils.toMaps(registry, Collections.singletonList(MetricFilter.ALL), me.metricFilter,
+ me.propertyFilter, false, false, true, false, (k, v) -> perRegistryTemp.add(k, v));
+ // extracted some metrics and there's no entry for this registry yet
+ if (perRegistryTemp.size() > 0) {
+ if (perRegistryResult == null) { // new results for this registry
+ result.add(registryName, perRegistryTemp);
+ } else {
+ // merge if needed
+ for (Iterator<Map.Entry<String, Object>> it = perRegistryTemp.iterator(); it.hasNext(); ) {
+ Map.Entry<String, Object> entry = it.next();
+ Object existing = perRegistryResult.get(entry.getKey());
+ if (existing == null) {
+ perRegistryResult.add(entry.getKey(), entry.getValue());
+ }
+ }
+ }
+ }
+ }
+ }
+ consumer.accept("metrics", result);
+ if (errors.size() > 0) {
+ consumer.accept("errors", errors);
+ }
+ }
+
@SuppressWarnings({"unchecked", "rawtypes"})
public void handleKeyRequest(String[] keys, BiConsumer<String, Object> consumer) throws Exception {
SimpleOrderedMap<Object> result = new SimpleOrderedMap<>();
@@ -148,7 +237,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
if (key == null || key.isEmpty()) {
continue;
}
- String[] parts = KEY_REGEX.split(key);
+ String[] parts = KEY_SPLIT_REGEX.split(key);
if (parts.length < 2 || parts.length > 3) {
errors.add(key, "at least two and at most three colon-separated parts must be provided");
continue;
@@ -202,7 +291,9 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
for (int i = 0; i < s.length(); i++) {
char c = s.charAt(i);
if (c == '\\') {
- continue;
+ if (i < s.length() - 1 && s.charAt(i + 1) == ':') {
+ continue;
+ }
}
sb.append(c);
}
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
index fba1239..2915ab3 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
@@ -19,6 +19,7 @@ package org.apache.solr.handler.admin;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import com.codahale.metrics.Counter;
@@ -362,6 +363,78 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
}
@Test
+ @SuppressWarnings("unchecked")
+ public void testExprMetrics() throws Exception {
+ MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
+
+ String key1 = "solr\\.core\\..*:.*/select\\.request.*:.*Rate";
+ SolrQueryResponse resp = new SolrQueryResponse();
+ handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
+ MetricsHandler.EXPR_PARAM, key1), resp);
+ // response structure is like in the case of non-key params
+ Object val = resp.getValues().findRecursive( "metrics", "solr.core.collection1", "QUERY./select.requestTimes");
+ assertNotNull(val);
+ assertTrue(val instanceof MapWriter);
+ Map<String, Object> map = new HashMap<>();
+ ((MapWriter) val).toMap(map);
+ assertEquals(map.toString(), 4, map.size()); // mean, 1, 5, 15
+ assertNotNull(map.toString(), map.get("meanRate"));
+ assertNotNull(map.toString(), map.get("1minRate"));
+ assertNotNull(map.toString(), map.get("5minRate"));
+ assertNotNull(map.toString(), map.get("15minRate"));
+ assertEquals(map.toString(), ((Number) map.get("1minRate")).doubleValue(), 0.0, 0.0);
+ map.clear();
+
+ String key2 = "solr\\.core\\..*:.*/select\\.request.*";
+ resp = new SolrQueryResponse();
+ handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
+ MetricsHandler.EXPR_PARAM, key2), resp);
+ // response structure is like in the case of non-key params
+ val = resp.getValues().findRecursive( "metrics", "solr.core.collection1");
+ assertNotNull(val);
+ Object v = ((SimpleOrderedMap<Object>) val).get("QUERY./select.requestTimes");
+ assertNotNull(v);
+ assertTrue(v instanceof MapWriter);
+ ((MapWriter) v).toMap(map);
+ assertEquals(map.toString(), 14, map.size());
+ assertNotNull(map.toString(), map.get("1minRate"));
+ assertEquals(map.toString(), ((Number) map.get("1minRate")).doubleValue(), 0.0, 0.0);
+ map.clear();
+ // select requests counter
+ v = ((SimpleOrderedMap<Object>) val).get("QUERY./select.requests");
+ assertNotNull(v);
+ assertTrue(v instanceof Number);
+
+ // test multiple expressions producing overlapping metrics - should be no dupes
+
+ // this key matches also sub-metrics of /select, eg. /select.distrib, /select.local, ...
+ String key3 = "solr\\.core\\..*:.*/select.*\\.requestTimes:count";
+ resp = new SolrQueryResponse();
+ // ORDER OF PARAMS MATTERS HERE! see the refguide
+ handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
+ MetricsHandler.EXPR_PARAM, key2, MetricsHandler.EXPR_PARAM, key1, MetricsHandler.EXPR_PARAM, key3), resp);
+ val = resp.getValues().findRecursive( "metrics", "solr.core.collection1");
+ assertNotNull(val);
+ // for requestTimes only the full set of values from the first expr should be present
+ assertNotNull(val);
+ SimpleOrderedMap<Object> values = (SimpleOrderedMap<Object>) val;
+ assertEquals(values.jsonStr(), 4, values.size());
+ List<Object> multipleVals = values.getAll("QUERY./select.requestTimes");
+ assertEquals(multipleVals.toString(), 1, multipleVals.size());
+ v = values.get("QUERY./select.local.requestTimes");
+ assertTrue(v instanceof MapWriter);
+ ((MapWriter) v).toMap(map);
+ assertEquals(map.toString(), 1, map.size());
+ assertTrue(map.toString(), map.containsKey("count"));
+ map.clear();
+ v = values.get("QUERY./select.distrib.requestTimes");
+ assertTrue(v instanceof MapWriter);
+ ((MapWriter) v).toMap(map);
+ assertEquals(map.toString(), 1, map.size());
+ assertTrue(map.toString(), map.containsKey("count"));
+ }
+
+ @Test
public void testMetricsUnload() throws Exception {
SolrCore core = h.getCoreContainer().getCore("collection1");//;.getRequestHandlers().put("/dumphandler", new DumpRequestHandler());
diff --git a/solr/solr-ref-guide/src/metrics-reporting.adoc b/solr/solr-ref-guide/src/metrics-reporting.adoc
index 7d317e8..e5d98ae 100644
--- a/solr/solr-ref-guide/src/metrics-reporting.adoc
+++ b/solr/solr-ref-guide/src/metrics-reporting.adoc
@@ -676,8 +676,8 @@ A few query parameters are available to limit your request to only certain metri
`property`:: Allows requesting only this metric from any compound metric. Multiple `property` parameters can be combined to act as an OR request. For example, to only get the 99th and 999th percentile values from all metric types and groups, you can add `&property=p99_ms&property=p999_ms` to your request. This can be combined with `group`, `type`, and `prefix` as necessary.
`key`:: fully-qualified metric name, which specifies one concrete metric instance (parameter can be
-specified multiple times to retrieve multiple concrete metrics). *NOTE: when this parameter is used, other
-selection methods listed above are ignored.* Fully-qualified name consists of registry name, colon and
+specified multiple times to retrieve multiple concrete metrics). *NOTE: when this parameter is used, any other
+selection methods are ignored.* Fully-qualified name consists of registry name, colon and
metric name, with optional colon and metric property. Colons in names can be escaped using back-slash `\`
character. Examples:
@@ -685,6 +685,23 @@ character. Examples:
* `key=solr.core.collection1:QUERY./select.requestTimes:max_ms`
* `key=solr.jvm:system.properties:user.name`
+`expr`:: Extended notation of the `key` selection criteria, which supports regular expressions for each of the
+parts supported by the `key` selector. This parameter can be specified multiple times to retrieve metrics that match
+any expression. The API guarantees that the output will consist only of unique metric names even if
+multiple expressions match the same metric name. Note: order of multiple `expr` parameters matters here - only the first value of the first matching expression will be recorded, subsequent values for the same metric name
+produced by matching other expressions will be skipped.
+
+Fully-qualified expression consists of at least two and at most three regex patterns separated by
+colons: a registry pattern, colon, a metric pattern, and then an optional colon and metric property pattern.
+Colons and other regex meta-characters in names and in regular expressions MUST be escaped using backslash (`\`) character.
+
+*NOTE: when this parameter is used, any other selection methods are ignored.*
+
+Examples:
+
+* `expr=solr\.core\..*:QUERY\..*\.requestTimes:max_ms`
+* `expr=solr\.jvm:system\.properties:user\..*`
+
`compact`:: When false, a more verbose format of the response will be returned. Instead of a response like this:
+
[source,json]