You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2017/08/19 12:31:32 UTC
[21/37] lucene-solr:feature/autoscaling_72: SOLR-11215: Make a metric
accessible through a single param.
SOLR-11215: Make a metric accessible through a single param.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ce8c43a1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ce8c43a1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ce8c43a1
Branch: refs/heads/feature/autoscaling_72
Commit: ce8c43a113c57b4d2e0f10484ee645332b2bc0c8
Parents: eb49230
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Thu Aug 17 12:32:22 2017 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Thu Aug 17 12:32:22 2017 +0200
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +-
.../plugin/AnalyticsStatisticsCollector.java | 2 +-
.../solr/handler/admin/MetricsHandler.java | 66 +++++++++++++++
.../apache/solr/metrics/SolrMetricManager.java | 11 +++
.../org/apache/solr/util/stats/MetricUtils.java | 38 ++++-----
.../solr/handler/admin/MetricsHandlerTest.java | 86 +++++++++++++++++++-
.../apache/solr/util/stats/MetricUtilsTest.java | 2 +-
7 files changed, 183 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce8c43a1/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index daad2ff..d8326b8 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -32,7 +32,6 @@ Jetty 9.3.14.v20161028
(No Changes)
-
================== 7.1.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
@@ -88,6 +87,7 @@ New Features
A new "sum" function is also added. Example :
{!payload_score f=payload_field func=sum operator=or}A B C" (Varun Thacker)
+* SOLR-11215: Make a metric accessible through a single param. (ab)
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce8c43a1/solr/contrib/analytics/src/java/org/apache/solr/analytics/plugin/AnalyticsStatisticsCollector.java
----------------------------------------------------------------------
diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/plugin/AnalyticsStatisticsCollector.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/plugin/AnalyticsStatisticsCollector.java
index 1670ad5..4c99323 100644
--- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/plugin/AnalyticsStatisticsCollector.java
+++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/plugin/AnalyticsStatisticsCollector.java
@@ -88,7 +88,7 @@ public class AnalyticsStatisticsCollector {
public Map<String, Object> getStatistics() {
Map<String, Object> map = new HashMap<>();
- MetricUtils.convertTimer("", requestTimes, MetricUtils.PropertyFilter.ALL, false, false, (k, v) -> {
+ MetricUtils.convertTimer("", requestTimes, MetricUtils.PropertyFilter.ALL, false, false, ".", (k, v) -> {
map.putAll((Map<String, Object>)v);
});
map.put("requests", numRequests.longValue());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce8c43a1/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
index 11f6821..a9a2d1f 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
@@ -23,12 +23,14 @@ import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.regex.Pattern;
import java.util.stream.Collectors;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
+import com.codahale.metrics.Metric;
import com.codahale.metrics.MetricFilter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
@@ -58,10 +60,13 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
public static final String PROPERTY_PARAM = "property";
public static final String REGISTRY_PARAM = "registry";
public static final String GROUP_PARAM = "group";
+ public static final String KEY_PARAM = "key";
public static final String TYPE_PARAM = "type";
public static final String ALL = "all";
+ private static final Pattern KEY_REGEX = Pattern.compile("(?<!" + Pattern.quote("\\") + ")" + Pattern.quote(":"));
+
public MetricsHandler() {
this.container = null;
this.metricManager = null;
@@ -84,6 +89,11 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
}
boolean compact = req.getParams().getBool(COMPACT_PARAM, true);
+ String[] keys = req.getParams().getParams(KEY_PARAM);
+ if (keys != null && keys.length > 0) {
+ handleKeyRequest(keys, req, rsp);
+ return;
+ }
MetricFilter mustMatchFilter = parseMustMatchFilter(req);
MetricUtils.PropertyFilter propertyFilter = parsePropertyFilter(req);
List<MetricType> metricTypes = parseMetricTypes(req);
@@ -103,6 +113,62 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
rsp.getValues().add("metrics", response);
}
+ private void handleKeyRequest(String[] keys, SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
+ SimpleOrderedMap result = new SimpleOrderedMap();
+ SimpleOrderedMap errors = new SimpleOrderedMap();
+ for (String key : keys) {
+ if (key == null || key.isEmpty()) {
+ continue;
+ }
+ String[] parts = KEY_REGEX.split(key);
+ if (parts.length < 2 || parts.length > 3) {
+ errors.add(key, "at least two and at most three colon-separated parts must be provided");
+ continue;
+ }
+ final String registryName = unescape(parts[0]);
+ final String metricName = unescape(parts[1]);
+ final String propertyName = parts.length > 2 ? unescape(parts[2]) : null;
+ if (!metricManager.hasRegistry(registryName)) {
+ errors.add(key, "registry '" + registryName + "' not found");
+ continue;
+ }
+ MetricRegistry registry = metricManager.registry(registryName);
+ Metric m = registry.getMetrics().get(metricName);
+ if (m == null) {
+ errors.add(key, "metric '" + metricName + "' not found");
+ continue;
+ }
+ MetricUtils.PropertyFilter propertyFilter = MetricUtils.PropertyFilter.ALL;
+ boolean simple = false;
+ if (propertyName != null) {
+ propertyFilter = (name) -> name.equals(propertyName);
+ simple = true;
+ // use escaped versions
+ key = parts[0] + ":" + parts[1];
+ }
+ MetricUtils.convertMetric(key, m, propertyFilter, false, true, true, simple, ":", (k, v) -> result.add(k, v));
+ }
+ rsp.getValues().add("metrics", result);
+ if (errors.size() > 0) {
+ rsp.getValues().add("errors", errors);
+ }
+ }
+
+ private static String unescape(String s) {
+ if (s.indexOf('\\') == -1) {
+ return s;
+ }
+ StringBuilder sb = new StringBuilder(s.length());
+ for (int i = 0; i < s.length(); i++) {
+ char c = s.charAt(i);
+ if (c == '\\') {
+ continue;
+ }
+ sb.append(c);
+ }
+ return sb.toString();
+ }
+
private MetricFilter parseMustMatchFilter(SolrQueryRequest req) {
String[] prefixes = req.getParams().getParams(PREFIX_PARAM);
MetricFilter prefixFilter = null;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce8c43a1/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
index 001717c..af642ed 100644
--- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
+++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
@@ -353,6 +353,17 @@ public class SolrMetricManager {
}
/**
+ * Check whether a registry with a given (overridable) name already exists.
+ * @param name registry name
+ * @return true if this name points to a registry that already exists, false otherwise
+ */
+ public boolean hasRegistry(String name) {
+ Set<String> names = registryNames();
+ name = overridableRegistryName(name);
+ return names.contains(name);
+ }
+
+ /**
* Return set of existing registry names that match a regex pattern
* @param patterns regex patterns. NOTE: users need to make sure that patterns that
* don't start with a wildcard use the full registry name starting with
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce8c43a1/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java
index a190a0f..29e2db2 100644
--- a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java
+++ b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java
@@ -210,7 +210,7 @@ public class MetricUtils {
.filter(s -> mustMatchFilter.matches(s, metrics.get(s)))
.forEach(n -> {
Metric metric = metrics.get(n);
- convertMetric(n, metric, propertyFilter, skipHistograms, skipAggregateValues, compact, simple, consumer);
+ convertMetric(n, metric, propertyFilter, skipHistograms, skipAggregateValues, compact, simple, ".", consumer);
});
}
@@ -247,7 +247,7 @@ public class MetricUtils {
names.stream()
.forEach(n -> {
Metric metric = metrics.get(n);
- convertMetric(n, metric, PropertyFilter.ALL, skipHistograms, skipAggregateValues, compact, simple, consumer);
+ convertMetric(n, metric, PropertyFilter.ALL, skipHistograms, skipAggregateValues, compact, simple, ".", consumer);
});
}
@@ -263,15 +263,15 @@ public class MetricUtils {
* only the selected (name "." key, value) pairs will be produced.
* @param consumer consumer that accepts produced objects
*/
- static void convertMetric(String n, Metric metric, PropertyFilter propertyFilter, boolean skipHistograms, boolean skipAggregateValues,
- boolean compact, boolean simple, BiConsumer<String, Object> consumer) {
+ public static void convertMetric(String n, Metric metric, PropertyFilter propertyFilter, boolean skipHistograms, boolean skipAggregateValues,
+ boolean compact, boolean simple, String separator, BiConsumer<String, Object> consumer) {
if (metric instanceof Counter) {
Counter counter = (Counter) metric;
convertCounter(n, counter, propertyFilter, compact, consumer);
} else if (metric instanceof Gauge) {
Gauge gauge = (Gauge) metric;
try {
- convertGauge(n, gauge, propertyFilter, simple, compact, consumer);
+ convertGauge(n, gauge, propertyFilter, simple, compact, separator, consumer);
} catch (InternalError ie) {
if (n.startsWith("memory.") && ie.getMessage().contains("Memory Pool not found")) {
LOG.warn("Error converting gauge '" + n + "', possible JDK bug: SOLR-10362", ie);
@@ -282,17 +282,17 @@ public class MetricUtils {
}
} else if (metric instanceof Meter) {
Meter meter = (Meter) metric;
- convertMeter(n, meter, propertyFilter, simple, consumer);
+ convertMeter(n, meter, propertyFilter, simple, separator, consumer);
} else if (metric instanceof Timer) {
Timer timer = (Timer) metric;
- convertTimer(n, timer, propertyFilter, skipHistograms, simple, consumer);
+ convertTimer(n, timer, propertyFilter, skipHistograms, simple, separator, consumer);
} else if (metric instanceof Histogram) {
if (!skipHistograms) {
Histogram histogram = (Histogram) metric;
- convertHistogram(n, histogram, propertyFilter, simple, consumer);
+ convertHistogram(n, histogram, propertyFilter, simple, separator, consumer);
}
} else if (metric instanceof AggregateMetric) {
- convertAggregateMetric(n, (AggregateMetric)metric, propertyFilter, skipAggregateValues, simple, consumer);
+ convertAggregateMetric(n, (AggregateMetric)metric, propertyFilter, skipAggregateValues, simple, separator, consumer);
}
}
@@ -308,10 +308,10 @@ public class MetricUtils {
*/
static void convertAggregateMetric(String name, AggregateMetric metric,
PropertyFilter propertyFilter,
- boolean skipAggregateValues, boolean simple, BiConsumer<String, Object> consumer) {
+ boolean skipAggregateValues, boolean simple, String separator, BiConsumer<String, Object> consumer) {
if (simple) {
if (propertyFilter.accept(MEAN)) {
- consumer.accept(name + "." + MEAN, metric.getMean());
+ consumer.accept(name + separator + MEAN, metric.getMean());
}
} else {
Map<String, Object> response = new LinkedHashMap<>();
@@ -353,11 +353,11 @@ public class MetricUtils {
* @param consumer consumer that accepts produced objects
*/
static void convertHistogram(String name, Histogram histogram, PropertyFilter propertyFilter,
- boolean simple, BiConsumer<String, Object> consumer) {
+ boolean simple, String separator, BiConsumer<String, Object> consumer) {
Snapshot snapshot = histogram.getSnapshot();
if (simple) {
if (propertyFilter.accept(MEAN)) {
- consumer.accept(name + "." + MEAN, snapshot.getMean());
+ consumer.accept(name + separator + MEAN, snapshot.getMean());
}
} else {
Map<String, Object> response = new LinkedHashMap<>();
@@ -411,11 +411,11 @@ public class MetricUtils {
* @param consumer consumer that accepts produced objects
*/
public static void convertTimer(String name, Timer timer, PropertyFilter propertyFilter, boolean skipHistograms,
- boolean simple, BiConsumer<String, Object> consumer) {
+ boolean simple, String separator, BiConsumer<String, Object> consumer) {
if (simple) {
String prop = "meanRate";
if (propertyFilter.accept(prop)) {
- consumer.accept(name + "." + prop, timer.getMeanRate());
+ consumer.accept(name + separator + prop, timer.getMeanRate());
}
} else {
Map<String, Object> response = new LinkedHashMap<>();
@@ -448,10 +448,10 @@ public class MetricUtils {
* only the selected (name "." key, value) pairs will be produced.
* @param consumer consumer that accepts produced objects
*/
- static void convertMeter(String name, Meter meter, PropertyFilter propertyFilter, boolean simple, BiConsumer<String, Object> consumer) {
+ static void convertMeter(String name, Meter meter, PropertyFilter propertyFilter, boolean simple, String separator, BiConsumer<String, Object> consumer) {
if (simple) {
if (propertyFilter.accept("count")) {
- consumer.accept(name + ".count", meter.getCount());
+ consumer.accept(name + separator + "count", meter.getCount());
}
} else {
Map<String, Object> response = new LinkedHashMap<>();
@@ -483,7 +483,7 @@ public class MetricUtils {
* @param consumer consumer that accepts produced objects
*/
static void convertGauge(String name, Gauge gauge, PropertyFilter propertyFilter, boolean simple, boolean compact,
- BiConsumer<String, Object> consumer) {
+ String separator, BiConsumer<String, Object> consumer) {
if (compact || simple) {
Object o = gauge.getValue();
if (o instanceof Map) {
@@ -491,7 +491,7 @@ public class MetricUtils {
for (Map.Entry<?, ?> entry : ((Map<?, ?>)o).entrySet()) {
String prop = entry.getKey().toString();
if (propertyFilter.accept(prop)) {
- consumer.accept(name + "." + prop, entry.getValue());
+ consumer.accept(name + separator + prop, entry.getValue());
}
}
} else {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce8c43a1/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
index d7ead6b..7eb8f12 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
@@ -19,6 +19,7 @@ package org.apache.solr.handler.admin;
import java.util.Map;
+import com.codahale.metrics.Counter;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.NamedList;
@@ -36,8 +37,13 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
initCore("solrconfig.xml", "schema.xml");
// manually register some metrics in solr.jvm and solr.jetty - TestHarness doesn't init them
- h.getCoreContainer().getMetricManager().counter(null, "solr.jvm", "foo");
- h.getCoreContainer().getMetricManager().counter(null, "solr.jetty", "foo");
+ Counter c = h.getCoreContainer().getMetricManager().counter(null, "solr.jvm", "foo");
+ c.inc();
+ c = h.getCoreContainer().getMetricManager().counter(null, "solr.jetty", "foo");
+ c.inc(2);
+ // test escapes
+ c = h.getCoreContainer().getMetricManager().counter(null, "solr.jetty", "foo:bar");
+ c.inc(3);
}
@Test
@@ -225,4 +231,80 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
assertNotNull(map.get("size"));
});
}
+
+ @Test
+ public void testKeyMetrics() throws Exception {
+ MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
+
+ String key1 = "solr.core.collection1:CACHE.core.fieldCache";
+ SolrQueryResponse resp = new SolrQueryResponse();
+ handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
+ MetricsHandler.KEY_PARAM, key1), resp);
+ NamedList values = resp.getValues();
+ Object val = values.findRecursive("metrics", key1);
+ assertNotNull(val);
+ assertTrue(val instanceof Map);
+ assertTrue(((Map)val).size() >= 2);
+
+ String key2 = "solr.core.collection1:CACHE.core.fieldCache:entries_count";
+ resp = new SolrQueryResponse();
+ handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
+ MetricsHandler.KEY_PARAM, key2), resp);
+ values = resp.getValues();
+ val = values.findRecursive("metrics", key2);
+ assertNotNull(val);
+ assertTrue(val instanceof Number);
+
+ String key3 = "solr.jetty:foo\\:bar";
+ resp = new SolrQueryResponse();
+ handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
+ MetricsHandler.KEY_PARAM, key3), resp);
+ values = resp.getValues();
+ val = values.findRecursive("metrics", key3);
+ assertNotNull(val);
+ assertTrue(val instanceof Number);
+ assertEquals(3, ((Number)val).intValue());
+
+ // test multiple keys
+ resp = new SolrQueryResponse();
+ handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
+ MetricsHandler.KEY_PARAM, key1, MetricsHandler.KEY_PARAM, key2, MetricsHandler.KEY_PARAM, key3), resp);
+ values = resp.getValues();
+ val = values.findRecursive("metrics", key1);
+ assertNotNull(val);
+ val = values.findRecursive("metrics", key2);
+ assertNotNull(val);
+ val = values.findRecursive("metrics", key3);
+ assertNotNull(val);
+
+ // test errors
+
+ // invalid keys
+ resp = new SolrQueryResponse();
+ handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
+ MetricsHandler.KEY_PARAM, "foo", MetricsHandler.KEY_PARAM, "foo:bar:baz:xyz"), resp);
+ values = resp.getValues();
+ NamedList metrics = (NamedList)values.get("metrics");
+ assertEquals(0, metrics.size());
+ assertNotNull(values.findRecursive("errors", "foo"));
+ assertNotNull(values.findRecursive("errors", "foo:bar:baz:xyz"));
+
+ // unknown registry
+ resp = new SolrQueryResponse();
+ handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
+ MetricsHandler.KEY_PARAM, "foo:bar:baz"), resp);
+ values = resp.getValues();
+ metrics = (NamedList)values.get("metrics");
+ assertEquals(0, metrics.size());
+ assertNotNull(values.findRecursive("errors", "foo:bar:baz"));
+
+ // unknown metric
+ resp = new SolrQueryResponse();
+ handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
+ MetricsHandler.KEY_PARAM, "solr.jetty:unknown:baz"), resp);
+ values = resp.getValues();
+ metrics = (NamedList)values.get("metrics");
+ assertEquals(0, metrics.size());
+ assertNotNull(values.findRecursive("errors", "solr.jetty:unknown:baz"));
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce8c43a1/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java b/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java
index 4eb1169..fa9ab51 100644
--- a/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java
+++ b/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java
@@ -47,7 +47,7 @@ public class MetricUtilsTest extends SolrTestCaseJ4 {
}
// obtain timer metrics
Map<String,Object> map = new HashMap<>();
- MetricUtils.convertTimer("", timer, MetricUtils.PropertyFilter.ALL, false, false, (k, v) -> {
+ MetricUtils.convertTimer("", timer, MetricUtils.PropertyFilter.ALL, false, false, ".", (k, v) -> {
map.putAll((Map<String,Object>)v);
});
NamedList lst = new NamedList(map);