You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2020/10/07 10:48:08 UTC
[lucene-solr] branch jira/solr-14691 updated: SOLR-14691: Make
MetricsMap implement MapWriter to avoid Map allocations during
serialization. Deprecate the use of MetricUtils.PropertyFilter.
This is an automated email from the ASF dual-hosted git repository.
ab pushed a commit to branch jira/solr-14691
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/jira/solr-14691 by this push:
new 1202ce4 SOLR-14691: Make MetricsMap implement MapWriter to avoid Map allocations during serialization. Deprecate the use of MetricUtils.PropertyFilter.
1202ce4 is described below
commit 1202ce4be8679967042aeac49f66df1ba58e8267
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Wed Oct 7 12:47:12 2020 +0200
SOLR-14691: Make MetricsMap implement MapWriter to avoid Map allocations
during serialization. Deprecate the use of MetricUtils.PropertyFilter.
---
.../apache/solr/handler/ReplicationHandler.java | 7 +-
.../apache/solr/handler/RequestHandlerBase.java | 4 +-
.../apache/solr/handler/admin/MetricsHandler.java | 11 +-
.../solr/handler/component/SuggestComponent.java | 4 +-
.../java/org/apache/solr/metrics/MetricsMap.java | 38 ++++-
.../solr/metrics/reporters/solr/SolrReporter.java | 2 +-
.../java/org/apache/solr/search/CaffeineCache.java | 2 +-
.../org/apache/solr/search/SolrFieldCacheBean.java | 4 +-
.../org/apache/solr/search/SolrIndexSearcher.java | 4 +-
.../apache/solr/servlet/SolrDispatchFilter.java | 4 +-
.../org/apache/solr/store/blockcache/Metrics.java | 2 +-
.../solr/store/hdfs/HdfsLocalityReporter.java | 2 +-
.../org/apache/solr/util/stats/MetricUtils.java | 182 ++++++++++++++++++---
.../test/org/apache/solr/core/MockInfoBean.java | 2 +-
.../solr/handler/admin/MetricsHandlerTest.java | 2 +-
.../apache/solr/util/stats/MetricUtilsTest.java | 24 +++
16 files changed, 239 insertions(+), 55 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
index 2b5eb19..f2049a6 100644
--- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
@@ -68,6 +68,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RateLimiter;
+import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.CommonParams;
@@ -894,7 +895,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
true, "isLeader", getCategory().toString(), scope);
solrMetricsContext.gauge(() -> isFollower,
true, "isFollower", getCategory().toString(), scope);
- final MetricsMap fetcherMap = new MetricsMap((detailed, map) -> {
+ final MetricsMap fetcherMap = new MetricsMap(map -> {
IndexFetcher fetcher = currentIndexFetcher;
if (fetcher != null) {
map.put(LEADER_URL, fetcher.getLeaderUrl());
@@ -1110,10 +1111,10 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
}
}
- private void addVal(Map<String, Object> map, String key, Properties props, @SuppressWarnings({"rawtypes"})Class clzz) {
+ private void addVal(MapWriter.EntryWriter ew, String key, Properties props, @SuppressWarnings({"rawtypes"})Class clzz) {
Object val = formatVal(key, props, clzz);
if (val != null) {
- map.put(key, val);
+ ew.putNoEx(key, val);
}
}
diff --git a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
index 1fcc183..d41ad54 100644
--- a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
+++ b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
@@ -157,8 +157,8 @@ public abstract class RequestHandlerBase implements SolrRequestHandler, SolrInfo
numClientErrors = solrMetricsContext.meter("clientErrors", getCategory().toString(), scope);
numTimeouts = solrMetricsContext.meter("timeouts", getCategory().toString(), scope);
requests = solrMetricsContext.counter("requests", getCategory().toString(), scope);
- MetricsMap metricsMap = new MetricsMap((detail, map) ->
- shardPurposes.forEach((k, v) -> map.put(k, v.getCount())));
+ MetricsMap metricsMap = new MetricsMap(map ->
+ shardPurposes.forEach((k, v) -> map.putNoEx(k, v.getCount())));
solrMetricsContext.gauge(metricsMap, true, "shardRequests", getCategory().toString(), scope);
requestTimes = solrMetricsContext.timer("requestTimes", getCategory().toString(), scope);
distribRequestTimes = solrMetricsContext.timer("requestTimes", getCategory().toString(), scope, "distrib");
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
index 26c7a2b..76bb04d 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.BiConsumer;
+import java.util.function.Predicate;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -113,7 +114,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
return;
}
MetricFilter mustMatchFilter = parseMustMatchFilter(params);
- MetricUtils.PropertyFilter propertyFilter = parsePropertyFilter(params);
+ Predicate<CharSequence> propertyFilter = parsePropertyFilter(params);
List<MetricType> metricTypes = parseMetricTypes(params);
List<MetricFilter> metricFilters = metricTypes.stream().map(MetricType::asMetricFilter).collect(Collectors.toList());
Set<String> requestedRegistries = parseRegistries(params);
@@ -157,7 +158,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
errors.add(key, "metric '" + metricName + "' not found");
continue;
}
- MetricUtils.PropertyFilter propertyFilter = MetricUtils.PropertyFilter.ALL;
+ Predicate<CharSequence> propertyFilter = MetricUtils.ALL_PROPERTIES;
if (propertyName != null) {
propertyFilter = (name) -> name.equals(propertyName);
// use escaped versions
@@ -230,10 +231,10 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
return mustMatchFilter;
}
- private MetricUtils.PropertyFilter parsePropertyFilter(SolrParams params) {
+ private Predicate<CharSequence> parsePropertyFilter(SolrParams params) {
String[] props = params.getParams(PROPERTY_PARAM);
if (props == null || props.length == 0) {
- return MetricUtils.PropertyFilter.ALL;
+ return MetricUtils.ALL_PROPERTIES;
}
final Set<String> filter = new HashSet<>();
for (String prop : props) {
@@ -242,7 +243,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
}
}
if (filter.isEmpty()) {
- return MetricUtils.PropertyFilter.ALL;
+ return MetricUtils.ALL_PROPERTIES;
} else {
return (name) -> filter.contains(name);
}
diff --git a/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java b/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java
index 59a9571..adfcb15 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java
@@ -357,10 +357,10 @@ public class SuggestComponent extends SearchComponent implements SolrCoreAware,
super.initializeMetrics(parentContext, scope);
this.solrMetricsContext.gauge(() -> ramBytesUsed(), true, "totalSizeInBytes", getCategory().toString());
- MetricsMap suggestersMap = new MetricsMap((detailed, map) -> {
+ MetricsMap suggestersMap = new MetricsMap(map -> {
for (Map.Entry<String, SolrSuggester> entry : suggesters.entrySet()) {
SolrSuggester suggester = entry.getValue();
- map.put(entry.getKey(), suggester.toString());
+ map.putNoEx(entry.getKey(), suggester.toString());
}
});
this.solrMetricsContext.gauge(suggestersMap, true, "suggesters", getCategory().toString(), scope);
diff --git a/solr/core/src/java/org/apache/solr/metrics/MetricsMap.java b/solr/core/src/java/org/apache/solr/metrics/MetricsMap.java
index bd9abaf..d2e0cec 100644
--- a/solr/core/src/java/org/apache/solr/metrics/MetricsMap.java
+++ b/solr/core/src/java/org/apache/solr/metrics/MetricsMap.java
@@ -28,6 +28,7 @@ import javax.management.ReflectionException;
import javax.management.openmbean.OpenMBeanAttributeInfoSupport;
import javax.management.openmbean.OpenType;
import javax.management.openmbean.SimpleType;
+import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.lang.reflect.Field;
import java.util.ArrayList;
@@ -38,6 +39,7 @@ import java.util.function.BiConsumer;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Metric;
import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,18 +53,32 @@ import org.slf4j.LoggerFactory;
* {@link javax.management.openmbean.OpenType#ALLOWED_CLASSNAMES_LIST}, otherwise only their toString()
* representation will be shown in JConsole.</p>
*/
-public class MetricsMap implements Gauge<Map<String,Object>>, DynamicMBean {
+public class MetricsMap implements Gauge<Map<String,Object>>, MapWriter, DynamicMBean {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
// set to true to use cached statistics between getMBeanInfo calls to work
// around over calling getStatistics on MBeanInfos when iterating over all attributes (SOLR-6586)
private final boolean useCachedStatsBetweenGetMBeanInfoCalls = Boolean.getBoolean("useCachedStatsBetweenGetMBeanInfoCalls");
- private BiConsumer<Boolean, Map<String, Object>> initializer;
+ private BiConsumer<Boolean, Map<String, Object>> mapInitializer;
+ private MapWriter initializer;
private Map<String, String> jmxAttributes = new HashMap<>();
private volatile Map<String,Object> cachedValue;
- public MetricsMap(BiConsumer<Boolean, Map<String,Object>> initializer) {
+ /**
+ * Create an instance that reports values to a Map.
+ * @param mapInitializer function to populate the Map result.
+ * @deprecated use {@link #MetricsMap(MapWriter)} instead.
+ */
+ public MetricsMap(BiConsumer<Boolean, Map<String,Object>> mapInitializer) {
+ this.mapInitializer = mapInitializer;
+ }
+
+ /**
+ * Create an instance that reports values to a MapWriter.
+ * @param initializer function to populate the MapWriter result.
+ */
+ public MetricsMap(MapWriter initializer) {
this.initializer = initializer;
}
@@ -73,7 +89,11 @@ public class MetricsMap implements Gauge<Map<String,Object>>, DynamicMBean {
public Map<String,Object> getValue(boolean detailed) {
Map<String,Object> map = new HashMap<>();
- initializer.accept(detailed, map);
+ if (mapInitializer != null) {
+ mapInitializer.accept(detailed, map);
+ } else {
+ initializer.toMap(map);
+ }
return map;
}
@@ -197,4 +217,14 @@ public class MetricsMap implements Gauge<Map<String,Object>>, DynamicMBean {
}
return null;
}
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ if (mapInitializer != null) {
+ Map<String, Object> value = getValue();
+ value.forEach((k, v) -> ew.putNoEx(k, v));
+ } else {
+ initializer.writeMap(ew);
+ }
+ }
}
\ No newline at end of file
diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java
index 929aa93..9606ef5 100644
--- a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java
+++ b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java
@@ -453,7 +453,7 @@ public class SolrReporter extends ScheduledReporter {
}
final String effectiveGroup = group;
MetricUtils.toSolrInputDocuments(metricManager.registry(registryName), Collections.singletonList(report.filter), MetricFilter.ALL,
- MetricUtils.PropertyFilter.ALL, skipHistograms, skipAggregateValues, compact, metadata, doc -> {
+ MetricUtils.ALL_PROPERTIES, skipHistograms, skipAggregateValues, compact, metadata, doc -> {
doc.setField(REGISTRY_ID, registryName);
doc.setField(GROUP_ID, effectiveGroup);
if (effectiveLabel != null) {
diff --git a/solr/core/src/java/org/apache/solr/search/CaffeineCache.java b/solr/core/src/java/org/apache/solr/search/CaffeineCache.java
index 8da244b..756718c 100644
--- a/solr/core/src/java/org/apache/solr/search/CaffeineCache.java
+++ b/solr/core/src/java/org/apache/solr/search/CaffeineCache.java
@@ -364,7 +364,7 @@ public class CaffeineCache<K, V> extends SolrCacheBase implements SolrCache<K, V
@Override
public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
solrMetricsContext = parentContext.getChildContext(this);
- cacheMap = new MetricsMap((detailed, map) -> {
+ cacheMap = new MetricsMap(map -> {
if (cache != null) {
CacheStats stats = cache.stats();
long insertCount = inserts.sum();
diff --git a/solr/core/src/java/org/apache/solr/search/SolrFieldCacheBean.java b/solr/core/src/java/org/apache/solr/search/SolrFieldCacheBean.java
index 5005627..193abbf 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrFieldCacheBean.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrFieldCacheBean.java
@@ -49,8 +49,8 @@ public class SolrFieldCacheBean implements SolrInfoBean {
@Override
public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
this.solrMetricsContext = parentContext;
- MetricsMap metricsMap = new MetricsMap((detailed, map) -> {
- if (detailed && !disableEntryList && !disableJmxEntryList) {
+ MetricsMap metricsMap = new MetricsMap(map -> {
+ if (!disableEntryList && !disableJmxEntryList) {
UninvertingReader.FieldCacheStats fieldCacheStats = UninvertingReader.getUninvertedStats();
String[] entries = fieldCacheStats.info;
map.put("entries_count", entries.length);
diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
index 3ebd43c..5d1af6d 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
@@ -2298,8 +2298,8 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
}, true, "indexCommitSize", Category.SEARCHER.toString(), scope);
// statsCache metrics
parentContext.gauge(
- new MetricsMap((detailed, map) -> {
- statsCache.getCacheMetrics().getSnapshot(map::put);
+ new MetricsMap(map -> {
+ statsCache.getCacheMetrics().getSnapshot(map::putNoEx);
map.put("statsCacheImpl", statsCache.getClass().getSimpleName());
}), true, "statsCache", Category.CACHE.toString(), scope);
}
diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
index 01630a8..95afda4 100644
--- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
+++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
@@ -235,10 +235,10 @@ public class SolrDispatchFilter extends BaseSolrFilter {
metricManager.registerAll(registryName, new GarbageCollectorMetricSet(), SolrMetricManager.ResolutionStrategy.IGNORE, "gc");
metricManager.registerAll(registryName, new MemoryUsageGaugeSet(), SolrMetricManager.ResolutionStrategy.IGNORE, "memory");
metricManager.registerAll(registryName, new ThreadStatesGaugeSet(), SolrMetricManager.ResolutionStrategy.IGNORE, "threads"); // todo should we use CachedThreadStatesGaugeSet instead?
- MetricsMap sysprops = new MetricsMap((detailed, map) -> {
+ MetricsMap sysprops = new MetricsMap(map -> {
System.getProperties().forEach((k, v) -> {
if (!hiddenSysProps.contains(k)) {
- map.put(String.valueOf(k), v);
+ map.putNoEx(String.valueOf(k), v);
}
});
});
diff --git a/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java b/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java
index 37d735e..76bb265 100644
--- a/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java
+++ b/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java
@@ -59,7 +59,7 @@ public class Metrics extends SolrCacheBase implements SolrInfoBean {
@Override
public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
solrMetricsContext = parentContext.getChildContext(this);
- metricsMap = new MetricsMap((detailed, map) -> {
+ metricsMap = new MetricsMap(map -> {
long now = System.nanoTime();
long delta = Math.max(now - previous, 1);
double seconds = delta / 1000000000.0;
diff --git a/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLocalityReporter.java b/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLocalityReporter.java
index 6ff88f2..fe19d12 100644
--- a/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLocalityReporter.java
+++ b/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLocalityReporter.java
@@ -93,7 +93,7 @@ public class HdfsLocalityReporter implements SolrInfoBean {
@Override
public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
solrMetricsContext = parentContext.getChildContext(this);
- MetricsMap metricsMap = new MetricsMap((detailed, map) -> {
+ MetricsMap metricsMap = new MetricsMap(map -> {
long totalBytes = 0;
long localBytes = 0;
int totalCount = 0;
diff --git a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java
index 544c0c2..28b0f52 100644
--- a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java
+++ b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java
@@ -34,6 +34,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
+import java.util.function.Predicate;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
@@ -45,12 +46,14 @@ import com.codahale.metrics.MetricFilter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Snapshot;
import com.codahale.metrics.Timer;
+import org.apache.solr.common.ConditionalKeyMapWriter;
import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.metrics.AggregateMetric;
+import org.apache.solr.metrics.SolrMetricManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -88,6 +91,7 @@ public class MetricUtils {
/**
* This filter can limit what properties of a metric are returned.
+ * @deprecated use {@link Predicate<CharSequence>} instead.
*/
public interface PropertyFilter {
PropertyFilter ALL = (name) -> true;
@@ -98,8 +102,14 @@ public class MetricUtils {
* @return true if this property should be returned, false otherwise.
*/
boolean accept(String name);
+
+ static Predicate<CharSequence> toPredicate(PropertyFilter filter) {
+ return (name) -> filter.accept(name.toString());
+ }
}
+ public static final Predicate<CharSequence> ALL_PROPERTIES = (name) -> true;
+
/**
* Adds metrics from a Timer to a NamedList, using well-known back-compat names.
* @param lst The NamedList to add the metrics data to
@@ -146,11 +156,39 @@ public class MetricUtils {
* @param metadata optional metadata. If not null and not empty then this map will be added under a
* {@code _metadata_} key.
* @param consumer consumer that accepts produced {@link SolrInputDocument}-s
+ * @deprecated use {@link #toSolrInputDocuments(MetricRegistry, List, MetricFilter, Predicate, boolean, boolean, boolean, Map, Consumer)} instead.
*/
public static void toSolrInputDocuments(MetricRegistry registry, List<MetricFilter> shouldMatchFilters,
MetricFilter mustMatchFilter, PropertyFilter propertyFilter, boolean skipHistograms,
boolean skipAggregateValues, boolean compact,
Map<String, Object> metadata, Consumer<SolrInputDocument> consumer) {
+ toSolrInputDocuments(registry, shouldMatchFilters, mustMatchFilter,
+ PropertyFilter.toPredicate(propertyFilter), skipHistograms,
+ skipAggregateValues, compact, metadata, consumer);
+ }
+ /**
+ * Provides a representation of the given metric registry as {@link SolrInputDocument}-s.
+ Only those metrics
+ * are converted which match at least one of the given MetricFilter instances.
+ *
+ * @param registry the {@link MetricRegistry} to be converted
+ * @param shouldMatchFilters a list of {@link MetricFilter} instances.
+ * A metric must match <em>any one</em> of the filters from this list to be
+ * included in the output
+ * @param mustMatchFilter a {@link MetricFilter}.
+ * A metric <em>must</em> match this filter to be included in the output.
+ * @param propertyFilter limit what properties of a metric are returned
+ * @param skipHistograms discard any {@link Histogram}-s and histogram parts of {@link Timer}-s.
+ * @param skipAggregateValues discard internal values of {@link AggregateMetric}-s.
+ * @param compact use compact representation for counters and gauges.
+ * @param metadata optional metadata. If not null and not empty then this map will be added under a
+ * {@code _metadata_} key.
+ * @param consumer consumer that accepts produced {@link SolrInputDocument}-s
+ */
+ public static void toSolrInputDocuments(MetricRegistry registry, List<MetricFilter> shouldMatchFilters,
+ MetricFilter mustMatchFilter, Predicate<CharSequence> propertyFilter, boolean skipHistograms,
+ boolean skipAggregateValues, boolean compact,
+ Map<String, Object> metadata, Consumer<SolrInputDocument> consumer) {
boolean addMetadata = metadata != null && !metadata.isEmpty();
toMaps(registry, shouldMatchFilters, mustMatchFilter, propertyFilter, skipHistograms, skipAggregateValues, compact, false, (k, v) -> {
SolrInputDocument doc = new SolrInputDocument();
@@ -221,9 +259,32 @@ public class MetricUtils {
* @param simple use simplified representation for complex metrics - instead of a (name, map)
* only the selected (name "." key, value) pairs will be produced.
* @param consumer consumer that accepts produced objects
+ * @deprecated use {@link #toMaps(MetricRegistry, List, MetricFilter, Predicate, boolean, boolean, boolean, boolean, BiConsumer)} instead.
+ */
+ public static void toMaps(MetricRegistry registry, List<MetricFilter> shouldMatchFilters,
+ MetricFilter mustMatchFilter, PropertyFilter propertyFilter,
+ boolean skipHistograms, boolean skipAggregateValues,
+ boolean compact, boolean simple,
+ BiConsumer<String, Object> consumer) {
+ toMaps(registry, shouldMatchFilters, mustMatchFilter,
+ PropertyFilter.toPredicate(propertyFilter), skipHistograms,
+ skipAggregateValues, compact, simple, consumer);
+ }
+ /**
+ * Convert selected metrics to maps or to flattened objects.
+ * @param registry source of metrics
+ * @param shouldMatchFilters metrics must match any of these filters
+ * @param mustMatchFilter metrics must match this filter
+ * @param propertyFilter limit what properties of a metric are returned
+ * @param skipHistograms discard any {@link Histogram}-s and histogram parts of {@link Timer}-s.
+ * @param skipAggregateValues discard internal values of {@link AggregateMetric}-s.
+ * @param compact use compact representation for counters and gauges.
+ * @param simple use simplified representation for complex metrics - instead of a (name, map)
+ * only the selected (name "." key, value) pairs will be produced.
+ * @param consumer consumer that accepts produced objects
*/
public static void toMaps(MetricRegistry registry, List<MetricFilter> shouldMatchFilters,
- MetricFilter mustMatchFilter, PropertyFilter propertyFilter,
+ MetricFilter mustMatchFilter, Predicate<CharSequence> propertyFilter,
boolean skipHistograms, boolean skipAggregateValues,
boolean compact, boolean simple,
BiConsumer<String, Object> consumer) {
@@ -286,8 +347,26 @@ public class MetricUtils {
* @param simple use simplified representation for complex metrics - instead of a (name, map)
* only the selected (name "." key, value) pairs will be produced.
* @param consumer consumer that accepts produced objects
+ * @deprecated use {@link #convertMetric(String, Metric, Predicate, boolean, boolean, boolean, boolean, String, BiConsumer)} instead.
*/
public static void convertMetric(String n, Metric metric, PropertyFilter propertyFilter, boolean skipHistograms, boolean skipAggregateValues,
+ boolean compact, boolean simple, String separator, BiConsumer<String, Object> consumer) {
+ convertMetric(n, metric, PropertyFilter.toPredicate(propertyFilter),
+ skipHistograms, skipAggregateValues, compact, simple, separator, consumer);
+ }
+ /**
+ * Convert a single instance of metric into a map or flattened object.
+ * @param n metric name
+ * @param metric metric instance
+ * @param propertyFilter limit what properties of a metric are returned
+ * @param skipHistograms discard any {@link Histogram}-s and histogram parts of {@link Timer}-s.
+ * @param skipAggregateValues discard internal values of {@link AggregateMetric}-s.
+ * @param compact use compact representation for counters and gauges.
+ * @param simple use simplified representation for complex metrics - instead of a (name, map)
+ * only the selected (name "." key, value) pairs will be produced.
+ * @param consumer consumer that accepts produced objects
+ */
+ public static void convertMetric(String n, Metric metric, Predicate<CharSequence> propertyFilter, boolean skipHistograms, boolean skipAggregateValues,
boolean compact, boolean simple, String separator, BiConsumer<String, Object> consumer) {
if (metric instanceof Counter) {
Counter counter = (Counter) metric;
@@ -295,8 +374,16 @@ public class MetricUtils {
} else if (metric instanceof Gauge) {
@SuppressWarnings({"rawtypes"})
Gauge gauge = (Gauge) metric;
+ // unwrap if needed
+ if (gauge instanceof SolrMetricManager.GaugeWrapper) {
+ gauge = ((SolrMetricManager.GaugeWrapper) gauge).getGauge();
+ }
try {
- convertGauge(n, gauge, propertyFilter, simple, compact, separator, consumer);
+ if (gauge instanceof MapWriter) {
+ convertMapWriter(n, (MapWriter) gauge, propertyFilter, simple, compact, separator, consumer);
+ } else {
+ convertGauge(n, gauge, propertyFilter, simple, compact, separator, consumer);
+ }
} catch (InternalError ie) {
if (n.startsWith("memory.") && ie.getMessage().contains("Memory Pool not found")) {
log.warn("Error converting gauge '{}', possible JDK bug: SOLR-10362", n, ie);
@@ -332,16 +419,16 @@ public class MetricUtils {
* @param consumer consumer that accepts produced objects
*/
static void convertAggregateMetric(String name, AggregateMetric metric,
- PropertyFilter propertyFilter,
+ Predicate<CharSequence> propertyFilter,
boolean skipAggregateValues, boolean simple, String separator, BiConsumer<String, Object> consumer) {
if (simple) {
- if (propertyFilter.accept(MEAN)) {
+ if (propertyFilter.test(MEAN)) {
consumer.accept(name + separator + MEAN, metric.getMean());
}
} else {
MapWriter writer = ew -> {
BiConsumer<String, Object> filter = (k, v) -> {
- if (propertyFilter.accept(k)) {
+ if (propertyFilter.test(k)) {
ew.putNoEx(k, v);
}
};
@@ -362,7 +449,9 @@ public class MetricUtils {
});
}
};
- consumer.accept(name, writer);
+ if (writer._size() > 0) {
+ consumer.accept(name, writer);
+ }
}
}
@@ -376,17 +465,17 @@ public class MetricUtils {
* only the selected (name "." key, value) pairs will be produced.
* @param consumer consumer that accepts produced objects
*/
- static void convertHistogram(String name, Histogram histogram, PropertyFilter propertyFilter,
+ static void convertHistogram(String name, Histogram histogram, Predicate<CharSequence> propertyFilter,
boolean simple, String separator, BiConsumer<String, Object> consumer) {
Snapshot snapshot = histogram.getSnapshot();
if (simple) {
- if (propertyFilter.accept(MEAN)) {
+ if (propertyFilter.test(MEAN)) {
consumer.accept(name + separator + MEAN, snapshot.getMean());
}
} else {
MapWriter writer = ew -> {
String prop = "count";
- if (propertyFilter.accept(prop)) {
+ if (propertyFilter.test(prop)) {
ew.putNoEx(prop, histogram.getCount());
}
// non-time based values
@@ -406,9 +495,9 @@ public class MetricUtils {
}
// some snapshots represent time in ns, other snapshots represent raw values (eg. chunk size)
- static void addSnapshot(MapWriter.EntryWriter ew, Snapshot snapshot, PropertyFilter propertyFilter, boolean ms) {
+ static void addSnapshot(MapWriter.EntryWriter ew, Snapshot snapshot, Predicate<CharSequence> propertyFilter, boolean ms) {
BiConsumer<String, Object> filter = (k, v) -> {
- if (propertyFilter.accept(k)) {
+ if (propertyFilter.test(k)) {
ew.putNoEx(k, v);
}
};
@@ -432,18 +521,34 @@ public class MetricUtils {
* @param simple use simplified representation for complex metrics - instead of a (name, map)
* only the selected (name "." key, value) pairs will be produced.
* @param consumer consumer that accepts produced objects
+ * @deprecated use {@link #convertTimer(String, Timer, Predicate, boolean, boolean, String, BiConsumer)} instead.
*/
public static void convertTimer(String name, Timer timer, PropertyFilter propertyFilter, boolean skipHistograms,
+ boolean simple, String separator, BiConsumer<String, Object> consumer) {
+ convertTimer(name, timer, PropertyFilter.toPredicate(propertyFilter),
+ skipHistograms, simple, separator, consumer);
+ }
+ /**
+ * Convert a {@link Timer} to a map.
+ * @param name metric name
+ * @param timer timer instance
+ * @param propertyFilter limit what properties of a metric are returned
+ * @param skipHistograms if true then discard the histogram part of the timer.
+ * @param simple use simplified representation for complex metrics - instead of a (name, map)
+ * only the selected (name "." key, value) pairs will be produced.
+ * @param consumer consumer that accepts produced objects
+ */
+ public static void convertTimer(String name, Timer timer, Predicate<CharSequence> propertyFilter, boolean skipHistograms,
boolean simple, String separator, BiConsumer<String, Object> consumer) {
if (simple) {
String prop = "meanRate";
- if (propertyFilter.accept(prop)) {
+ if (propertyFilter.test(prop)) {
consumer.accept(name + separator + prop, timer.getMeanRate());
}
} else {
MapWriter writer = ew -> {
BiConsumer<String,Object> filter = (k, v) -> {
- if (propertyFilter.accept(k)) {
+ if (propertyFilter.test(k)) {
ew.putNoEx(k, v);
}
};
@@ -457,7 +562,9 @@ public class MetricUtils {
addSnapshot(ew, timer.getSnapshot(), propertyFilter, true);
}
};
- consumer.accept(name, writer);
+ if (writer._size() > 0) {
+ consumer.accept(name, writer);
+ }
}
}
@@ -470,15 +577,15 @@ public class MetricUtils {
* only the selected (name "." key, value) pairs will be produced.
* @param consumer consumer that accepts produced objects
*/
- static void convertMeter(String name, Meter meter, PropertyFilter propertyFilter, boolean simple, String separator, BiConsumer<String, Object> consumer) {
+ static void convertMeter(String name, Meter meter, Predicate<CharSequence> propertyFilter, boolean simple, String separator, BiConsumer<String, Object> consumer) {
if (simple) {
- if (propertyFilter.accept("count")) {
+ if (propertyFilter.test("count")) {
consumer.accept(name + separator + "count", meter.getCount());
}
} else {
MapWriter writer = ew -> {
BiConsumer<String, Object> filter = (k, v) -> {
- if (propertyFilter.accept(k)) {
+ if (propertyFilter.test(k)) {
ew.putNoEx(k, v);
}
};
@@ -488,10 +595,31 @@ public class MetricUtils {
filter.accept("5minRate", meter.getFiveMinuteRate());
filter.accept("15minRate", meter.getFifteenMinuteRate());
};
- consumer.accept(name, writer);
+ if (writer._size() > 0) {
+ consumer.accept(name, writer);
+ }
}
}
+ static void convertMapWriter(String name, MapWriter metric,
+ Predicate<CharSequence> propertyFilter, boolean simple, boolean compact,
+ String separator, BiConsumer<String, Object> consumer) {
+ ConditionalKeyMapWriter filteredMetric = new ConditionalKeyMapWriter(metric, propertyFilter);
+ if (compact || simple) {
+ if (simple) {
+ filteredMetric._forEachEntry((k, v) ->
+ consumer.accept(name + separator + k, v));
+ } else {
+ if (filteredMetric._size() > 0) {
+ consumer.accept(name, filteredMetric);
+ }
+ }
+ } else {
+ if (filteredMetric._size() > 0) {
+ consumer.accept(name, (MapWriter) ew -> ew.putNoEx("value", filteredMetric));
+ }
+ }
+ }
/**
* Convert a {@link Gauge}.
* @param name metric name
@@ -505,7 +633,7 @@ public class MetricUtils {
*/
static void convertGauge(String name,
@SuppressWarnings({"rawtypes"})Gauge gauge,
- PropertyFilter propertyFilter, boolean simple, boolean compact,
+ Predicate<CharSequence> propertyFilter, boolean simple, boolean compact,
String separator, BiConsumer<String, Object> consumer) {
if (compact || simple) {
Object o = gauge.getValue();
@@ -513,17 +641,17 @@ public class MetricUtils {
if (simple) {
for (Map.Entry<?, ?> entry : ((Map<?, ?>)o).entrySet()) {
String prop = entry.getKey().toString();
- if (propertyFilter.accept(prop)) {
+ if (propertyFilter.test(prop)) {
consumer.accept(name + separator + prop, entry.getValue());
}
}
} else {
boolean notEmpty = ((Map<?, ?>)o).entrySet().stream()
- .anyMatch(entry -> propertyFilter.accept(entry.getKey().toString()));
+ .anyMatch(entry -> propertyFilter.test(entry.getKey().toString()));
MapWriter writer = ew -> {
for (Map.Entry<?, ?> entry : ((Map<?, ?>)o).entrySet()) {
String prop = entry.getKey().toString();
- if (propertyFilter.accept(prop)) {
+ if (propertyFilter.test(prop)) {
ew.putNoEx(prop, entry.getValue());
}
}
@@ -539,13 +667,13 @@ public class MetricUtils {
Object o = gauge.getValue();
if (o instanceof Map) {
boolean notEmpty = ((Map<?, ?>)o).entrySet().stream()
- .anyMatch(entry -> propertyFilter.accept(entry.getKey().toString()));
+ .anyMatch(entry -> propertyFilter.test(entry.getKey().toString()));
if (notEmpty) {
consumer.accept(name, (MapWriter) ew -> {
ew.putNoEx("value", (MapWriter) ew1 -> {
for (Map.Entry<?, ?> entry : ((Map<?, ?>)o).entrySet()) {
String prop = entry.getKey().toString();
- if (propertyFilter.accept(prop)) {
+ if (propertyFilter.test(prop)) {
ew1.put(prop, entry.getValue());
}
}
@@ -553,7 +681,7 @@ public class MetricUtils {
});
}
} else {
- if (propertyFilter.accept("value")) {
+ if (propertyFilter.test("value")) {
consumer.accept(name, (MapWriter) ew -> ew.putNoEx("value", o));
}
}
@@ -567,11 +695,11 @@ public class MetricUtils {
* @param compact if true then only return {@link Counter#getCount()}. If false
* then return a map with a "count" field.
*/
- static void convertCounter(String name, Counter counter, PropertyFilter propertyFilter, boolean compact, BiConsumer<String, Object> consumer) {
+ static void convertCounter(String name, Counter counter, Predicate<CharSequence> propertyFilter, boolean compact, BiConsumer<String, Object> consumer) {
if (compact) {
consumer.accept(name, counter.getCount());
} else {
- if (propertyFilter.accept("count")) {
+ if (propertyFilter.test("count")) {
consumer.accept(name, (MapWriter) ew -> ew.putNoEx("count", counter.getCount()));
}
}
diff --git a/solr/core/src/test/org/apache/solr/core/MockInfoBean.java b/solr/core/src/test/org/apache/solr/core/MockInfoBean.java
index ce73a02..1217283 100644
--- a/solr/core/src/test/org/apache/solr/core/MockInfoBean.java
+++ b/solr/core/src/test/org/apache/solr/core/MockInfoBean.java
@@ -50,7 +50,7 @@ class MockInfoBean implements SolrInfoBean, SolrMetricProducer {
@Override
public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
solrMetricsContext = parentContext.getChildContext(this);
- MetricsMap metricsMap = new MetricsMap((detailed, map) -> {
+ MetricsMap metricsMap = new MetricsMap(map -> {
map.put("Integer", 123);
map.put("Double",567.534);
map.put("Long", 32352463l);
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
index 525c63c..8f2bdba 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
@@ -463,7 +463,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
@Override
public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
super.initializeMetrics(parentContext, scope);
- MetricsMap metrics = new MetricsMap((detailed, map) -> map.putAll(gaugevals));
+ MetricsMap metrics = new MetricsMap(map -> gaugevals.forEach((k, v) -> map.putNoEx(k, v)));
solrMetricsContext.gauge(
metrics, true, "dumphandlergauge", getCategory().toString(), scope);
diff --git a/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java b/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java
index efc6c94..0405727 100644
--- a/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java
+++ b/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java
@@ -34,6 +34,8 @@ import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.metrics.AggregateMetric;
+import org.apache.solr.metrics.MetricsMap;
+import org.apache.solr.metrics.SolrMetricManager;
import org.junit.Test;
public class MetricUtilsTest extends SolrTestCaseJ4 {
@@ -104,6 +106,19 @@ public class MetricUtilsTest extends SolrTestCaseJ4 {
registry.register("gauge", gauge);
Gauge<Long> error = () -> {throw new InternalError("Memory Pool not found error");};
registry.register("memory.expected.error", error);
+
+ MetricsMap metricsMapWithMap = new MetricsMap((detailed, map) -> {
+ map.put("foo", "bar");
+ });
+ registry.register("mapWithMap", metricsMapWithMap);
+ MetricsMap metricsMap = new MetricsMap(map -> {
+ map.putNoEx("foo", "bar");
+ });
+ registry.register("map", metricsMap);
+
+ SolrMetricManager.GaugeWrapper<MapWriter> gaugeWrapper = new SolrMetricManager.GaugeWrapper(metricsMap, "foo-tag");
+ registry.register("wrappedGauge", gaugeWrapper);
+
MetricUtils.toMaps(registry, Collections.singletonList(MetricFilter.ALL), MetricFilter.ALL,
MetricUtils.PropertyFilter.ALL, false, false, false, false, (k, o) -> {
@SuppressWarnings({"rawtypes"})
@@ -144,6 +159,10 @@ public class MetricUtilsTest extends SolrTestCaseJ4 {
assertEquals(0D, v.get("mean"));
} else if (k.startsWith("memory.expected.error")) {
assertTrue(v.isEmpty());
+ } else if (k.startsWith("map") || k.startsWith("wrapped")) {
+ assertNotNull(v.toString(), v.get("value"));
+ assertTrue(v.toString(), v.get("value") instanceof Map);
+ assertEquals(v.toString(), "bar", ((Map) v.get("value")).get("foo"));
}
});
// test compact format
@@ -201,6 +220,11 @@ public class MetricUtilsTest extends SolrTestCaseJ4 {
assertEquals(1, update.get("updateCount"));
} else if (k.startsWith("memory.expected.error")) {
assertNull(o);
+ } else if (k.startsWith("map") || k.startsWith("wrapped")) {
+ assertTrue(o instanceof MapWriter);
+ MapWriter writer = (MapWriter) o;
+ assertEquals(1, writer._size());
+ assertEquals("bar", writer._get("foo", null));
} else {
assertTrue(o instanceof MapWriter);
Map<String, Object> v = new HashMap<>();