You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2020/10/07 18:45:43 UTC
[lucene-solr] branch branch_8x updated: SOLR-14691: Metrics
reporting should avoid creating objects.
This is an automated email from the ASF dual-hosted git repository.
ab pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8x by this push:
new 5c4168d SOLR-14691: Metrics reporting should avoid creating objects.
5c4168d is described below
commit 5c4168d3b2e1c05a09b608fac048be26ae8050d4
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Wed Oct 7 20:45:13 2020 +0200
SOLR-14691: Metrics reporting should avoid creating objects.
---
solr/CHANGES.txt | 2 +
.../apache/solr/handler/ReplicationHandler.java | 11 +-
.../apache/solr/handler/RequestHandlerBase.java | 4 +-
.../apache/solr/handler/admin/MetricsHandler.java | 24 +-
.../solr/handler/component/SuggestComponent.java | 4 +-
.../java/org/apache/solr/metrics/MetricsMap.java | 69 +++-
.../solr/metrics/reporters/solr/SolrReporter.java | 2 +-
.../java/org/apache/solr/search/CaffeineCache.java | 2 +-
.../org/apache/solr/search/SolrFieldCacheBean.java | 4 +-
.../org/apache/solr/search/SolrIndexSearcher.java | 4 +-
.../apache/solr/servlet/SolrDispatchFilter.java | 4 +-
.../org/apache/solr/store/blockcache/Metrics.java | 2 +-
.../solr/store/hdfs/HdfsLocalityReporter.java | 2 +-
.../org/apache/solr/util/stats/MetricUtils.java | 380 +++++++++++++++------
.../test/org/apache/solr/core/MockInfoBean.java | 2 +-
.../solr/handler/admin/MetricsHandlerTest.java | 63 ++--
.../apache/solr/util/stats/MetricUtilsTest.java | 75 ++--
.../java/org/apache/solr/common/MapWriterMap.java | 5 +
.../org/apache/solr/common/NavigableObject.java | 6 +
.../org/apache/solr/common/util/NamedList.java | 5 +
.../java/org/apache/solr/common/util/Utils.java | 58 +++-
21 files changed, 510 insertions(+), 218 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 882db89..a8f290b 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -65,6 +65,8 @@ Improvements
* SOLR-14905: Update commons-io version to 2.8.0 due to security vulnerability. (Nazerke Seidan via Bruno Roustant)
+* SOLR-14691: Metrics reporting should avoid creating objects. (ab, Noble Paul)
+
Optimizations
---------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
index b3ae624..ce850ac 100644
--- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
@@ -70,6 +70,7 @@ import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RateLimiter;
import org.apache.lucene.util.Version;
+import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.CommonParams;
@@ -930,10 +931,10 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
solrMetricsContext.gauge(this, () -> (core != null && !core.isClosed() ? core.getIndexDir() : ""),
true, "indexPath", getCategory().toString(), scope);
solrMetricsContext.gauge(this, () -> isLeader,
- true, "isMaster", getCategory().toString(), scope);
+ true, "isLeader", getCategory().toString(), scope);
solrMetricsContext.gauge(this, () -> isFollower,
- true, "isSlave", getCategory().toString(), scope);
- final MetricsMap fetcherMap = new MetricsMap((detailed, map) -> {
+ true, "isFollower", getCategory().toString(), scope);
+ final MetricsMap fetcherMap = new MetricsMap(map -> {
IndexFetcher fetcher = currentIndexFetcher;
if (fetcher != null) {
map.put(LEGACY_LEADER_URL, fetcher.getLeaderUrl());
@@ -1149,10 +1150,10 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
}
}
- private void addVal(Map<String, Object> map, String key, Properties props, @SuppressWarnings({"rawtypes"})Class clzz) {
+ private void addVal(MapWriter.EntryWriter ew, String key, Properties props, @SuppressWarnings({"rawtypes"})Class clzz) {
Object val = formatVal(key, props, clzz);
if (val != null) {
- map.put(key, val);
+ ew.putNoEx(key, val);
}
}
diff --git a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
index fe5119c..c2dbd0e 100644
--- a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
+++ b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
@@ -159,8 +159,8 @@ public abstract class RequestHandlerBase implements SolrRequestHandler, SolrInfo
numClientErrors = solrMetricsContext.meter(this, "clientErrors", getCategory().toString(), scope);
numTimeouts = solrMetricsContext.meter(this, "timeouts", getCategory().toString(), scope);
requests = solrMetricsContext.counter(this, "requests", getCategory().toString(), scope);
- MetricsMap metricsMap = new MetricsMap((detail, map) ->
- shardPurposes.forEach((k, v) -> map.put(k, v.getCount())));
+ MetricsMap metricsMap = new MetricsMap(map ->
+ shardPurposes.forEach((k, v) -> map.putNoEx(k, v.getCount())));
solrMetricsContext.gauge(this, metricsMap, true, "shardRequests", getCategory().toString(), scope);
requestTimes = solrMetricsContext.timer(this, "requestTimes", getCategory().toString(), scope);
distribRequestTimes = solrMetricsContext.timer(this, "requestTimes", getCategory().toString(), scope, "distrib");
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
index e6d8017..76bb04d 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.BiConsumer;
+import java.util.function.Predicate;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -36,6 +37,7 @@ import com.codahale.metrics.Metric;
import com.codahale.metrics.MetricFilter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
+import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.CommonTestInjection;
@@ -112,17 +114,15 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
return;
}
MetricFilter mustMatchFilter = parseMustMatchFilter(params);
- MetricUtils.PropertyFilter propertyFilter = parsePropertyFilter(params);
+ Predicate<CharSequence> propertyFilter = parsePropertyFilter(params);
List<MetricType> metricTypes = parseMetricTypes(params);
List<MetricFilter> metricFilters = metricTypes.stream().map(MetricType::asMetricFilter).collect(Collectors.toList());
Set<String> requestedRegistries = parseRegistries(params);
- @SuppressWarnings({"rawtypes"})
- NamedList response = new SimpleOrderedMap();
+ NamedList<Object> response = new SimpleOrderedMap<>();
for (String registryName : requestedRegistries) {
MetricRegistry registry = metricManager.registry(registryName);
- @SuppressWarnings({"rawtypes"})
- SimpleOrderedMap result = new SimpleOrderedMap();
+ SimpleOrderedMap<Object> result = new SimpleOrderedMap<>();
MetricUtils.toMaps(registry, metricFilters, mustMatchFilter, propertyFilter, false,
false, compact, false, (k, v) -> result.add(k, v));
if (result.size() > 0) {
@@ -134,8 +134,8 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
@SuppressWarnings({"unchecked", "rawtypes"})
public void handleKeyRequest(String[] keys, BiConsumer<String, Object> consumer) throws Exception {
- SimpleOrderedMap result = new SimpleOrderedMap();
- SimpleOrderedMap errors = new SimpleOrderedMap();
+ SimpleOrderedMap<Object> result = new SimpleOrderedMap<>();
+ SimpleOrderedMap<Object> errors = new SimpleOrderedMap<>();
for (String key : keys) {
if (key == null || key.isEmpty()) {
continue;
@@ -158,7 +158,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
errors.add(key, "metric '" + metricName + "' not found");
continue;
}
- MetricUtils.PropertyFilter propertyFilter = MetricUtils.PropertyFilter.ALL;
+ Predicate<CharSequence> propertyFilter = MetricUtils.ALL_PROPERTIES;
if (propertyName != null) {
propertyFilter = (name) -> name.equals(propertyName);
// use escaped versions
@@ -173,6 +173,8 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
MetricUtils.convertMetric(key, m, propertyFilter, false, true, true, false, ":", (k, v) -> {
if ((v instanceof Map) && propertyName != null) {
((Map)v).forEach((k1, v1) -> result.add(k + ":" + k1, v1));
+ } else if ((v instanceof MapWriter) && propertyName != null) {
+ ((MapWriter) v)._forEachEntry((k1, v1) -> result.add(k + ":" + k1, v1));
} else {
result.add(k, v);
}
@@ -229,10 +231,10 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
return mustMatchFilter;
}
- private MetricUtils.PropertyFilter parsePropertyFilter(SolrParams params) {
+ private Predicate<CharSequence> parsePropertyFilter(SolrParams params) {
String[] props = params.getParams(PROPERTY_PARAM);
if (props == null || props.length == 0) {
- return MetricUtils.PropertyFilter.ALL;
+ return MetricUtils.ALL_PROPERTIES;
}
final Set<String> filter = new HashSet<>();
for (String prop : props) {
@@ -241,7 +243,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
}
}
if (filter.isEmpty()) {
- return MetricUtils.PropertyFilter.ALL;
+ return MetricUtils.ALL_PROPERTIES;
} else {
return (name) -> filter.contains(name);
}
diff --git a/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java b/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java
index 0c26d4c..6f52a23 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java
@@ -364,10 +364,10 @@ public class SuggestComponent extends SearchComponent implements SolrCoreAware,
this.metricsContext = parentContext.getChildContext(this);
this.metricsContext.gauge(this, () -> ramBytesUsed(), true, "totalSizeInBytes", getCategory().toString());
- MetricsMap suggestersMap = new MetricsMap((detailed, map) -> {
+ MetricsMap suggestersMap = new MetricsMap(map -> {
for (Map.Entry<String, SolrSuggester> entry : suggesters.entrySet()) {
SolrSuggester suggester = entry.getValue();
- map.put(entry.getKey(), suggester.toString());
+ map.putNoEx(entry.getKey(), suggester.toString());
}
});
this.metricsContext.gauge(this, suggestersMap, true, "suggesters", getCategory().toString(), scope);
diff --git a/solr/core/src/java/org/apache/solr/metrics/MetricsMap.java b/solr/core/src/java/org/apache/solr/metrics/MetricsMap.java
index bd9abaf..cd1ed4d 100644
--- a/solr/core/src/java/org/apache/solr/metrics/MetricsMap.java
+++ b/solr/core/src/java/org/apache/solr/metrics/MetricsMap.java
@@ -28,6 +28,7 @@ import javax.management.ReflectionException;
import javax.management.openmbean.OpenMBeanAttributeInfoSupport;
import javax.management.openmbean.OpenType;
import javax.management.openmbean.SimpleType;
+import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.lang.reflect.Field;
import java.util.ArrayList;
@@ -38,6 +39,7 @@ import java.util.function.BiConsumer;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Metric;
import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,18 +53,33 @@ import org.slf4j.LoggerFactory;
* {@link javax.management.openmbean.OpenType#ALLOWED_CLASSNAMES_LIST}, otherwise only their toString()
* representation will be shown in JConsole.</p>
*/
-public class MetricsMap implements Gauge<Map<String,Object>>, DynamicMBean {
+public class MetricsMap implements Gauge<Map<String,Object>>, MapWriter, DynamicMBean {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
// set to true to use cached statistics between getMBeanInfo calls to work
// around over calling getStatistics on MBeanInfos when iterating over all attributes (SOLR-6586)
private final boolean useCachedStatsBetweenGetMBeanInfoCalls = Boolean.getBoolean("useCachedStatsBetweenGetMBeanInfoCalls");
- private BiConsumer<Boolean, Map<String, Object>> initializer;
- private Map<String, String> jmxAttributes = new HashMap<>();
+ private BiConsumer<Boolean, Map<String, Object>> mapInitializer;
+ private MapWriter initializer;
+ private Map<String, String> jmxAttributes;
private volatile Map<String,Object> cachedValue;
- public MetricsMap(BiConsumer<Boolean, Map<String,Object>> initializer) {
+ /**
+ * Create an instance that reports values to a Map.
+ * @param mapInitializer function to populate the Map result.
+ * @deprecated use {@link #MetricsMap(MapWriter)} instead.
+ */
+ @Deprecated()
+ public MetricsMap(BiConsumer<Boolean, Map<String,Object>> mapInitializer) {
+ this.mapInitializer = mapInitializer;
+ }
+
+ /**
+ * Create an instance that reports values to a MapWriter.
+ * @param initializer function to populate the MapWriter result.
+ */
+ public MetricsMap(MapWriter initializer) {
this.initializer = initializer;
}
@@ -73,7 +90,11 @@ public class MetricsMap implements Gauge<Map<String,Object>>, DynamicMBean {
public Map<String,Object> getValue(boolean detailed) {
Map<String,Object> map = new HashMap<>();
- initializer.accept(detailed, map);
+ if (mapInitializer != null) {
+ mapInitializer.accept(detailed, map);
+ } else {
+ initializer.toMap(map);
+ }
return map;
}
@@ -81,13 +102,22 @@ public class MetricsMap implements Gauge<Map<String,Object>>, DynamicMBean {
return getValue().toString();
}
+ // lazy init
+ private synchronized void initJmxAttributes() {
+ if (jmxAttributes == null) {
+ jmxAttributes = new HashMap<>();
+ }
+ }
+
@Override
public Object getAttribute(String attribute) throws AttributeNotFoundException, MBeanException, ReflectionException {
Object val;
// jmxAttributes override any real values
- val = jmxAttributes.get(attribute);
- if (val != null) {
- return val;
+ if (jmxAttributes != null) {
+ val = jmxAttributes.get(attribute);
+ if (val != null) {
+ return val;
+ }
}
Map<String,Object> stats = null;
if (useCachedStatsBetweenGetMBeanInfoCalls) {
@@ -117,6 +147,7 @@ public class MetricsMap implements Gauge<Map<String,Object>>, DynamicMBean {
@Override
public void setAttribute(Attribute attribute) throws AttributeNotFoundException, InvalidAttributeValueException, MBeanException, ReflectionException {
+ initJmxAttributes();
jmxAttributes.put(attribute.getName(), String.valueOf(attribute.getValue()));
}
@@ -150,13 +181,15 @@ public class MetricsMap implements Gauge<Map<String,Object>>, DynamicMBean {
if (useCachedStatsBetweenGetMBeanInfoCalls) {
cachedValue = stats;
}
- jmxAttributes.forEach((k, v) -> {
- attrInfoList.add(new MBeanAttributeInfo(k, String.class.getName(),
- null, true, false, false));
- });
+ if (jmxAttributes != null) {
+ jmxAttributes.forEach((k, v) -> {
+ attrInfoList.add(new MBeanAttributeInfo(k, String.class.getName(),
+ null, true, false, false));
+ });
+ }
try {
stats.forEach((k, v) -> {
- if (jmxAttributes.containsKey(k)) {
+ if (jmxAttributes != null && jmxAttributes.containsKey(k)) {
return;
}
@SuppressWarnings({"rawtypes"})
@@ -197,4 +230,14 @@ public class MetricsMap implements Gauge<Map<String,Object>>, DynamicMBean {
}
return null;
}
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ if (mapInitializer != null) {
+ Map<String, Object> value = getValue();
+ value.forEach((k, v) -> ew.putNoEx(k, v));
+ } else {
+ initializer.writeMap(ew);
+ }
+ }
}
\ No newline at end of file
diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java
index 929aa93..9606ef5 100644
--- a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java
+++ b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java
@@ -453,7 +453,7 @@ public class SolrReporter extends ScheduledReporter {
}
final String effectiveGroup = group;
MetricUtils.toSolrInputDocuments(metricManager.registry(registryName), Collections.singletonList(report.filter), MetricFilter.ALL,
- MetricUtils.PropertyFilter.ALL, skipHistograms, skipAggregateValues, compact, metadata, doc -> {
+ MetricUtils.ALL_PROPERTIES, skipHistograms, skipAggregateValues, compact, metadata, doc -> {
doc.setField(REGISTRY_ID, registryName);
doc.setField(GROUP_ID, effectiveGroup);
if (effectiveLabel != null) {
diff --git a/solr/core/src/java/org/apache/solr/search/CaffeineCache.java b/solr/core/src/java/org/apache/solr/search/CaffeineCache.java
index 52df874..f9a74e9 100644
--- a/solr/core/src/java/org/apache/solr/search/CaffeineCache.java
+++ b/solr/core/src/java/org/apache/solr/search/CaffeineCache.java
@@ -369,7 +369,7 @@ public class CaffeineCache<K, V> extends SolrCacheBase implements SolrCache<K, V
@Override
public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
solrMetricsContext = parentContext.getChildContext(this);
- cacheMap = new MetricsMap((detailed, map) -> {
+ cacheMap = new MetricsMap(map -> {
if (cache != null) {
CacheStats stats = cache.stats();
long insertCount = inserts.sum();
diff --git a/solr/core/src/java/org/apache/solr/search/SolrFieldCacheBean.java b/solr/core/src/java/org/apache/solr/search/SolrFieldCacheBean.java
index b6deb7c..ba2c8e2 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrFieldCacheBean.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrFieldCacheBean.java
@@ -58,8 +58,8 @@ public class SolrFieldCacheBean implements SolrInfoBean, SolrMetricProducer {
@Override
public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
this.solrMetricsContext = parentContext;
- MetricsMap metricsMap = new MetricsMap((detailed, map) -> {
- if (detailed && !disableEntryList && !disableJmxEntryList) {
+ MetricsMap metricsMap = new MetricsMap(map -> {
+ if (!disableEntryList && !disableJmxEntryList) {
UninvertingReader.FieldCacheStats fieldCacheStats = UninvertingReader.getUninvertedStats();
String[] entries = fieldCacheStats.info;
map.put("entries_count", entries.length);
diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
index 89aefad..3233a99 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
@@ -2300,8 +2300,8 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
}, true, "indexCommitSize", Category.SEARCHER.toString(), scope);
// statsCache metrics
parentContext.gauge(this,
- new MetricsMap((detailed, map) -> {
- statsCache.getCacheMetrics().getSnapshot(map::put);
+ new MetricsMap(map -> {
+ statsCache.getCacheMetrics().getSnapshot(map::putNoEx);
map.put("statsCacheImpl", statsCache.getClass().getSimpleName());
}), true, "statsCache", Category.CACHE.toString(), scope);
}
diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
index 4d99854..4583692 100644
--- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
+++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
@@ -215,10 +215,10 @@ public class SolrDispatchFilter extends BaseSolrFilter {
metricManager.registerAll(registryName, new GarbageCollectorMetricSet(), SolrMetricManager.ResolutionStrategy.IGNORE, "gc");
metricManager.registerAll(registryName, new MemoryUsageGaugeSet(), SolrMetricManager.ResolutionStrategy.IGNORE, "memory");
metricManager.registerAll(registryName, new ThreadStatesGaugeSet(), SolrMetricManager.ResolutionStrategy.IGNORE, "threads"); // todo should we use CachedThreadStatesGaugeSet instead?
- MetricsMap sysprops = new MetricsMap((detailed, map) -> {
+ MetricsMap sysprops = new MetricsMap(map -> {
System.getProperties().forEach((k, v) -> {
if (!hiddenSysProps.contains(k)) {
- map.put(String.valueOf(k), v);
+ map.putNoEx(String.valueOf(k), v);
}
});
});
diff --git a/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java b/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java
index d7654b6..5ad34f3 100644
--- a/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java
+++ b/solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java
@@ -60,7 +60,7 @@ public class Metrics extends SolrCacheBase implements SolrInfoBean, SolrMetricPr
@Override
public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
solrMetricsContext = parentContext.getChildContext(this);
- metricsMap = new MetricsMap((detailed, map) -> {
+ metricsMap = new MetricsMap(map -> {
long now = System.nanoTime();
long delta = Math.max(now - previous, 1);
double seconds = delta / 1000000000.0;
diff --git a/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLocalityReporter.java b/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLocalityReporter.java
index d4a0b37..12a16b0 100644
--- a/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLocalityReporter.java
+++ b/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLocalityReporter.java
@@ -105,7 +105,7 @@ public class HdfsLocalityReporter implements SolrInfoBean, SolrMetricProducer {
@Override
public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
solrMetricsContext = parentContext.getChildContext(this);
- MetricsMap metricsMap = new MetricsMap((detailed, map) -> {
+ MetricsMap metricsMap = new MetricsMap(map -> {
long totalBytes = 0;
long localBytes = 0;
int totalCount = 0;
diff --git a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java
index a604c83..46cbfbe 100644
--- a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java
+++ b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java
@@ -20,14 +20,13 @@ import java.beans.BeanInfo;
import java.beans.IntrospectionException;
import java.beans.Introspector;
import java.beans.PropertyDescriptor;
+import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.lang.management.OperatingSystemMXBean;
import java.lang.management.PlatformManagedObject;
import java.lang.reflect.InvocationTargetException;
import java.util.Collection;
-import java.util.Collections;
import java.util.HashMap;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.SortedSet;
@@ -35,6 +34,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
+import java.util.function.Predicate;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
@@ -46,10 +46,14 @@ import com.codahale.metrics.MetricFilter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Snapshot;
import com.codahale.metrics.Timer;
+import org.apache.solr.common.ConditionalKeyMapWriter;
+import org.apache.solr.common.IteratorWriter;
+import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.metrics.AggregateMetric;
+import org.apache.solr.metrics.SolrMetricManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -87,7 +91,9 @@ public class MetricUtils {
/**
* This filter can limit what properties of a metric are returned.
+ * @deprecated use {@link Predicate} instead.
*/
+ @Deprecated()
public interface PropertyFilter {
PropertyFilter ALL = (name) -> true;
@@ -97,8 +103,14 @@ public class MetricUtils {
* @return true if this property should be returned, false otherwise.
*/
boolean accept(String name);
+
+ static Predicate<CharSequence> toPredicate(PropertyFilter filter) {
+ return (name) -> filter.accept(name.toString());
+ }
}
+ public static final Predicate<CharSequence> ALL_PROPERTIES = (name) -> true;
+
/**
* Adds metrics from a Timer to a NamedList, using well-known back-compat names.
* @param lst The NamedList to add the metrics data to
@@ -145,11 +157,40 @@ public class MetricUtils {
* @param metadata optional metadata. If not null and not empty then this map will be added under a
* {@code _metadata_} key.
* @param consumer consumer that accepts produced {@link SolrInputDocument}-s
+ * @deprecated use {@link #toSolrInputDocuments(MetricRegistry, List, MetricFilter, Predicate, boolean, boolean, boolean, Map, Consumer)} instead.
*/
+ @Deprecated()
public static void toSolrInputDocuments(MetricRegistry registry, List<MetricFilter> shouldMatchFilters,
MetricFilter mustMatchFilter, PropertyFilter propertyFilter, boolean skipHistograms,
boolean skipAggregateValues, boolean compact,
Map<String, Object> metadata, Consumer<SolrInputDocument> consumer) {
+ toSolrInputDocuments(registry, shouldMatchFilters, mustMatchFilter,
+ PropertyFilter.toPredicate(propertyFilter), skipHistograms,
+ skipAggregateValues, compact, metadata, consumer);
+ }
+ /**
+ * Provides a representation of the given metric registry as {@link SolrInputDocument}-s.
+ Only those metrics
+ * are converted which match at least one of the given MetricFilter instances.
+ *
+ * @param registry the {@link MetricRegistry} to be converted
+ * @param shouldMatchFilters a list of {@link MetricFilter} instances.
+ * A metric must match <em>any one</em> of the filters from this list to be
+ * included in the output
+ * @param mustMatchFilter a {@link MetricFilter}.
+ * A metric <em>must</em> match this filter to be included in the output.
+ * @param propertyFilter limit what properties of a metric are returned
+ * @param skipHistograms discard any {@link Histogram}-s and histogram parts of {@link Timer}-s.
+ * @param skipAggregateValues discard internal values of {@link AggregateMetric}-s.
+ * @param compact use compact representation for counters and gauges.
+ * @param metadata optional metadata. If not null and not empty then this map will be added under a
+ * {@code _metadata_} key.
+ * @param consumer consumer that accepts produced {@link SolrInputDocument}-s
+ */
+ public static void toSolrInputDocuments(MetricRegistry registry, List<MetricFilter> shouldMatchFilters,
+ MetricFilter mustMatchFilter, Predicate<CharSequence> propertyFilter, boolean skipHistograms,
+ boolean skipAggregateValues, boolean compact,
+ Map<String, Object> metadata, Consumer<SolrInputDocument> consumer) {
boolean addMetadata = metadata != null && !metadata.isEmpty();
toMaps(registry, shouldMatchFilters, mustMatchFilter, propertyFilter, skipHistograms, skipAggregateValues, compact, false, (k, v) -> {
SolrInputDocument doc = new SolrInputDocument();
@@ -169,20 +210,42 @@ public class MetricUtils {
* @param o an instance of converted metric, either a Map or a flat Object
*/
static void toSolrInputDocument(String prefix, SolrInputDocument doc, Object o) {
- if (!(o instanceof Map)) {
- String key = prefix != null ? prefix : VALUE;
- doc.addField(key, o);
- return;
- }
- @SuppressWarnings({"unchecked"})
- Map<String, Object> map = (Map<String, Object>)o;
- for (Map.Entry<String, Object> entry : map.entrySet()) {
- if (entry.getValue() instanceof Map) { // flatten recursively
- toSolrInputDocument(entry.getKey(), doc, entry.getValue());
+ final BiConsumer<Object, Object> consumer = (k, v) -> {
+ if ((v instanceof Map) || (v instanceof MapWriter) || (v instanceof IteratorWriter)) {
+ toSolrInputDocument(k.toString(), doc, v);
} else {
- String key = prefix != null ? prefix + "." + entry.getKey() : entry.getKey();
- doc.addField(key, entry.getValue());
+ String key = prefix != null ? prefix + "." + k : k.toString();
+ doc.addField(key, v);
+ }
+ };
+ if (o instanceof MapWriter) {
+ @SuppressWarnings({"unchecked"})
+ MapWriter writer = (MapWriter) o;
+ writer._forEachEntry(consumer);
+ } else if (o instanceof Map) {
+ @SuppressWarnings({"unchecked"})
+ Map<String, Object> map = (Map<String, Object>) o;
+ for (Map.Entry<String, Object> entry : map.entrySet()) {
+ consumer.accept(entry.getKey(), entry.getValue());
}
+ } else if (o instanceof IteratorWriter) {
+ @SuppressWarnings({"unchecked"})
+ IteratorWriter writer = (IteratorWriter) o;
+ final String name = prefix != null ? prefix : "value";
+ try {
+ writer.writeIter(new IteratorWriter.ItemWriter() {
+ @Override
+ public IteratorWriter.ItemWriter add(Object o) throws IOException {
+ consumer.accept(name, o);
+ return this;
+ }
+ });
+ } catch (IOException e) {
+ throw new RuntimeException("this should never happen", e);
+ }
+ } else {
+ String key = prefix != null ? prefix : VALUE;
+ doc.addField(key, o);
}
}
@@ -198,9 +261,33 @@ public class MetricUtils {
* @param simple use simplified representation for complex metrics - instead of a (name, map)
* only the selected (name "." key, value) pairs will be produced.
* @param consumer consumer that accepts produced objects
+ * @deprecated use {@link #toMaps(MetricRegistry, List, MetricFilter, Predicate, boolean, boolean, boolean, boolean, BiConsumer)} instead.
+ */
+ @Deprecated()
+ public static void toMaps(MetricRegistry registry, List<MetricFilter> shouldMatchFilters,
+ MetricFilter mustMatchFilter, PropertyFilter propertyFilter,
+ boolean skipHistograms, boolean skipAggregateValues,
+ boolean compact, boolean simple,
+ BiConsumer<String, Object> consumer) {
+ toMaps(registry, shouldMatchFilters, mustMatchFilter,
+ PropertyFilter.toPredicate(propertyFilter), skipHistograms,
+ skipAggregateValues, compact, simple, consumer);
+ }
+ /**
+ * Convert selected metrics to maps or to flattened objects.
+ * @param registry source of metrics
+ * @param shouldMatchFilters metrics must match any of these filters
+ * @param mustMatchFilter metrics must match this filter
+ * @param propertyFilter limit what properties of a metric are returned
+ * @param skipHistograms discard any {@link Histogram}-s and histogram parts of {@link Timer}-s.
+ * @param skipAggregateValues discard internal values of {@link AggregateMetric}-s.
+ * @param compact use compact representation for counters and gauges.
+ * @param simple use simplified representation for complex metrics - instead of a (name, map)
+ * only the selected (name "." key, value) pairs will be produced.
+ * @param consumer consumer that accepts produced objects
*/
public static void toMaps(MetricRegistry registry, List<MetricFilter> shouldMatchFilters,
- MetricFilter mustMatchFilter, PropertyFilter propertyFilter,
+ MetricFilter mustMatchFilter, Predicate<CharSequence> propertyFilter,
boolean skipHistograms, boolean skipAggregateValues,
boolean compact, boolean simple,
BiConsumer<String, Object> consumer) {
@@ -263,8 +350,27 @@ public class MetricUtils {
* @param simple use simplified representation for complex metrics - instead of a (name, map)
* only the selected (name "." key, value) pairs will be produced.
* @param consumer consumer that accepts produced objects
+ * @deprecated use {@link #convertMetric(String, Metric, Predicate, boolean, boolean, boolean, boolean, String, BiConsumer)} instead.
*/
+ @Deprecated()
public static void convertMetric(String n, Metric metric, PropertyFilter propertyFilter, boolean skipHistograms, boolean skipAggregateValues,
+ boolean compact, boolean simple, String separator, BiConsumer<String, Object> consumer) {
+ convertMetric(n, metric, PropertyFilter.toPredicate(propertyFilter),
+ skipHistograms, skipAggregateValues, compact, simple, separator, consumer);
+ }
+ /**
+ * Convert a single instance of metric into a map or flattened object.
+ * @param n metric name
+ * @param metric metric instance
+ * @param propertyFilter limit what properties of a metric are returned
+ * @param skipHistograms discard any {@link Histogram}-s and histogram parts of {@link Timer}-s.
+ * @param skipAggregateValues discard internal values of {@link AggregateMetric}-s.
+ * @param compact use compact representation for counters and gauges.
+ * @param simple use simplified representation for complex metrics - instead of a (name, map)
+ * only the selected (name "." key, value) pairs will be produced.
+ * @param consumer consumer that accepts produced objects
+ */
+ public static void convertMetric(String n, Metric metric, Predicate<CharSequence> propertyFilter, boolean skipHistograms, boolean skipAggregateValues,
boolean compact, boolean simple, String separator, BiConsumer<String, Object> consumer) {
if (metric instanceof Counter) {
Counter counter = (Counter) metric;
@@ -272,8 +378,16 @@ public class MetricUtils {
} else if (metric instanceof Gauge) {
@SuppressWarnings({"rawtypes"})
Gauge gauge = (Gauge) metric;
+ // unwrap if needed
+ if (gauge instanceof SolrMetricManager.GaugeWrapper) {
+ gauge = ((SolrMetricManager.GaugeWrapper) gauge).getGauge();
+ }
try {
- convertGauge(n, gauge, propertyFilter, simple, compact, separator, consumer);
+ if (gauge instanceof MapWriter) {
+ convertMapWriter(n, (MapWriter) gauge, propertyFilter, simple, compact, separator, consumer);
+ } else {
+ convertGauge(n, gauge, propertyFilter, simple, compact, separator, consumer);
+ }
} catch (InternalError ie) {
if (n.startsWith("memory.") && ie.getMessage().contains("Memory Pool not found")) {
log.warn("Error converting gauge '{}', possible JDK bug: SOLR-10362", n, ie);
@@ -309,37 +423,38 @@ public class MetricUtils {
* @param consumer consumer that accepts produced objects
*/
static void convertAggregateMetric(String name, AggregateMetric metric,
- PropertyFilter propertyFilter,
+ Predicate<CharSequence> propertyFilter,
boolean skipAggregateValues, boolean simple, String separator, BiConsumer<String, Object> consumer) {
if (simple) {
- if (propertyFilter.accept(MEAN)) {
+ if (propertyFilter.test(MEAN)) {
consumer.accept(name + separator + MEAN, metric.getMean());
}
} else {
- Map<String, Object> response = new LinkedHashMap<>();
- BiConsumer<String, Object> filter = (k, v) -> {
- if (propertyFilter.accept(k)) {
- response.put(k, v);
+ MapWriter writer = ew -> {
+ BiConsumer<String, Object> filter = (k, v) -> {
+ if (propertyFilter.test(k)) {
+ ew.putNoEx(k, v);
+ }
+ };
+ filter.accept("count", metric.size());
+ filter.accept(MAX, metric.getMax());
+ filter.accept(MIN, metric.getMin());
+ filter.accept(MEAN, metric.getMean());
+ filter.accept(STDDEV, metric.getStdDev());
+ filter.accept(SUM, metric.getSum());
+ if (!(metric.isEmpty() || skipAggregateValues)) {
+ ew.putNoEx(VALUES, (MapWriter) ew1 -> {
+ metric.getValues().forEach((k, v) -> {
+ ew1.putNoEx(k, (MapWriter) ew2 -> {
+ ew2.putNoEx("value", v.value);
+ ew2.putNoEx("updateCount", v.updateCount.get());
+ });
+ });
+ });
}
};
- filter.accept("count", metric.size());
- filter.accept(MAX, metric.getMax());
- filter.accept(MIN, metric.getMin());
- filter.accept(MEAN, metric.getMean());
- filter.accept(STDDEV, metric.getStdDev());
- filter.accept(SUM, metric.getSum());
- if (!(metric.isEmpty() || skipAggregateValues)) {
- Map<String, Object> values = new LinkedHashMap<>();
- response.put(VALUES, values);
- metric.getValues().forEach((k, v) -> {
- Map<String, Object> map = new LinkedHashMap<>();
- map.put("value", v.value);
- map.put("updateCount", v.updateCount.get());
- values.put(k, map);
- });
- }
- if (!response.isEmpty()) {
- consumer.accept(name, response);
+ if (writer._size() > 0) {
+ consumer.accept(name, writer);
}
}
}
@@ -354,24 +469,23 @@ public class MetricUtils {
* only the selected (name "." key, value) pairs will be produced.
* @param consumer consumer that accepts produced objects
*/
- static void convertHistogram(String name, Histogram histogram, PropertyFilter propertyFilter,
+ static void convertHistogram(String name, Histogram histogram, Predicate<CharSequence> propertyFilter,
boolean simple, String separator, BiConsumer<String, Object> consumer) {
Snapshot snapshot = histogram.getSnapshot();
if (simple) {
- if (propertyFilter.accept(MEAN)) {
+ if (propertyFilter.test(MEAN)) {
consumer.accept(name + separator + MEAN, snapshot.getMean());
}
} else {
- Map<String, Object> response = new LinkedHashMap<>();
- String prop = "count";
- if (propertyFilter.accept(prop)) {
- response.put(prop, histogram.getCount());
- }
- // non-time based values
- addSnapshot(response, snapshot, propertyFilter, false);
- if (!response.isEmpty()) {
- consumer.accept(name, response);
- }
+ MapWriter writer = ew -> {
+ String prop = "count";
+ if (propertyFilter.test(prop)) {
+ ew.putNoEx(prop, histogram.getCount());
+ }
+ // non-time based values
+ addSnapshot(ew, snapshot, propertyFilter, false);
+ };
+ consumer.accept(name, writer);
}
}
@@ -385,10 +499,10 @@ public class MetricUtils {
}
// some snapshots represent time in ns, other snapshots represent raw values (eg. chunk size)
- static void addSnapshot(Map<String, Object> response, Snapshot snapshot, PropertyFilter propertyFilter, boolean ms) {
+ static void addSnapshot(MapWriter.EntryWriter ew, Snapshot snapshot, Predicate<CharSequence> propertyFilter, boolean ms) {
BiConsumer<String, Object> filter = (k, v) -> {
- if (propertyFilter.accept(k)) {
- response.put(k, v);
+ if (propertyFilter.test(k)) {
+ ew.putNoEx(k, v);
}
};
filter.accept((ms ? MIN_MS: MIN), nsToMs(ms, snapshot.getMin()));
@@ -411,32 +525,50 @@ public class MetricUtils {
* @param simple use simplified representation for complex metrics - instead of a (name, map)
* only the selected (name "." key, value) pairs will be produced.
* @param consumer consumer that accepts produced objects
+ * @deprecated use {@link #convertTimer(String, Timer, Predicate, boolean, boolean, String, BiConsumer)} instead.
*/
+ @Deprecated()
public static void convertTimer(String name, Timer timer, PropertyFilter propertyFilter, boolean skipHistograms,
+ boolean simple, String separator, BiConsumer<String, Object> consumer) {
+ convertTimer(name, timer, PropertyFilter.toPredicate(propertyFilter),
+ skipHistograms, simple, separator, consumer);
+ }
+ /**
+ * Convert a {@link Timer} to a map.
+ * @param name metric name
+ * @param timer timer instance
+ * @param propertyFilter limit what properties of a metric are returned
+ * @param skipHistograms if true then discard the histogram part of the timer.
+ * @param simple use simplified representation for complex metrics - instead of a (name, map)
+ * only the selected (name "." key, value) pairs will be produced.
+ * @param consumer consumer that accepts produced objects
+ */
+ public static void convertTimer(String name, Timer timer, Predicate<CharSequence> propertyFilter, boolean skipHistograms,
boolean simple, String separator, BiConsumer<String, Object> consumer) {
if (simple) {
String prop = "meanRate";
- if (propertyFilter.accept(prop)) {
+ if (propertyFilter.test(prop)) {
consumer.accept(name + separator + prop, timer.getMeanRate());
}
} else {
- Map<String, Object> response = new LinkedHashMap<>();
- BiConsumer<String,Object> filter = (k, v) -> {
- if (propertyFilter.accept(k)) {
- response.put(k, v);
+ MapWriter writer = ew -> {
+ BiConsumer<String,Object> filter = (k, v) -> {
+ if (propertyFilter.test(k)) {
+ ew.putNoEx(k, v);
+ }
+ };
+ filter.accept("count", timer.getCount());
+ filter.accept("meanRate", timer.getMeanRate());
+ filter.accept("1minRate", timer.getOneMinuteRate());
+ filter.accept("5minRate", timer.getFiveMinuteRate());
+ filter.accept("15minRate", timer.getFifteenMinuteRate());
+ if (!skipHistograms) {
+ // time-based values in nanoseconds
+ addSnapshot(ew, timer.getSnapshot(), propertyFilter, true);
}
};
- filter.accept("count", timer.getCount());
- filter.accept("meanRate", timer.getMeanRate());
- filter.accept("1minRate", timer.getOneMinuteRate());
- filter.accept("5minRate", timer.getFiveMinuteRate());
- filter.accept("15minRate", timer.getFifteenMinuteRate());
- if (!skipHistograms) {
- // time-based values in nanoseconds
- addSnapshot(response, timer.getSnapshot(), propertyFilter, true);
- }
- if (!response.isEmpty()) {
- consumer.accept(name, response);
+ if (writer._size() > 0) {
+ consumer.accept(name, writer);
}
}
}
@@ -450,29 +582,49 @@ public class MetricUtils {
* only the selected (name "." key, value) pairs will be produced.
* @param consumer consumer that accepts produced objects
*/
- static void convertMeter(String name, Meter meter, PropertyFilter propertyFilter, boolean simple, String separator, BiConsumer<String, Object> consumer) {
+ static void convertMeter(String name, Meter meter, Predicate<CharSequence> propertyFilter, boolean simple, String separator, BiConsumer<String, Object> consumer) {
if (simple) {
- if (propertyFilter.accept("count")) {
+ if (propertyFilter.test("count")) {
consumer.accept(name + separator + "count", meter.getCount());
}
} else {
- Map<String, Object> response = new LinkedHashMap<>();
- BiConsumer<String, Object> filter = (k, v) -> {
- if (propertyFilter.accept(k)) {
- response.put(k, v);
- }
+ MapWriter writer = ew -> {
+ BiConsumer<String, Object> filter = (k, v) -> {
+ if (propertyFilter.test(k)) {
+ ew.putNoEx(k, v);
+ }
+ };
+ filter.accept("count", meter.getCount());
+ filter.accept("meanRate", meter.getMeanRate());
+ filter.accept("1minRate", meter.getOneMinuteRate());
+ filter.accept("5minRate", meter.getFiveMinuteRate());
+ filter.accept("15minRate", meter.getFifteenMinuteRate());
};
- filter.accept("count", meter.getCount());
- filter.accept("meanRate", meter.getMeanRate());
- filter.accept("1minRate", meter.getOneMinuteRate());
- filter.accept("5minRate", meter.getFiveMinuteRate());
- filter.accept("15minRate", meter.getFifteenMinuteRate());
- if (!response.isEmpty()) {
- consumer.accept(name, response);
+ if (writer._size() > 0) {
+ consumer.accept(name, writer);
}
}
}
+ static void convertMapWriter(String name, MapWriter metric,
+ Predicate<CharSequence> propertyFilter, boolean simple, boolean compact,
+ String separator, BiConsumer<String, Object> consumer) {
+ ConditionalKeyMapWriter filteredMetric = new ConditionalKeyMapWriter(metric, propertyFilter);
+ if (compact || simple) {
+ if (simple) {
+ filteredMetric._forEachEntry((k, v) ->
+ consumer.accept(name + separator + k, v));
+ } else {
+ if (filteredMetric._size() > 0) {
+ consumer.accept(name, filteredMetric);
+ }
+ }
+ } else {
+ if (filteredMetric._size() > 0) {
+ consumer.accept(name, (MapWriter) ew -> ew.putNoEx("value", filteredMetric));
+ }
+ }
+ }
/**
* Convert a {@link Gauge}.
* @param name metric name
@@ -486,7 +638,7 @@ public class MetricUtils {
*/
static void convertGauge(String name,
@SuppressWarnings({"rawtypes"})Gauge gauge,
- PropertyFilter propertyFilter, boolean simple, boolean compact,
+ Predicate<CharSequence> propertyFilter, boolean simple, boolean compact,
String separator, BiConsumer<String, Object> consumer) {
if (compact || simple) {
Object o = gauge.getValue();
@@ -494,20 +646,23 @@ public class MetricUtils {
if (simple) {
for (Map.Entry<?, ?> entry : ((Map<?, ?>)o).entrySet()) {
String prop = entry.getKey().toString();
- if (propertyFilter.accept(prop)) {
+ if (propertyFilter.test(prop)) {
consumer.accept(name + separator + prop, entry.getValue());
}
}
} else {
- Map<String, Object> val = new HashMap<>();
- for (Map.Entry<?, ?> entry : ((Map<?, ?>)o).entrySet()) {
- String prop = entry.getKey().toString();
- if (propertyFilter.accept(prop)) {
- val.put(prop, entry.getValue());
+ boolean notEmpty = ((Map<?, ?>)o).entrySet().stream()
+ .anyMatch(entry -> propertyFilter.test(entry.getKey().toString()));
+ MapWriter writer = ew -> {
+ for (Map.Entry<?, ?> entry : ((Map<?, ?>)o).entrySet()) {
+ String prop = entry.getKey().toString();
+ if (propertyFilter.test(prop)) {
+ ew.putNoEx(prop, entry.getValue());
+ }
}
- }
- if (!val.isEmpty()) {
- consumer.accept(name, val);
+ };
+ if (notEmpty) {
+ consumer.accept(name, writer);
}
}
} else {
@@ -515,21 +670,24 @@ public class MetricUtils {
}
} else {
Object o = gauge.getValue();
- Map<String, Object> response = new LinkedHashMap<>();
if (o instanceof Map) {
- for (Map.Entry<?, ?> entry : ((Map<?, ?>)o).entrySet()) {
- String prop = entry.getKey().toString();
- if (propertyFilter.accept(prop)) {
- response.put(prop, entry.getValue());
- }
- }
- if (!response.isEmpty()) {
- consumer.accept(name, Collections.singletonMap("value", response));
+ boolean notEmpty = ((Map<?, ?>)o).entrySet().stream()
+ .anyMatch(entry -> propertyFilter.test(entry.getKey().toString()));
+ if (notEmpty) {
+ consumer.accept(name, (MapWriter) ew -> {
+ ew.putNoEx("value", (MapWriter) ew1 -> {
+ for (Map.Entry<?, ?> entry : ((Map<?, ?>)o).entrySet()) {
+ String prop = entry.getKey().toString();
+ if (propertyFilter.test(prop)) {
+ ew1.put(prop, entry.getValue());
+ }
+ }
+ });
+ });
}
} else {
- if (propertyFilter.accept("value")) {
- response.put("value", o);
- consumer.accept(name, response);
+ if (propertyFilter.test("value")) {
+ consumer.accept(name, (MapWriter) ew -> ew.putNoEx("value", o));
}
}
}
@@ -542,14 +700,12 @@ public class MetricUtils {
* @param compact if true then only return {@link Counter#getCount()}. If false
* then return a map with a "count" field.
*/
- static void convertCounter(String name, Counter counter, PropertyFilter propertyFilter, boolean compact, BiConsumer<String, Object> consumer) {
+ static void convertCounter(String name, Counter counter, Predicate<CharSequence> propertyFilter, boolean compact, BiConsumer<String, Object> consumer) {
if (compact) {
consumer.accept(name, counter.getCount());
} else {
- if (propertyFilter.accept("count")) {
- Map<String, Object> response = new LinkedHashMap<>();
- response.put("count", counter.getCount());
- consumer.accept(name, response);
+ if (propertyFilter.test("count")) {
+ consumer.accept(name, (MapWriter) ew -> ew.putNoEx("count", counter.getCount()));
}
}
}
diff --git a/solr/core/src/test/org/apache/solr/core/MockInfoBean.java b/solr/core/src/test/org/apache/solr/core/MockInfoBean.java
index 806c6b8..cdd48dc 100644
--- a/solr/core/src/test/org/apache/solr/core/MockInfoBean.java
+++ b/solr/core/src/test/org/apache/solr/core/MockInfoBean.java
@@ -60,7 +60,7 @@ class MockInfoBean implements SolrInfoBean, SolrMetricProducer {
this.metricManager = manager;
this.registryName = registryName;
registry = manager.registry(registryName);
- MetricsMap metricsMap = new MetricsMap((detailed, map) -> {
+ MetricsMap metricsMap = new MetricsMap(map -> {
map.put("Integer", 123);
map.put("Double",567.534);
map.put("Long", 32352463l);
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
index f6e1eb1..fba1239 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
@@ -18,10 +18,12 @@
package org.apache.solr.handler.admin;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.Map;
import com.codahale.metrics.Counter;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.MapWriter;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
@@ -84,15 +86,17 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
assertNotNull(nl);
Object o = nl.get("SEARCHER.new.errors");
assertNotNull(o); // counter type
- assertTrue(o instanceof Map);
+ assertTrue(o instanceof MapWriter);
// response wasn't serialized so we get here whatever MetricUtils produced instead of NamedList
- assertNotNull(((Map) o).get("count"));
- assertEquals(0L, ((Map) nl.get("SEARCHER.new.errors")).get("count"));
+ assertNotNull(((MapWriter) o)._get("count", null));
+ assertEquals(0L, ((MapWriter) nl.get("SEARCHER.new.errors"))._get("count", null));
nl = (NamedList) values.get("solr.node");
assertNotNull(nl.get("CONTAINER.cores.loaded")); // int gauge
- assertEquals(1, ((Map) nl.get("CONTAINER.cores.loaded")).get("value"));
+ assertEquals(1, ((MapWriter) nl.get("CONTAINER.cores.loaded"))._get("value", null));
assertNotNull(nl.get("ADMIN./admin/authorization.clientErrors")); // timer type
- assertEquals(5, ((Map) nl.get("ADMIN./admin/authorization.clientErrors")).size());
+ Map<String, Object> map = new HashMap<>();
+ ((MapWriter) nl.get("ADMIN./admin/authorization.clientErrors")).toMap(map);
+ assertEquals(5, map.size());
resp = new SolrQueryResponse();
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", MetricsHandler.COMPACT_PARAM, "false", CommonParams.WT, "json", "group", "jvm,jetty"), resp);
@@ -184,9 +188,9 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
values = (NamedList) values.get("solr.core.collection1");
assertEquals(1, values.size());
@SuppressWarnings({"rawtypes"})
- Map m = (Map) values.get("CACHE.core.fieldCache");
- assertNotNull(m);
- assertNotNull(m.get("entries_count"));
+ MapWriter writer = (MapWriter) values.get("CACHE.core.fieldCache");
+ assertNotNull(writer);
+ assertNotNull(writer._get("entries_count", null));
resp = new SolrQueryResponse();
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", MetricsHandler.COMPACT_PARAM, "false", CommonParams.WT, "json", "group", "jvm", "prefix", "CONTAINER.cores"), resp);
@@ -200,8 +204,8 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
values = resp.getValues();
assertNotNull(values.get("metrics"));
@SuppressWarnings({"rawtypes"})
- SimpleOrderedMap map = (SimpleOrderedMap) values.get("metrics");
- assertEquals(0, map.size());
+ SimpleOrderedMap map1 = (SimpleOrderedMap) values.get("metrics");
+ assertEquals(0, map1.size());
handler.close();
}
@@ -243,9 +247,9 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
assertNotNull(nl);
assertTrue(nl.size() > 0);
nl.forEach((k, v) -> {
- assertTrue(v instanceof Map);
- @SuppressWarnings({"rawtypes"})
- Map map = (Map) v;
+ assertTrue(v instanceof MapWriter);
+ Map<String, Object> map = new HashMap<>();
+ ((MapWriter) v).toMap(map);
assertTrue(map.size() > 2);
});
@@ -259,10 +263,10 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
assertNotNull(nl);
assertTrue(nl.size() > 0);
nl.forEach((k, v) -> {
- assertTrue(v instanceof Map);
- @SuppressWarnings({"rawtypes"})
- Map map = (Map) v;
- assertEquals(2, map.size());
+ assertTrue(v instanceof MapWriter);
+ Map<String, Object> map = new HashMap<>();
+ ((MapWriter) v).toMap(map);
+ assertEquals("k=" + k + ", v=" + map, 2, map.size());
assertNotNull(map.get("inserts"));
assertNotNull(map.get("size"));
});
@@ -281,15 +285,14 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
NamedList values = resp.getValues();
Object val = values.findRecursive("metrics", key1);
assertNotNull(val);
- assertTrue(val instanceof Map);
- assertTrue(((Map) val).size() >= 2);
+ assertTrue(val instanceof MapWriter);
+ assertTrue(((MapWriter)val)._size() >= 2);
String key2 = "solr.core.collection1:CACHE.core.fieldCache:entries_count";
resp = new SolrQueryResponse();
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
MetricsHandler.KEY_PARAM, key2), resp);
- values = resp.getValues();
- val = values.findRecursive("metrics", key2);
+ val = resp.getValues()._get("metrics/" + key2, null);
assertNotNull(val);
assertTrue(val instanceof Number);
@@ -297,8 +300,8 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
resp = new SolrQueryResponse();
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
MetricsHandler.KEY_PARAM, key3), resp);
- values = resp.getValues();
- val = values.findRecursive("metrics", key3);
+
+ val = resp.getValues()._get( "metrics/" + key3, null);
assertNotNull(val);
assertTrue(val instanceof Number);
assertEquals(3, ((Number) val).intValue());
@@ -307,20 +310,20 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
resp = new SolrQueryResponse();
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
MetricsHandler.KEY_PARAM, key1, MetricsHandler.KEY_PARAM, key2, MetricsHandler.KEY_PARAM, key3), resp);
- values = resp.getValues();
- val = values.findRecursive("metrics", key1);
+
+ val = resp.getValues()._get( "metrics/" + key1, null);
assertNotNull(val);
- val = values.findRecursive("metrics", key2);
+ val = resp.getValues()._get( "metrics/" + key2, null);
assertNotNull(val);
- val = values.findRecursive("metrics", key3);
+ val = resp.getValues()._get( "metrics/" + key3, null);
assertNotNull(val);
String key4 = "solr.core.collection1:QUERY./select.requestTimes:1minRate";
resp = new SolrQueryResponse();
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
MetricsHandler.KEY_PARAM, key4), resp);
- values = resp.getValues();
- val = values.findRecursive("metrics", key4);
+ // the key contains a slash, need explicit list of path elements
+ val = resp.getValues()._get(Arrays.asList("metrics", key4), null);
assertNotNull(val);
assertTrue(val instanceof Number);
@@ -460,7 +463,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
@Override
public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
super.initializeMetrics(parentContext, scope);
- MetricsMap metrics = new MetricsMap((detailed, map) -> map.putAll(gaugevals));
+ MetricsMap metrics = new MetricsMap(map -> gaugevals.forEach((k, v) -> map.putNoEx(k, v)));
solrMetricsContext.gauge(this,
metrics, true, "dumphandlergauge", getCategory().toString(), scope);
diff --git a/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java b/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java
index f9595e1..e95bea4 100644
--- a/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java
+++ b/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java
@@ -31,8 +31,11 @@ import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Snapshot;
import com.codahale.metrics.Timer;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.MapWriter;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.metrics.AggregateMetric;
+import org.apache.solr.metrics.MetricsMap;
+import org.apache.solr.metrics.SolrMetricManager;
import org.junit.Test;
public class MetricUtilsTest extends SolrTestCaseJ4 {
@@ -48,8 +51,8 @@ public class MetricUtilsTest extends SolrTestCaseJ4 {
}
// obtain timer metrics
Map<String,Object> map = new HashMap<>();
- MetricUtils.convertTimer("", timer, MetricUtils.PropertyFilter.ALL, false, false, ".", (k, v) -> {
- map.putAll((Map<String,Object>)v);
+ MetricUtils.convertTimer("", timer, MetricUtils.ALL_PROPERTIES, false, false, ".", (k, v) -> {
+ ((MapWriter) v).toMap(map);
});
@SuppressWarnings({"rawtypes"})
NamedList lst = new NamedList(map);
@@ -103,10 +106,26 @@ public class MetricUtilsTest extends SolrTestCaseJ4 {
registry.register("gauge", gauge);
Gauge<Long> error = () -> {throw new InternalError("Memory Pool not found error");};
registry.register("memory.expected.error", error);
+
+ MetricsMap metricsMapWithMap = new MetricsMap((detailed, map) -> {
+ map.put("foo", "bar");
+ });
+ registry.register("mapWithMap", metricsMapWithMap);
+ MetricsMap metricsMap = new MetricsMap(map -> {
+ map.putNoEx("foo", "bar");
+ });
+ registry.register("map", metricsMap);
+
+ SolrMetricManager.GaugeWrapper<Map<String,Object>> gaugeWrapper = new SolrMetricManager.GaugeWrapper<>(metricsMap, "foo-tag");
+ registry.register("wrappedGauge", gaugeWrapper);
+
MetricUtils.toMaps(registry, Collections.singletonList(MetricFilter.ALL), MetricFilter.ALL,
- MetricUtils.PropertyFilter.ALL, false, false, false, false, (k, o) -> {
+ MetricUtils.ALL_PROPERTIES, false, false, false, false, (k, o) -> {
@SuppressWarnings({"rawtypes"})
- Map v = (Map)o;
+ Map<String, Object> v = new HashMap<>();
+ if (o != null) {
+ ((MapWriter) o).toMap(v);
+ }
if (k.startsWith("counter")) {
assertEquals(1L, v.get("count"));
} else if (k.startsWith("gauge")) {
@@ -139,12 +158,16 @@ public class MetricUtilsTest extends SolrTestCaseJ4 {
assertEquals(0D, v.get("max"));
assertEquals(0D, v.get("mean"));
} else if (k.startsWith("memory.expected.error")) {
- assertNull(v);
+ assertTrue(v.isEmpty());
+ } else if (k.startsWith("map") || k.startsWith("wrapped")) {
+ assertNotNull(v.toString(), v.get("value"));
+ assertTrue(v.toString(), v.get("value") instanceof Map);
+ assertEquals(v.toString(), "bar", ((Map) v.get("value")).get("foo"));
}
});
// test compact format
MetricUtils.toMaps(registry, Collections.singletonList(MetricFilter.ALL), MetricFilter.ALL,
- MetricUtils.PropertyFilter.ALL, false, false, true, false, (k, o) -> {
+ MetricUtils.ALL_PROPERTIES, false, false, true, false, (k, o) -> {
if (k.startsWith("counter")) {
assertTrue(o instanceof Long);
assertEquals(1L, o);
@@ -152,25 +175,25 @@ public class MetricUtilsTest extends SolrTestCaseJ4 {
assertTrue(o instanceof String);
assertEquals("foobar", o);
} else if (k.startsWith("timer")) {
- assertTrue(o instanceof Map);
- @SuppressWarnings({"rawtypes"})
- Map v = (Map)o;
+ assertTrue(o instanceof MapWriter);
+ Map<String, Object> v = new HashMap<>();
+ ((MapWriter) o).toMap(v);
assertEquals(1L, v.get("count"));
assertTrue(((Number)v.get("min_ms")).intValue() > 100);
} else if (k.startsWith("meter")) {
- assertTrue(o instanceof Map);
- @SuppressWarnings({"rawtypes"})
- Map v = (Map)o;
+ assertTrue(o instanceof MapWriter);
+ Map<String, Object> v = new HashMap<>();
+ ((MapWriter) o).toMap(v);
assertEquals(1L, v.get("count"));
} else if (k.startsWith("histogram")) {
- assertTrue(o instanceof Map);
- @SuppressWarnings({"rawtypes"})
- Map v = (Map)o;
+ assertTrue(o instanceof MapWriter);
+ Map<String, Object> v = new HashMap<>();
+ ((MapWriter) o).toMap(v);
assertEquals(1L, v.get("count"));
} else if (k.startsWith("aggregate1")) {
- assertTrue(o instanceof Map);
- @SuppressWarnings({"rawtypes"})
- Map v = (Map)o;
+ assertTrue(o instanceof MapWriter);
+ Map<String, Object> v = new HashMap<>();
+ ((MapWriter) o).toMap(v);
assertEquals(4, v.get("count"));
Map<String, Object> values = (Map<String, Object>)v.get("values");
assertNotNull(values);
@@ -182,9 +205,9 @@ public class MetricUtilsTest extends SolrTestCaseJ4 {
assertEquals(-2, update.get("value"));
assertEquals(2, update.get("updateCount"));
} else if (k.startsWith("aggregate2")) {
- assertTrue(o instanceof Map);
- @SuppressWarnings({"rawtypes"})
- Map v = (Map)o;
+ assertTrue(o instanceof MapWriter);
+ Map<String, Object> v = new HashMap<>();
+ ((MapWriter) o).toMap(v);
assertEquals(2, v.get("count"));
Map<String, Object> values = (Map<String, Object>)v.get("values");
assertNotNull(values);
@@ -197,9 +220,15 @@ public class MetricUtilsTest extends SolrTestCaseJ4 {
assertEquals(1, update.get("updateCount"));
} else if (k.startsWith("memory.expected.error")) {
assertNull(o);
+ } else if (k.startsWith("map") || k.startsWith("wrapped")) {
+ assertTrue(o instanceof MapWriter);
+ MapWriter writer = (MapWriter) o;
+ assertEquals(1, writer._size());
+ assertEquals("bar", writer._get("foo", null));
} else {
- @SuppressWarnings({"rawtypes"})
- Map v = (Map)o;
+ assertTrue(o instanceof MapWriter);
+ Map<String, Object> v = new HashMap<>();
+ ((MapWriter) o).toMap(v);
assertEquals(1L, v.get("count"));
}
});
diff --git a/solr/solrj/src/java/org/apache/solr/common/MapWriterMap.java b/solr/solrj/src/java/org/apache/solr/common/MapWriterMap.java
index c5f6164..5e6dfeb 100644
--- a/solr/solrj/src/java/org/apache/solr/common/MapWriterMap.java
+++ b/solr/solrj/src/java/org/apache/solr/common/MapWriterMap.java
@@ -51,6 +51,11 @@ public class MapWriterMap implements MapWriter {
@Override
+ public int _size() {
+ return delegate.size();
+ }
+
+ @Override
@SuppressWarnings("unchecked")
public Map<String,Object> toMap(Map<String, Object> map) {
return delegate;
diff --git a/solr/solrj/src/java/org/apache/solr/common/NavigableObject.java b/solr/solrj/src/java/org/apache/solr/common/NavigableObject.java
index 22dca2b..4f88de5 100644
--- a/solr/solrj/src/java/org/apache/solr/common/NavigableObject.java
+++ b/solr/solrj/src/java/org/apache/solr/common/NavigableObject.java
@@ -86,4 +86,10 @@ public interface NavigableObject {
Object v = Utils.getObjectByPath(this, false, path);
return v == null ? def : String.valueOf(v);
}
+
+ default int _size() {
+ int[] size = new int[1];
+ _forEachEntry((k, v) -> size[0]++);
+ return size[0];
+ }
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java b/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java
index 165ba76..4be22f2 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java
@@ -890,6 +890,11 @@ public class NamedList<T> implements Cloneable, Serializable, Iterable<Map.Entry
}
@Override
+ public int _size() {
+ return size();
+ }
+
+ @Override
public void forEachEntry(BiConsumer<String, ? super T> fun) {
forEach(fun);
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
index f362da9..a82b843 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
@@ -228,9 +228,49 @@ public class Utils {
@Override
@SuppressWarnings({"rawtypes"})
public void handleUnknownClass(Object o) {
+ // avoid materializing MapWriter / IteratorWriter to Map / List
+ // instead serialize them directly
if (o instanceof MapWriter) {
- Map m = ((MapWriter) o).toMap(new LinkedHashMap<>());
- write(m);
+ MapWriter mapWriter = (MapWriter) o;
+ startObject();
+ final boolean[] first = new boolean[1];
+ first[0] = true;
+ int sz = mapWriter._size();
+ mapWriter._forEachEntry((k, v) -> {
+ if (first[0]) {
+ first[0] = false;
+ } else {
+ writeValueSeparator();
+ }
+ if (sz > 1) indent();
+ writeString(k.toString());
+ writeNameSeparator();
+ write(v);
+ });
+ endObject();
+ } else if (o instanceof IteratorWriter) {
+ IteratorWriter iteratorWriter = (IteratorWriter) o;
+ startArray();
+ final boolean[] first = new boolean[1];
+ first[0] = true;
+ try {
+ iteratorWriter.writeIter(new IteratorWriter.ItemWriter() {
+ @Override
+ public IteratorWriter.ItemWriter add(Object o) throws IOException {
+ if (first[0]) {
+ first[0] = false;
+ } else {
+ writeValueSeparator();
+ }
+ indent();
+ write(o);
+ return this;
+ }
+ });
+ } catch (IOException e) {
+ throw new RuntimeException("this should never happen", e);
+ }
+ endArray();
} else {
super.handleUnknownClass(o);
}
@@ -240,13 +280,13 @@ public class Utils {
public static byte[] toJSON(Object o) {
if (o == null) return new byte[0];
CharArr out = new CharArr();
- if (!(o instanceof List) && !(o instanceof Map)) {
- if (o instanceof MapWriter) {
- o = ((MapWriter) o).toMap(new LinkedHashMap<>());
- } else if (o instanceof IteratorWriter) {
- o = ((IteratorWriter) o).toList(new ArrayList<>());
- }
- }
+// if (!(o instanceof List) && !(o instanceof Map)) {
+// if (o instanceof MapWriter) {
+// o = ((MapWriter) o).toMap(new LinkedHashMap<>());
+// } else if (o instanceof IteratorWriter) {
+// o = ((IteratorWriter) o).toList(new ArrayList<>());
+// }
+// }
new MapWriterJSONWriter(out, 2).write(o); // indentation by default
return toUTF8(out);
}