You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2017/01/25 13:21:10 UTC
lucene-solr:jira/solr-9858: SOLR-9858 Add SolrNodeReporter. Refactor
the common code in SolrReporter to be more expressive.
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-9858 [created] 493544b6e
SOLR-9858 Add SolrNodeReporter. Refactor the common code in SolrReporter to be more
expressive.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/493544b6
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/493544b6
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/493544b6
Branch: refs/heads/jira/solr-9858
Commit: 493544b6e6bc32352f0832d1920202e7cbf40598
Parents: 83c177e
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Wed Jan 25 14:20:27 2017 +0100
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Wed Jan 25 14:20:27 2017 +0100
----------------------------------------------------------------------
.../org/apache/solr/cloud/ElectionContext.java | 5 +-
.../java/org/apache/solr/cloud/Overseer.java | 7 +-
.../solr/cloud/OverseerNodePrioritizer.java | 2 +-
.../solr/cloud/OverseerTaskProcessor.java | 6 +-
.../org/apache/solr/cloud/ZkController.java | 2 +-
.../org/apache/solr/core/CoreContainer.java | 16 +-
.../org/apache/solr/core/SolrInfoMBean.java | 2 +-
.../handler/admin/MetricsCollectorHandler.java | 95 ++++----
.../solr/handler/admin/MetricsHandler.java | 2 +-
.../apache/solr/metrics/AggregateMetric.java | 64 ++++--
.../apache/solr/metrics/SolrMetricManager.java | 154 ++++++++++++-
.../reporters/solr/MetricsReportRequest.java | 102 ---------
.../reporters/solr/SolrNodeReporter.java | 178 +++++++++++++++
.../reporters/solr/SolrReplicaReporter.java | 23 +-
.../metrics/reporters/solr/SolrReporter.java | 221 +++++++++++++------
.../org/apache/solr/util/stats/MetricUtils.java | 25 ++-
.../reporters/solr/SolrReplicaReporterTest.java | 3 +
.../apache/solr/util/stats/MetricUtilsTest.java | 2 +-
18 files changed, 629 insertions(+), 280 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/493544b6/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
index b3cd585..7ce2c69 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
@@ -709,14 +709,13 @@ final class OverseerElectionContext extends ElectionContext {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final SolrZkClient zkClient;
private Overseer overseer;
- public static final String OVERSEER_ELECT = "/overseer_elect";
public OverseerElectionContext(SolrZkClient zkClient, Overseer overseer, final String zkNodeName) {
- super(zkNodeName, OVERSEER_ELECT, OVERSEER_ELECT + "/leader", null, zkClient);
+ super(zkNodeName, Overseer.OVERSEER_ELECT, Overseer.OVERSEER_ELECT + "/leader", null, zkClient);
this.overseer = overseer;
this.zkClient = zkClient;
try {
- new ZkCmdExecutor(zkClient.getZkClientTimeout()).ensureExists(OVERSEER_ELECT, zkClient);
+ new ZkCmdExecutor(zkClient.getZkClientTimeout()).ensureExists(Overseer.OVERSEER_ELECT, zkClient);
} catch (KeeperException e) {
throw new SolrException(ErrorCode.SERVER_ERROR, e);
} catch (InterruptedException e) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/493544b6/solr/core/src/java/org/apache/solr/cloud/Overseer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
index a618874..b2467fa 100644
--- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
@@ -64,7 +64,8 @@ public class Overseer implements Closeable {
public static final int STATE_UPDATE_DELAY = 1500; // delay between cloud state updates
public static final int NUM_RESPONSES_TO_STORE = 10000;
-
+ public static final String OVERSEER_ELECT = "/overseer_elect";
+
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
enum LeaderStatus {DONT_KNOW, NO, YES}
@@ -280,7 +281,7 @@ public class Overseer implements Closeable {
private void checkIfIamStillLeader() {
if (zkController != null && zkController.getCoreContainer().isShutDown()) return;//shutting down no need to go further
org.apache.zookeeper.data.Stat stat = new org.apache.zookeeper.data.Stat();
- String path = OverseerElectionContext.OVERSEER_ELECT + "/leader";
+ String path = OVERSEER_ELECT + "/leader";
byte[] data;
try {
data = zkClient.getData(path, null, stat, true);
@@ -393,7 +394,7 @@ public class Overseer implements Closeable {
boolean success = true;
try {
ZkNodeProps props = ZkNodeProps.load(zkClient.getData(
- OverseerElectionContext.OVERSEER_ELECT + "/leader", null, null, true));
+ OVERSEER_ELECT + "/leader", null, null, true));
if (myId.equals(props.getStr("id"))) {
return LeaderStatus.YES;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/493544b6/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java b/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java
index 6512d26..798eca3 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java
@@ -65,7 +65,7 @@ public class OverseerNodePrioritizer {
String ldr = OverseerTaskProcessor.getLeaderNode(zk);
if(overseerDesignates.contains(ldr)) return;
log.info("prioritizing overseer nodes at {} overseer designates are {}", overseerId, overseerDesignates);
- List<String> electionNodes = OverseerTaskProcessor.getSortedElectionNodes(zk, OverseerElectionContext.OVERSEER_ELECT + LeaderElector.ELECTION_NODE);
+ List<String> electionNodes = OverseerTaskProcessor.getSortedElectionNodes(zk, Overseer.OVERSEER_ELECT + LeaderElector.ELECTION_NODE);
if(electionNodes.size()<2) return;
log.info("sorted nodes {}", electionNodes);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/493544b6/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java
index ad53346..bed71a6 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java
@@ -337,7 +337,7 @@ public class OverseerTaskProcessor implements Runnable, Closeable {
public static List<String> getSortedOverseerNodeNames(SolrZkClient zk) throws KeeperException, InterruptedException {
List<String> children = null;
try {
- children = zk.getChildren(OverseerElectionContext.OVERSEER_ELECT + LeaderElector.ELECTION_NODE, null, true);
+ children = zk.getChildren(Overseer.OVERSEER_ELECT + LeaderElector.ELECTION_NODE, null, true);
} catch (Exception e) {
log.warn("error ", e);
return new ArrayList<>();
@@ -370,7 +370,7 @@ public class OverseerTaskProcessor implements Runnable, Closeable {
public static String getLeaderId(SolrZkClient zkClient) throws KeeperException,InterruptedException{
byte[] data = null;
try {
- data = zkClient.getData(OverseerElectionContext.OVERSEER_ELECT + "/leader", null, new Stat(), true);
+ data = zkClient.getData(Overseer.OVERSEER_ELECT + "/leader", null, new Stat(), true);
} catch (KeeperException.NoNodeException e) {
return null;
}
@@ -384,7 +384,7 @@ public class OverseerTaskProcessor implements Runnable, Closeable {
boolean success = true;
try {
ZkNodeProps props = ZkNodeProps.load(zkStateReader.getZkClient().getData(
- OverseerElectionContext.OVERSEER_ELECT + "/leader", null, null, true));
+ Overseer.OVERSEER_ELECT + "/leader", null, null, true));
if (myId.equals(props.getStr("id"))) {
return LeaderStatus.YES;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/493544b6/solr/core/src/java/org/apache/solr/cloud/ZkController.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index eba7067..23d5d2e 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -1713,7 +1713,7 @@ public class ZkController {
//however delete it . This is possible when the last attempt at deleting the election node failed.
if (electionNode.startsWith(getNodeName())) {
try {
- zkClient.delete(OverseerElectionContext.OVERSEER_ELECT + LeaderElector.ELECTION_NODE + "/" + electionNode, -1, true);
+ zkClient.delete(Overseer.OVERSEER_ELECT + LeaderElector.ELECTION_NODE + "/" + electionNode, -1, true);
} catch (NoNodeException e) {
//no problem
} catch (InterruptedException e) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/493544b6/solr/core/src/java/org/apache/solr/core/CoreContainer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 07a33ea..46c3c8e 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -542,6 +542,10 @@ public class CoreContainer {
metricManager.register(SolrMetricManager.getRegistryName(SolrInfoMBean.Group.node),
unloadedCores, true, "unloaded",SolrInfoMBean.Category.CONTAINER.toString(), "cores");
+ if (isZooKeeperAware()) {
+ metricManager.loadNodeReporter(this, this.getZkController().getNodeName());
+ }
+
// setup executor to load cores in parallel
ExecutorService coreLoadExecutor = MetricUtils.instrumentedExecutorService(
ExecutorUtil.newMDCAwareFixedThreadPool(
@@ -665,10 +669,16 @@ public class CoreContainer {
isShutDown = true;
ExecutorUtil.shutdownAndAwaitTermination(coreContainerWorkExecutor);
+ if (metricManager != null) {
+ metricManager.closeReporters(SolrMetricManager.getRegistryName(SolrInfoMBean.Group.node));
+ }
if (isZooKeeperAware()) {
cancelCoreRecoveries();
- zkSys.zkController.publishNodeAsDown(zkSys.zkController.getNodeName());
+ zkSys.zkController.publishNodeAsDown(zkSys.zkController.getNodeName());
+ if (metricManager != null) {
+ metricManager.closeReporters(SolrMetricManager.overridableRegistryName(zkSys.zkController.getNodeName()));
+ }
}
try {
@@ -718,10 +728,6 @@ public class CoreContainer {
}
}
- if (metricManager != null) {
- metricManager.closeReporters(SolrMetricManager.getRegistryName(SolrInfoMBean.Group.node));
- }
-
// It should be safe to close the authorization plugin at this point.
try {
if(authorizationPlugin != null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/493544b6/solr/core/src/java/org/apache/solr/core/SolrInfoMBean.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrInfoMBean.java b/solr/core/src/java/org/apache/solr/core/SolrInfoMBean.java
index bf77db4..d7424a0 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrInfoMBean.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrInfoMBean.java
@@ -38,7 +38,7 @@ public interface SolrInfoMBean {
/**
* Top-level group of beans for a subsystem.
*/
- enum Group { jvm, jetty, node, core }
+ enum Group { jvm, jetty, node, core, overseer }
/**
* Simple common usage name, e.g. BasicQueryHandler,
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/493544b6/solr/core/src/java/org/apache/solr/handler/admin/MetricsCollectorHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsCollectorHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsCollectorHandler.java
index 494eab0..0dc7412 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsCollectorHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsCollectorHandler.java
@@ -56,11 +56,11 @@ import org.slf4j.LoggerFactory;
public class MetricsCollectorHandler extends RequestHandlerBase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- public static final String HANDLER_PATH = "/admin/metricsCollector";
+ public static final String HANDLER_PATH = "/admin/metrics/collector";
private final CoreContainer coreContainer;
private final SolrMetricManager metricManager;
- private final Map<String, ContentStreamLoader> registry = new HashMap<>();
+ private final Map<String, ContentStreamLoader> loaders = new HashMap<>();
private SolrParams params;
public MetricsCollectorHandler(final CoreContainer coreContainer) {
@@ -77,35 +77,32 @@ public class MetricsCollectorHandler extends RequestHandlerBase {
} else {
params = new ModifiableSolrParams();
}
- registry.put("application/xml", new XMLLoader().init(params) );
- registry.put("application/json", new JsonLoader().init(params) );
- registry.put("application/csv", new CSVLoader().init(params) );
- registry.put("application/javabin", new JavabinLoader().init(params) );
- registry.put("text/csv", registry.get("application/csv") );
- registry.put("text/xml", registry.get("application/xml") );
- registry.put("text/json", registry.get("application/json"));
+ loaders.put("application/xml", new XMLLoader().init(params) );
+ loaders.put("application/json", new JsonLoader().init(params) );
+ loaders.put("application/csv", new CSVLoader().init(params) );
+ loaders.put("application/javabin", new JavabinLoader().init(params) );
+ loaders.put("text/csv", loaders.get("application/csv") );
+ loaders.put("text/xml", loaders.get("application/xml") );
+ loaders.put("text/json", loaders.get("application/json"));
}
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
+ if (coreContainer == null || coreContainer.isShutDown()) {
+ // silently drop request
+ return;
+ }
//log.info("#### " + req.toString());
for (ContentStream cs : req.getContentStreams()) {
if (cs.getContentType() == null) {
log.warn("Missing content type - ignoring");
continue;
}
- ContentStreamLoader loader = registry.get(cs.getContentType());
+ ContentStreamLoader loader = loaders.get(cs.getContentType());
if (loader == null) {
throw new SolrException(SolrException.ErrorCode.UNSUPPORTED_MEDIA_TYPE, "Unsupported content type for stream: " + cs.getSourceInfo() + ", contentType=" + cs.getContentType());
}
- String reporterId = req.getParams().get(SolrReporter.REPORTER_ID);
- String group = req.getParams().get(SolrReporter.GROUP_ID);
- if (reporterId == null || group == null) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Missing " + SolrReporter.REPORTER_ID +
- " or " + SolrReporter.GROUP_ID + " in request params: " + req.getParamString());
- }
- MetricRegistry registry = metricManager.registry(group);
- loader.load(req, rsp, cs, new MetricUpdateProcessor(registry, reporterId, group));
+ loader.load(req, rsp, cs, new MetricUpdateProcessor(metricManager));
}
}
@@ -115,15 +112,11 @@ public class MetricsCollectorHandler extends RequestHandlerBase {
}
private static class MetricUpdateProcessor extends UpdateRequestProcessor {
- private final MetricRegistry registry;
- private final String reporterId;
- private final String group;
+ private final SolrMetricManager metricManager;
- public MetricUpdateProcessor(MetricRegistry registry, String reporterId, String group) {
+ public MetricUpdateProcessor(SolrMetricManager metricManager) {
super(null);
- this.registry = registry;
- this.reporterId = reporterId;
- this.group = group;
+ this.metricManager = metricManager;
}
@Override
@@ -132,39 +125,49 @@ public class MetricsCollectorHandler extends RequestHandlerBase {
if (doc == null) {
return;
}
- String metricName = (String)doc.getFieldValue(MetricUtils.NAME);
+ String metricName = (String)doc.getFieldValue(MetricUtils.METRIC_NAME);
if (metricName == null) {
- log.warn("Missing metric 'name' field in document, skipping: " + doc);
+ log.warn("Missing metric " + MetricUtils.METRIC_NAME + " field in document, skipping: " + doc);
+ return;
+ }
+ doc.remove(MetricUtils.METRIC_NAME);
+ // XXX we could modify keys by using this original registry name
+ doc.remove(SolrReporter.REGISTRY_ID);
+ String groupId = (String)doc.getFieldValue(SolrReporter.GROUP_ID);
+ if (groupId == null) {
+ log.warn("Missing metric " + SolrReporter.GROUP_ID + " field in document, skipping: " + doc);
return;
}
- doc.remove(MetricUtils.NAME);
- // already known
- doc.remove(SolrReporter.REPORTER_ID);
doc.remove(SolrReporter.GROUP_ID);
- // remaining fields should only contain numeric values
+ String reporterId = (String)doc.getFieldValue(SolrReporter.REPORTER_ID);
+ if (reporterId == null) {
+ log.warn("Missing metric " + SolrReporter.REPORTER_ID + " field in document, skipping: " + doc);
+ return;
+ }
+ doc.remove(SolrReporter.REPORTER_ID);
+ String labelId = (String)doc.getFieldValue(SolrReporter.LABEL_ID);
+ doc.remove(SolrReporter.LABEL_ID);
doc.forEach(f -> {
- if (f.getFirstValue() instanceof Number) {
- String key = MetricRegistry.name(metricName, f.getName());
- AggregateMetric metric = (AggregateMetric)registry.getMetrics().get(key);
- if (metric == null) {
- metric = new AggregateMetric();
- registry.register(key, metric);
- }
- Object o = f.getFirstValue();
- if (o != null && (o instanceof Number)) {
- metric.set(reporterId, ((Number)o).doubleValue());
- } else {
- // silently discard
- }
+ String key = MetricRegistry.name(labelId, metricName, f.getName());
+ MetricRegistry registry = metricManager.registry(groupId);
+ AggregateMetric metric = (AggregateMetric)registry.getMetrics().get(key);
+ if (metric == null) {
+ metric = new AggregateMetric();
+ registry.register(key, metric);
+ }
+ Object o = f.getFirstValue();
+ if (o != null) {
+ metric.set(reporterId, o);
} else {
- // silently discard
+ // remove missing values
+ metric.clear(reporterId);
}
});
}
@Override
public void processDelete(DeleteUpdateCommand cmd) throws IOException {
- super.processDelete(cmd);
+ throw new UnsupportedOperationException("processDelete");
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/493544b6/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
index 56a37fe..b53c818 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
@@ -79,7 +79,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
NamedList response = new NamedList();
for (String registryName : requestedRegistries) {
MetricRegistry registry = metricManager.registry(registryName);
- response.add(registryName, MetricUtils.toNamedList(registry, metricFilters, mustMatchFilter, false, null));
+ response.add(registryName, MetricUtils.toNamedList(registry, metricFilters, mustMatchFilter, false, false, null));
}
rsp.getValues().add("metrics", response);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/493544b6/solr/core/src/java/org/apache/solr/metrics/AggregateMetric.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/AggregateMetric.java b/solr/core/src/java/org/apache/solr/metrics/AggregateMetric.java
index 2123bfc..aea209c 100644
--- a/solr/core/src/java/org/apache/solr/metrics/AggregateMetric.java
+++ b/solr/core/src/java/org/apache/solr/metrics/AggregateMetric.java
@@ -16,14 +16,14 @@ public class AggregateMetric implements Metric {
* Simple class to represent current value and how many times it was set.
*/
public static class Update {
- public Number value;
+ public Object value;
public final AtomicInteger updateCount = new AtomicInteger();
- public Update(Number value) {
+ public Update(Object value) {
update(value);
}
- public void update(Number value) {
+ public void update(Object value) {
this.value = value;
updateCount.incrementAndGet();
}
@@ -39,7 +39,7 @@ public class AggregateMetric implements Metric {
private final Map<String, Update> values = new ConcurrentHashMap<>();
- public void set(String name, Number value) {
+ public void set(String name, Object value) {
final Update existing = values.get(name);
if (existing == null) {
final Update created = new Update(value);
@@ -79,12 +79,16 @@ public class AggregateMetric implements Metric {
}
Double res = null;
for (Update u : values.values()) {
+ if (!(u.value instanceof Number)) {
+ continue;
+ }
+ Number n = (Number)u.value;
if (res == null) {
- res = u.value.doubleValue();
+ res = n.doubleValue();
continue;
}
- if (u.value.doubleValue() > res) {
- res = u.value.doubleValue();
+ if (n.doubleValue() > res) {
+ res = n.doubleValue();
}
}
return res;
@@ -96,12 +100,16 @@ public class AggregateMetric implements Metric {
}
Double res = null;
for (Update u : values.values()) {
+ if (!(u.value instanceof Number)) {
+ continue;
+ }
+ Number n = (Number)u.value;
if (res == null) {
- res = u.value.doubleValue();
+ res = n.doubleValue();
continue;
}
- if (u.value.doubleValue() < res) {
- res = u.value.doubleValue();
+ if (n.doubleValue() < res) {
+ res = n.doubleValue();
}
}
return res;
@@ -113,7 +121,11 @@ public class AggregateMetric implements Metric {
}
double total = 0;
for (Update u : values.values()) {
- total += u.value.doubleValue();
+ if (!(u.value instanceof Number)) {
+ continue;
+ }
+ Number n = (Number)u.value;
+ total += n.doubleValue();
}
return total / values.size();
}
@@ -125,21 +137,47 @@ public class AggregateMetric implements Metric {
}
final double mean = getMean();
double sum = 0;
+ int count = 0;
for (Update u : values.values()) {
- final double diff = u.value.doubleValue() - mean;
+ if (!(u.value instanceof Number)) {
+ continue;
+ }
+ count++;
+ Number n = (Number)u.value;
+ final double diff = n.doubleValue() - mean;
sum += diff * diff;
}
- final double variance = sum / (size - 1);
+ if (count < 2) {
+ return 0;
+ }
+ final double variance = sum / (count - 1);
return Math.sqrt(variance);
}
+ public double getSum() {
+ if (values.isEmpty()) {
+ return 0;
+ }
+ double res = 0;
+ for (Update u : values.values()) {
+ if (!(u.value instanceof Number)) {
+ continue;
+ }
+ Number n = (Number)u.value;
+ res += n.doubleValue();
+ }
+ return res;
+ }
+
@Override
public String toString() {
return "AggregateMetric{" +
"size=" + size() +
", max=" + getMax() +
", min=" + getMin() +
+ ", mean=" + getMean() +
", stddev=" + getStdDev() +
+ ", sum=" + getSum() +
", values=" + values +
'}';
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/493544b6/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
index dc08103..e807925 100644
--- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
+++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
@@ -18,9 +18,13 @@ package org.apache.solr.metrics;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
@@ -29,6 +33,9 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+import java.util.stream.Collectors;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Histogram;
@@ -39,11 +46,14 @@ import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.MetricSet;
import com.codahale.metrics.SharedMetricRegistries;
import com.codahale.metrics.Timer;
+import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoMBean;
import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.metrics.reporters.solr.SolrNodeReporter;
import org.apache.solr.metrics.reporters.solr.SolrReplicaReporter;
import org.apache.solr.metrics.reporters.solr.SolrReporter;
import org.apache.solr.util.plugin.SolrCoreAware;
@@ -95,23 +105,33 @@ public class SolrMetricManager {
/**
* An implementation of {@link MetricFilter} that selects metrics
- * with names that start with a prefix.
+ * with names that start with one of prefixes.
*/
public static class PrefixFilter implements MetricFilter {
- private final String[] prefixes;
+ private final Set<String> prefixes = new HashSet<>();
private final Set<String> matched = new HashSet<>();
private boolean allMatch = false;
/**
- * Create a filter that uses the provided prefix.
+ * Create a filter that uses the provided prefixes.
* @param prefixes prefixes to use, must not be null. If empty then any
* name will match, if not empty then match on any prefix will
* succeed (logical OR).
*/
public PrefixFilter(String... prefixes) {
Objects.requireNonNull(prefixes);
- this.prefixes = prefixes;
- if (prefixes.length == 0) {
+ if (prefixes.length > 0) {
+ this.prefixes.addAll(Arrays.asList(prefixes));
+ }
+ if (this.prefixes.isEmpty()) {
+ allMatch = true;
+ }
+ }
+
+ public PrefixFilter(Collection<String> prefixes) {
+ Objects.requireNonNull(prefixes);
+ this.prefixes.addAll(prefixes);
+ if (this.prefixes.isEmpty()) {
allMatch = true;
}
}
@@ -148,13 +168,111 @@ public class SolrMetricManager {
}
/**
+ * An implementation of {@link MetricFilter} that selects metrics
+ * with names that match regular expression patterns.
+ */
+ public static class RegexFilter implements MetricFilter {
+ private final Set<Pattern> compiledPatterns = new HashSet<>();
+ private final Set<String> matched = new HashSet<>();
+ private boolean allMatch = false;
+
+ /**
+ * Create a filter that uses the provided prefix.
+ * @param patterns regex patterns to use, must not be null. If empty then any
+ * name will match, if not empty then match on any pattern will
+ * succeed (logical OR).
+ */
+ public RegexFilter(String... patterns) throws PatternSyntaxException {
+ this(patterns != null ? Arrays.asList(patterns) : Collections.emptyList());
+ }
+
+ public RegexFilter(Collection<String> patterns) throws PatternSyntaxException {
+ Objects.requireNonNull(patterns);
+ if (patterns.isEmpty()) {
+ allMatch = true;
+ return;
+ }
+ patterns.forEach(p -> {
+ Pattern pattern = Pattern.compile(p);
+ compiledPatterns.add(pattern);
+ });
+ if (patterns.isEmpty()) {
+ allMatch = true;
+ }
+ }
+
+ @Override
+ public boolean matches(String name, Metric metric) {
+ if (allMatch) {
+ matched.add(name);
+ return true;
+ }
+ for (Pattern p : compiledPatterns) {
+ if (p.matcher(name).matches()) {
+ matched.add(name);
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Return the set of names that matched this filter.
+ * @return matching names
+ */
+ public Set<String> getMatched() {
+ return Collections.unmodifiableSet(matched);
+ }
+
+ /**
+ * Clear the set of names that matched.
+ */
+ public void reset() {
+ matched.clear();
+ }
+ }
+
+ /**
* Return a set of existing registry names.
*/
public Set<String> registryNames() {
Set<String> set = new HashSet<>();
set.addAll(registries.keySet());
set.addAll(SharedMetricRegistries.names());
- return Collections.unmodifiableSet(set);
+ return set;
+ }
+
+ /**
+ * Return set of existing registry names that match a regex pattern
+ * @param patterns regex patterns. NOTE: users need to make sure that patterns that
+ * don't start with a wildcard use the full registry name starting with
+ * {@link #REGISTRY_NAME_PREFIX}
+ * @return set of existing registry names where at least one pattern matched.
+ */
+ public Set<String> registryNames(String... patterns) throws PatternSyntaxException {
+ if (patterns == null || patterns.length == 0) {
+ return registryNames();
+ }
+ List<Pattern> compiled = new ArrayList<>();
+ for (String pattern : patterns) {
+ compiled.add(Pattern.compile(pattern));
+ }
+ return registryNames((Pattern[])compiled.toArray(new Pattern[compiled.size()]));
+ }
+
+ public Set<String> registryNames(Pattern... patterns) {
+ Set<String> allNames = registryNames();
+ if (patterns == null || patterns.length == 0) {
+ return allNames;
+ }
+ return allNames.stream().filter(s -> {
+ for (Pattern p : patterns) {
+ if (p.matcher(s).matches()) {
+ return true;
+ }
+ }
+ return false;
+ }).collect(Collectors.toSet());
}
/**
@@ -694,7 +812,7 @@ public class SolrMetricManager {
attrs.put("name", "replica");
attrs.put("class", SolrReplicaReporter.class.getName());
NamedList initArgs = new NamedList();
- initArgs.add(SolrReporter.GROUP_ID, leaderRegistryName);
+ initArgs.add("groupId", leaderRegistryName);
initArgs.add("period", 5);
PluginInfo pluginInfo = new PluginInfo("reporter", attrs, initArgs, null);
try {
@@ -704,4 +822,26 @@ public class SolrMetricManager {
log.warn("Could not load shard reporter, pluginInfo=" + pluginInfo, e);
}
}
+
+ public void loadNodeReporter(CoreContainer cc, String nodeName) {
+ // don't load for non-cloud instances
+ if (!cc.isZooKeeperAware()) {
+ return;
+ }
+ // load even for non-leader replicas, as their status may change unexpectedly
+ Map<String, String> attrs = new HashMap<>();
+ attrs.put("name", "node");
+ attrs.put("class", SolrNodeReporter.class.getName());
+ NamedList initArgs = new NamedList();
+ initArgs.add("period", 5);
+ PluginInfo pluginInfo = new PluginInfo("reporter", attrs, initArgs, null);
+ String registryName = overridableRegistryName(nodeName);
+ try {
+ SolrMetricReporter reporter = loadReporter(registryName, cc.getResourceLoader(), pluginInfo);
+ ((SolrNodeReporter)reporter).setCoreContainer(cc);
+ } catch (Exception e) {
+ log.warn("Could not load node reporter, pluginInfo=" + pluginInfo, e);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/493544b6/solr/core/src/java/org/apache/solr/metrics/reporters/solr/MetricsReportRequest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/MetricsReportRequest.java b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/MetricsReportRequest.java
deleted file mode 100644
index cd5a8a6..0000000
--- a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/MetricsReportRequest.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.metrics.reporters.solr;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.Reader;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.impl.BinaryRequestWriter;
-import org.apache.solr.client.solrj.response.SimpleSolrResponse;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.ContentStream;
-import org.apache.solr.common.util.JavaBinCodec;
-import org.apache.solr.common.util.NamedList;
-
-/**
- *
- */
-public class MetricsReportRequest extends SolrRequest<SimpleSolrResponse> {
- private SimpleSolrResponse response = new SimpleSolrResponse();
- private SolrParams params;
- private List<ContentStream> contentStreams = null;
- private NamedList content;
-
- public MetricsReportRequest(String path, SolrParams params, NamedList content) {
- super(METHOD.POST, path);
- this.content = content;
- this.params = params;
- }
-
- @Override
- public SolrParams getParams() {
- return params;
- }
-
- @Override
- public Collection<ContentStream> getContentStreams() throws IOException {
- if (contentStreams == null) {
- final BinaryRequestWriter.BAOS baos = new BinaryRequestWriter.BAOS();
- new JavaBinCodec().marshal(content, baos);
- ContentStream cs = new ContentStream() {
- @Override
- public String getName() {
- return null;
- }
-
- @Override
- public String getSourceInfo() {
- return "javabin";
- }
-
- @Override
- public String getContentType() {
- return "application/javabin";
- }
-
- @Override
- public Long getSize() {
- return new Long(baos.size());
- }
-
- @Override
- public InputStream getStream() {
- return new ByteArrayInputStream(baos.getbuf(), 0, baos.size());
- }
-
- @Override
- public Reader getReader() {
- throw new RuntimeException("No reader available . this is a binarystream");
- }
- };
- contentStreams = Collections.singletonList(cs);
- }
- return contentStreams;
- }
-
- @Override
- protected SimpleSolrResponse createResponse(SolrClient client) {
- return response;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/493544b6/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrNodeReporter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrNodeReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrNodeReporter.java
new file mode 100644
index 0000000..fd0b008
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrNodeReporter.java
@@ -0,0 +1,178 @@
+package org.apache.solr.metrics.reporters.solr;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.MalformedURLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
+import org.apache.http.client.HttpClient;
+import org.apache.solr.cloud.Overseer;
+import org.apache.solr.cloud.ZkController;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrInfoMBean;
+import org.apache.solr.handler.admin.MetricsCollectorHandler;
+import org.apache.solr.metrics.SolrMetricManager;
+import org.apache.solr.metrics.SolrMetricReporter;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class SolrNodeReporter extends SolrMetricReporter {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public static final String OVERSEER_GROUP = SolrMetricManager.overridableRegistryName(SolrInfoMBean.Group.overseer.toString());
+
+ public static final List<SolrReporter.Specification> DEFAULT_METRICS = new ArrayList<SolrReporter.Specification>() {{
+ add(new SolrReporter.Specification(OVERSEER_GROUP, "jetty",
+ SolrMetricManager.overridableRegistryName(SolrInfoMBean.Group.jetty.toString()),
+ Collections.singleton(".*\\.mean"))); // all metrics
+ add(new SolrReporter.Specification(OVERSEER_GROUP, "jvm",
+ SolrMetricManager.overridableRegistryName(SolrInfoMBean.Group.jvm.toString()),
+ Collections.emptySet())); // all metrics
+ add(new SolrReporter.Specification(OVERSEER_GROUP, "node", SolrMetricManager.overridableRegistryName(SolrInfoMBean.Group.node.toString()),
+ Collections.emptySet())); // all metrics
+ add(new SolrReporter.Specification(OVERSEER_GROUP, "leader.$1", "solr\\.core\\.(.*)\\.leader",
+ new HashSet<String>(){{
+ add("UPDATE\\./update/.*");
+ add("QUERY\\./select.*");
+ add("INDEX\\..*");
+ add("TLOG\\..*");
+ }}));
+ }};
+
+ private String handler = MetricsCollectorHandler.HANDLER_PATH;
+ private int period = 60;
+ private List<SolrReporter.Specification> metrics = DEFAULT_METRICS;
+
+ private SolrReporter reporter;
+
+ /**
+ * Create a reporter for metrics managed in a named registry.
+ *
+ * @param metricManager metric manager
+ * @param registryName unlike in other reporters, this is the node id
+ */
+ public SolrNodeReporter(SolrMetricManager metricManager, String registryName) {
+ super(metricManager, registryName);
+ }
+
+ public void setHandler(String handler) {
+ this.handler = handler;
+ }
+
+ public void setPeriod(int period) {
+ this.period = period;
+ }
+
+ @Override
+ protected void validate() throws IllegalStateException {
+ if (period < 1) {
+ throw new IllegalStateException("Period must be greater than 0");
+ }
+ // start in setCoreContainer(...)
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (reporter != null) {
+ reporter.close();;
+ }
+ }
+
+ public void setCoreContainer(CoreContainer cc) {
+ // start reporter only in cloud mode
+ if (!cc.isZooKeeperAware()) {
+ return;
+ }
+ HttpClient httpClient = cc.getUpdateShardHandler().getHttpClient();
+ ZkController zk = cc.getZkController();
+ String reporterId = zk.getNodeName();
+ reporter = SolrReporter.Builder.forRegistries(metricManager, metrics)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .withHandler(handler)
+ .withReporterId(reporterId)
+ .cloudClient(false) // we want to send reports specifically to a selected leader instance
+ .skipAggregateValues(true) // we don't want to transport details of aggregates
+ .skipHistograms(true) // we don't want to transport histograms
+ .build(httpClient, new OverseerUrlSupplier(zk));
+
+ reporter.start(period, TimeUnit.SECONDS);
+
+ }
+
+ // TODO: fix this when there is an elegant way to retrieve URL of a node that runs Overseer leader.
+ private static class OverseerUrlSupplier implements Supplier<String> {
+ private static final long DEFAULT_INTERVAL = 30000; // 30s
+ private ZkController zk;
+ private String lastKnownUrl = null;
+ private long lastCheckTime = 0;
+ private long interval = DEFAULT_INTERVAL;
+
+ OverseerUrlSupplier(ZkController zk) {
+ this.zk = zk;
+ }
+
+ @Override
+ public String get() {
+ if (zk == null) {
+ return null;
+ }
+ // primitive caching for interval
+ long now = System.currentTimeMillis();
+ if (lastKnownUrl != null && (now - lastCheckTime) < interval) {
+ return lastKnownUrl;
+ }
+ if (!zk.isConnected()) {
+ return lastKnownUrl;
+ }
+ lastCheckTime = now;
+ SolrZkClient zkClient = zk.getZkClient();
+ ZkNodeProps props;
+ try {
+ props = ZkNodeProps.load(zkClient.getData(
+ Overseer.OVERSEER_ELECT + "/leader", null, null, true));
+ } catch (KeeperException e) {
+ log.warn("Could not obtain overseer's address, skipping.", e);
+ return lastKnownUrl;
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return lastKnownUrl;
+ }
+ if (props == null) {
+ return lastKnownUrl;
+ }
+ String oid = props.getStr("id");
+ if (oid == null) {
+ return lastKnownUrl;
+ }
+ String[] ids = oid.split("-");
+ if (ids.length != 3) { // unknown format
+ log.warn("Unknown format of leader id, skipping: " + oid);
+ return lastKnownUrl;
+ }
+ // convert nodeName back to URL
+ String url = zk.getZkStateReader().getBaseUrlForNodeName(ids[1]);
+ // check that it's parseable
+ try {
+ new java.net.URL(url);
+ } catch (MalformedURLException mue) {
+ log.warn("Malformed Overseer's leader URL: url", mue);
+ return lastKnownUrl;
+ }
+ lastKnownUrl = url;
+ return url;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/493544b6/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReplicaReporter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReplicaReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReplicaReporter.java
index bc4b356..c058591 100644
--- a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReplicaReporter.java
+++ b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReplicaReporter.java
@@ -18,10 +18,11 @@ package org.apache.solr.metrics.reporters.solr;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
-import com.codahale.metrics.MetricFilter;
import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
@@ -40,10 +41,10 @@ public class SolrReplicaReporter extends SolrMetricReporter {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String[] DEFAULT_METRICS = {
- "TLOG", "REPLICATION", "INDEX", "UPDATE./update/", "QUERY./select"
+ "TLOG.*", "REPLICATION.*", "INDEX.flush.*", "INDEX.merge.major.*", "UPDATE\\./update/.*", "QUERY\\./select.*"
};
- private String solrGroupId;
+ private String groupId;
private String handler = MetricsCollectorHandler.HANDLER_PATH;
private int period = 60;
private String[] metrics = DEFAULT_METRICS;
@@ -61,8 +62,8 @@ public class SolrReplicaReporter extends SolrMetricReporter {
super(metricManager, registryName);
}
- public void setSolrGroupId(String solrGroupId) {
- this.solrGroupId = solrGroupId;
+ public void setGroupId(String groupId) {
+ this.groupId = groupId;
}
public void setHandler(String handler) {
@@ -107,17 +108,17 @@ public class SolrReplicaReporter extends SolrMetricReporter {
log.warn("Not initializing replica reporter for non-cloud core " + core.getName());
return;
}
- // our id is nodeName
+ // our id is coreNodeName
String id = core.getCoreDescriptor().getCloudDescriptor().getCoreNodeName();
- MetricFilter filter = new SolrMetricManager.PrefixFilter(metrics);
- reporter = SolrReporter.Builder.forRegistry(metricManager.registry(registryName))
+ SolrReporter.Specification spec = new SolrReporter.Specification(groupId, null, registryName, Arrays.asList(metrics));
+ reporter = SolrReporter.Builder.forRegistries(metricManager, Collections.singletonList(spec))
.convertRatesTo(TimeUnit.SECONDS)
.convertDurationsTo(TimeUnit.MILLISECONDS)
.withHandler(handler)
- .filter(filter)
- .withId(id)
+ .withReporterId(id)
.cloudClient(false) // we want to send reports specifically to a selected leader instance
- .withGroup(solrGroupId)
+ .skipAggregateValues(true) // we don't want to transport details of aggregates
+ .skipHistograms(true) // we don't want to transport histograms
.build(core.getCoreDescriptor().getCoreContainer().getUpdateShardHandler().getHttpClient(), new LeaderUrlSupplier(core));
reporter.start(period, TimeUnit.SECONDS);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/493544b6/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java
index 36a9f5a..a2fe50b 100644
--- a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java
+++ b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java
@@ -16,35 +16,37 @@
*/
package org.apache.solr.metrics.reporters.solr;
-import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.SortedMap;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricFilter;
-import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.ScheduledReporter;
import com.codahale.metrics.Timer;
import org.apache.http.client.HttpClient;
import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.io.SolrClientCache;
import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.JavaBinCodec;
-import org.apache.solr.common.util.NamedList;
import org.apache.solr.handler.admin.MetricsCollectorHandler;
+import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.util.stats.MetricUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,28 +57,72 @@ import org.slf4j.LoggerFactory;
public class SolrReporter extends ScheduledReporter {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ public static final String REGISTRY_ID = "_registry_";
+ public static final String REPORTER_ID = "_reporter_";
+ public static final String GROUP_ID = "_group_";
+ public static final String LABEL_ID = "_label_";
+
+
+ /**
+ * Specification of what registries and what metrics to send.
+ */
+ public static final class Specification {
+ public String groupPattern;
+ public String labelPattern;
+ public String registryPattern;
+ public Set<String> metricPatterns = new HashSet<>();
+
+ /**
+ * Create a specification
+ * @param groupPattern logical group for these metrics. This is used in {@link MetricsCollectorHandler} to select
+ * the target registry for metrics to aggregate. It may contain back-references to capture groups from
+ * {@code registryPattern}
+ * @param labelPattern name of this group of metrics. This is used in {@link MetricsCollectorHandler} to prefix
+ * metric names. May be null or empty. It may contain back-references to capture groups from
+ * {@code registryPattern}.
+ * @param registryPattern pattern for selecting matching registries, see {@link SolrMetricManager#registryNames(String...)}
+ * @param metricPatterns patterns for selecting matching metrics, see {@link SolrMetricManager.RegexFilter}
+ */
+ public Specification(String groupPattern, String labelPattern, String registryPattern, Collection<String> metricPatterns) {
+ this.groupPattern = groupPattern;
+ this.labelPattern = labelPattern;
+ this.registryPattern = registryPattern;
+ if (metricPatterns != null) {
+ this.metricPatterns.addAll(metricPatterns);
+ }
+ }
+ }
+
public static class Builder {
- private final MetricRegistry registry;
- private String id;
- private String group;
+ private final SolrMetricManager metricManager;
+ private final List<Specification> metrics;
+ private String reporterId;
private TimeUnit rateUnit;
private TimeUnit durationUnit;
- private MetricFilter filter;
private String handler;
private boolean skipHistograms;
+ private boolean skipAggregateValues;
private boolean cloudClient;
private SolrParams params;
- public static Builder forRegistry(MetricRegistry registry) {
- return new Builder(registry);
+ /**
+ * Create a builder for SolrReporter.
+ * @param metricManager metric manager that is the source of metrics
+ * @param metrics patterns to select registries, see {@link SolrMetricManager#registryNames(String...)},
+ * and the corresponding metrics prefixes, see {@link org.apache.solr.metrics.SolrMetricManager.PrefixFilter}.
+ * @return builder
+ */
+ public static Builder forRegistries(SolrMetricManager metricManager, List<Specification> metrics) {
+ return new Builder(metricManager, metrics);
}
- private Builder(MetricRegistry registry) {
- this.registry = registry;
+ private Builder(SolrMetricManager metricManager, List<Specification> metrics) {
+ this.metricManager = metricManager;
+ this.metrics = metrics;
this.rateUnit = TimeUnit.SECONDS;
this.durationUnit = TimeUnit.MILLISECONDS;
- this.filter = MetricFilter.ALL;
this.skipHistograms = false;
+ this.skipAggregateValues = false;
this.cloudClient = false;
this.params = null;
}
@@ -113,35 +159,34 @@ public class SolrReporter extends ScheduledReporter {
}
/**
- * Handler name to use at the remote end.
- *
- * @param handler handler name, eg. "/admin/metricsCollector"
+ * Individual values from {@link org.apache.solr.metrics.AggregateMetric} may not be worth to report.
+ * @param skipAggregateValues
* @return {@code this}
*/
- public Builder withHandler(String handler) {
- this.handler = handler;
+ public Builder skipAggregateValues(boolean skipAggregateValues) {
+ this.skipAggregateValues = skipAggregateValues;
return this;
}
/**
- * Use this id to identify metrics from this instance.
+ * Handler name to use at the remote end.
*
- * @param id
+ * @param handler handler name, eg. "/admin/metricsCollector"
* @return {@code this}
*/
- public Builder withId(String id) {
- this.id = id;
+ public Builder withHandler(String handler) {
+ this.handler = handler;
return this;
}
/**
- * Use this id to identify a logical group of reports.
+ * Use this id to identify metrics from this instance.
*
- * @param group
+ * @param reporterId
* @return {@code this}
*/
- public Builder withGroup(String group) {
- this.group = group;
+ public Builder withReporterId(String reporterId) {
+ this.reporterId = reporterId;
return this;
}
@@ -168,17 +213,6 @@ public class SolrReporter extends ScheduledReporter {
}
/**
- * Only report metrics which match the given filter.
- *
- * @param filter a {@link MetricFilter}
- * @return {@code this}
- */
- public Builder filter(MetricFilter filter) {
- this.filter = filter;
- return this;
- }
-
- /**
* Build it.
* @param client an instance of {@link HttpClient} to be used for making calls.
* @param urlProvider function that returns the base URL of Solr instance to target. May return
@@ -187,59 +221,78 @@ public class SolrReporter extends ScheduledReporter {
* @return configured instance of reporter
*/
public SolrReporter build(HttpClient client, Supplier<String> urlProvider) {
- return new SolrReporter(client, urlProvider, registry, handler, id, group, rateUnit, durationUnit,
- filter, params, skipHistograms, cloudClient);
+ return new SolrReporter(client, urlProvider, metricManager, metrics, handler, reporterId, rateUnit, durationUnit,
+ params, skipHistograms, skipAggregateValues, cloudClient);
}
}
- public static final String REPORTER_ID = "solrReporterId";
- public static final String GROUP_ID = "solrGroupId";
-
- private String id;
- private String group;
+ private String reporterId;
private String handler;
private Supplier<String> urlProvider;
private SolrClientCache clientCache;
- private List<MetricFilter> filters;
- private MetricRegistry visibleRegistry;
+ private List<CompiledSpecification> specs;
+ private SolrMetricManager metricManager;
private boolean skipHistograms;
+ private boolean skipAggregateValues;
private boolean cloudClient;
private ModifiableSolrParams params;
private Map<String, Object> metadata;
- public SolrReporter(HttpClient httpClient, Supplier<String> urlProvider, MetricRegistry registry, String handler,
- String id, String group, TimeUnit rateUnit, TimeUnit durationUnit, MetricFilter filter,
- SolrParams params, boolean skipHistograms, boolean cloudClient) {
- super(registry, "solr-reporter", filter, rateUnit, durationUnit);
+ private static final class CompiledSpecification {
+ String group;
+ String label;
+ Pattern registryPattern;
+ MetricFilter filter;
+
+ CompiledSpecification(Specification spec) throws PatternSyntaxException {
+ this.group = spec.groupPattern;
+ this.label = spec.labelPattern;
+ this.registryPattern = Pattern.compile(spec.registryPattern);
+ this.filter = new SolrMetricManager.RegexFilter(spec.metricPatterns);
+ }
+ }
+
+ public SolrReporter(HttpClient httpClient, Supplier<String> urlProvider, SolrMetricManager metricManager,
+ List<Specification> metrics, String handler,
+ String reporterId, TimeUnit rateUnit, TimeUnit durationUnit,
+ SolrParams params, boolean skipHistograms, boolean skipAggregateValues, boolean cloudClient) {
+ super(null, "solr-reporter", MetricFilter.ALL, rateUnit, durationUnit);
+ this.metricManager = metricManager;
this.urlProvider = urlProvider;
- this.id = id;
- this.group = group;
+ this.reporterId = reporterId;
if (handler == null) {
handler = MetricsCollectorHandler.HANDLER_PATH;
}
this.handler = handler;
this.clientCache = new SolrClientCache(httpClient);
- // the one in superclass is invisible... :(
- this.visibleRegistry = registry;
- if (filter == null) {
- filter = MetricFilter.ALL;
- }
- this.filters = Collections.singletonList(filter);
+ this.specs = new ArrayList<>();
+ metrics.forEach(spec -> {
+ MetricFilter filter = new SolrMetricManager.RegexFilter(spec.metricPatterns);
+ try {
+ CompiledSpecification cs = new CompiledSpecification(spec);
+ specs.add(cs);
+ } catch (PatternSyntaxException e) {
+ log.warn("Skipping spec with invalid registryPattern: " + spec.registryPattern, e);
+ }
+ });
this.skipHistograms = skipHistograms;
+ this.skipAggregateValues = skipAggregateValues;
this.cloudClient = cloudClient;
this.params = new ModifiableSolrParams();
- this.params.set(REPORTER_ID, id);
- this.params.set(GROUP_ID, group);
+ this.params.set(REPORTER_ID, reporterId);
// allow overrides to take precedence
if (params != null) {
this.params.add(params);
}
metadata = new HashMap<>();
- metadata.put(REPORTER_ID, id);
- if (group != null) {
- metadata.put(GROUP_ID, group);
- }
+ metadata.put(REPORTER_ID, reporterId);
+ }
+
+ @Override
+ public void close() {
+ clientCache.close();
+ super.close();
}
@Override
@@ -258,16 +311,40 @@ public class SolrReporter extends ScheduledReporter {
}
UpdateRequest req = new UpdateRequest(handler);
req.setParams(params);
- MetricUtils.toSolrInputDocuments(visibleRegistry, filters, MetricFilter.ALL,
- skipHistograms, metadata, doc -> req.add(doc));
+ specs.forEach(spec -> {
+ Set<String> registryNames = metricManager.registryNames(spec.registryPattern);
+ registryNames.forEach(registryName -> {
+ String label = spec.label;
+ if (label != null && label.indexOf('$') != -1) {
+ // label with back-references
+ Matcher m = spec.registryPattern.matcher(registryName);
+ label = m.replaceFirst(label);
+ }
+ final String effectiveLabel = label;
+ String group = spec.group;
+ if (group.indexOf('$') != -1) {
+ // group with back-references
+ Matcher m = spec.registryPattern.matcher(registryName);
+ group = m.replaceFirst(group);
+ }
+ final String effectiveGroup = group;
+ MetricUtils.toSolrInputDocuments(metricManager.registry(registryName), Collections.singletonList(spec.filter), MetricFilter.ALL,
+ skipHistograms, skipAggregateValues, metadata, doc -> {
+ doc.setField(REGISTRY_ID, registryName);
+ doc.setField(GROUP_ID, effectiveGroup);
+ if (effectiveLabel != null) {
+ doc.setField(LABEL_ID, effectiveLabel);
+ }
+ req.add(doc);
+ });
+ });
+ });
try {
//log.info("%%% sending to " + url + ": " + req.getParams());
solr.request(req);
- } catch (SolrServerException sse) {
- log.warn("Error sending metric report", sse);
- } catch (IOException ioe) {
- log.warn("Error sending metric report", ioe);
+ } catch (Exception e) {
+ log.debug("Error sending metric report", e.toString());
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/493544b6/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java
index 13add27..6c5a6f0 100644
--- a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java
+++ b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java
@@ -16,7 +16,6 @@
*/
package org.apache.solr.util.stats;
-import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
@@ -46,7 +45,7 @@ import org.apache.solr.metrics.AggregateMetric;
*/
public class MetricUtils {
- public static final String NAME = "name";
+ public static final String METRIC_NAME = "metric";
public static final String VALUES = "values";
static final String MS = "_ms";
@@ -61,6 +60,7 @@ public class MetricUtils {
static final String MEDIAN_MS = MEDIAN + MS;
static final String STDDEV = "stddev";
static final String STDDEV_MS = STDDEV + MS;
+ static final String SUM = "sum";
static final String P75 = "p75";
static final String P75_MS = P75 + MS;
static final String P95 = "p95";
@@ -115,9 +115,10 @@ public class MetricUtils {
*/
public static NamedList toNamedList(MetricRegistry registry, List<MetricFilter> shouldMatchFilters,
MetricFilter mustMatchFilter, boolean skipHistograms,
+ boolean skipAggregateValues,
Map<String, Object> metadata) {
NamedList result = new NamedList();
- toNamedMaps(registry, shouldMatchFilters, mustMatchFilter, skipHistograms, (k, v) -> {
+ toNamedMaps(registry, shouldMatchFilters, mustMatchFilter, skipHistograms, skipAggregateValues, (k, v) -> {
result.add(k, new NamedList(v));
});
if (metadata != null && !metadata.isEmpty()) {
@@ -144,9 +145,11 @@ public class MetricUtils {
*/
public static List<SolrInputDocument> toSolrInputDocuments(MetricRegistry registry, List<MetricFilter> shouldMatchFilters,
MetricFilter mustMatchFilter, boolean skipHistograms,
+ boolean skipAggregateValues,
Map<String, Object> metadata) {
List<SolrInputDocument> result = new LinkedList<>();
- toSolrInputDocuments(registry, shouldMatchFilters, mustMatchFilter, skipHistograms, metadata, doc -> {
+ toSolrInputDocuments(registry, shouldMatchFilters, mustMatchFilter, skipHistograms,
+ skipAggregateValues, metadata, doc -> {
result.add(doc);
});
return result;
@@ -154,11 +157,12 @@ public class MetricUtils {
public static void toSolrInputDocuments(MetricRegistry registry, List<MetricFilter> shouldMatchFilters,
MetricFilter mustMatchFilter, boolean skipHistograms,
+ boolean skipAggregateValues,
Map<String, Object> metadata, Consumer<SolrInputDocument> consumer) {
boolean addMetadata = metadata != null && !metadata.isEmpty();
- toNamedMaps(registry, shouldMatchFilters, mustMatchFilter, skipHistograms, (k, v) -> {
+ toNamedMaps(registry, shouldMatchFilters, mustMatchFilter, skipHistograms, skipAggregateValues, (k, v) -> {
SolrInputDocument doc = new SolrInputDocument();
- doc.setField(NAME, k);
+ doc.setField(METRIC_NAME, k);
toSolrInputDocument(null, doc, v);
if (addMetadata) {
toSolrInputDocument(null, doc, metadata);
@@ -179,7 +183,7 @@ public class MetricUtils {
}
public static void toNamedMaps(MetricRegistry registry, List<MetricFilter> shouldMatchFilters,
- MetricFilter mustMatchFilter, boolean skipHistograms,
+ MetricFilter mustMatchFilter, boolean skipHistograms, boolean skipAggregateValues,
BiConsumer<String, Map<String, Object>> consumer) {
Map<String, Metric> metrics = registry.getMetrics();
SortedSet<String> names = registry.getNames();
@@ -206,19 +210,20 @@ public class MetricUtils {
consumer.accept(n, histogramToMap(histogram));
}
} else if (metric instanceof AggregateMetric) {
- consumer.accept(n, aggregateMetricToMap((AggregateMetric)metric));
+ consumer.accept(n, aggregateMetricToMap((AggregateMetric)metric, skipAggregateValues));
}
});
}
- static Map<String, Object> aggregateMetricToMap(AggregateMetric metric) {
+ static Map<String, Object> aggregateMetricToMap(AggregateMetric metric, boolean skipAggregateValues) {
Map<String, Object> response = new LinkedHashMap<>();
response.put("count", metric.size());
response.put(MAX, metric.getMax());
response.put(MIN, metric.getMin());
response.put(MEAN, metric.getMean());
response.put(STDDEV, metric.getStdDev());
- if (!metric.isEmpty()) {
+ response.put(SUM, metric.getSum());
+ if (!(metric.isEmpty() || skipAggregateValues)) {
Map<String, Object> values = new LinkedHashMap<>();
response.put(VALUES, values);
metric.getValues().forEach((k, v) -> {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/493544b6/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrReplicaReporterTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrReplicaReporterTest.java b/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrReplicaReporterTest.java
index 70a785e..2f6dbfb 100644
--- a/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrReplicaReporterTest.java
+++ b/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrReplicaReporterTest.java
@@ -86,5 +86,8 @@ public class SolrReplicaReporterTest extends AbstractFullDistribZkTestBase {
metricManager.registryNames().contains(registryName));
}
}
+ SolrMetricManager metricManager = controlJetty.getCoreContainer().getMetricManager();
+ assertTrue(metricManager.registryNames().contains("solr.overseer"));
+ Map<String, Metric> metrics = metricManager.registry("solr.overseer").getMetrics();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/493544b6/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java b/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java
index 8dcf063..a1993aa 100644
--- a/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java
+++ b/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java
@@ -79,7 +79,7 @@ public class MetricUtilsTest extends SolrTestCaseJ4 {
am.set("bar", 1);
am.set("bar", 2);
MetricUtils.toNamedMaps(registry, Collections.singletonList(MetricFilter.ALL), MetricFilter.ALL,
- false, (k, v) -> {
+ false, false, (k, v) -> {
if (k.startsWith("counter")) {
assertEquals(1L, v.get("count"));
} else if (k.startsWith("timer")) {