You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2017/01/26 17:09:20 UTC
lucene-solr:jira/solr-9858: SOLR-9858 Process plugin configuration
from solr.xml. Add unit tests.
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-9858 493544b6e -> cba0956f3
SOLR-9858 Process plugin configuration from solr.xml. Add unit tests.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/cba0956f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/cba0956f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/cba0956f
Branch: refs/heads/jira/solr-9858
Commit: cba0956f36308e52385b8457b64c312be7dff08e
Parents: 493544b
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Thu Jan 26 18:08:26 2017 +0100
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Thu Jan 26 18:08:26 2017 +0100
----------------------------------------------------------------------
.../org/apache/solr/core/CoreContainer.java | 4 +-
.../org/apache/solr/core/SolrInfoMBean.java | 2 +-
.../org/apache/solr/core/SolrXmlConfig.java | 3 +-
.../handler/admin/MetricsCollectorHandler.java | 24 ++-
.../solr/metrics/SolrCoreMetricManager.java | 2 +-
.../apache/solr/metrics/SolrMetricManager.java | 124 ++++++++++---
.../reporters/solr/SolrNodeReporter.java | 178 ------------------
.../reporters/solr/SolrOverseerReporter.java | 186 +++++++++++++++++++
.../reporters/solr/SolrReplicaReporter.java | 12 +-
.../src/test-files/solr/solr-solrreporter.xml | 48 +++++
.../reporters/solr/SolrCloudReportersTest.java | 122 ++++++++++++
11 files changed, 486 insertions(+), 219 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cba0956f/solr/core/src/java/org/apache/solr/core/CoreContainer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 46c3c8e..29b2229 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -543,7 +543,7 @@ public class CoreContainer {
unloadedCores, true, "unloaded",SolrInfoMBean.Category.CONTAINER.toString(), "cores");
if (isZooKeeperAware()) {
- metricManager.loadNodeReporter(this, this.getZkController().getNodeName());
+ metricManager.loadOverseerReporters(cfg.getMetricReporterPlugins(), this);
}
// setup executor to load cores in parallel
@@ -677,7 +677,7 @@ public class CoreContainer {
cancelCoreRecoveries();
zkSys.zkController.publishNodeAsDown(zkSys.zkController.getNodeName());
if (metricManager != null) {
- metricManager.closeReporters(SolrMetricManager.overridableRegistryName(zkSys.zkController.getNodeName()));
+ metricManager.closeReporters(SolrMetricManager.getRegistryName(SolrInfoMBean.Group.overseer));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cba0956f/solr/core/src/java/org/apache/solr/core/SolrInfoMBean.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrInfoMBean.java b/solr/core/src/java/org/apache/solr/core/SolrInfoMBean.java
index d7424a0..e083621 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrInfoMBean.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrInfoMBean.java
@@ -38,7 +38,7 @@ public interface SolrInfoMBean {
/**
* Top-level group of beans for a subsystem.
*/
- enum Group { jvm, jetty, node, core, overseer }
+ enum Group { jvm, jetty, node, core, replica, overseer }
/**
* Simple common usage name, e.g. BasicQueryHandler,
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cba0956f/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
index e41cd8d..951d8d5 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
@@ -451,7 +451,8 @@ public class SolrXmlConfig {
return new PluginInfo[0];
PluginInfo[] configs = new PluginInfo[nodes.getLength()];
for (int i = 0; i < nodes.getLength(); i++) {
- configs[i] = new PluginInfo(nodes.item(i), "SolrMetricReporter", true, true);
+ // we don't require class in order to support predefined replica and node reporter classes
+ configs[i] = new PluginInfo(nodes.item(i), "SolrMetricReporter", true, false);
}
return configs;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cba0956f/solr/core/src/java/org/apache/solr/handler/admin/MetricsCollectorHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsCollectorHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsCollectorHandler.java
index 0dc7412..829541e 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsCollectorHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsCollectorHandler.java
@@ -150,11 +150,7 @@ public class MetricsCollectorHandler extends RequestHandlerBase {
doc.forEach(f -> {
String key = MetricRegistry.name(labelId, metricName, f.getName());
MetricRegistry registry = metricManager.registry(groupId);
- AggregateMetric metric = (AggregateMetric)registry.getMetrics().get(key);
- if (metric == null) {
- metric = new AggregateMetric();
- registry.register(key, metric);
- }
+ AggregateMetric metric = getOrRegister(registry, key, new AggregateMetric());
Object o = f.getFirstValue();
if (o != null) {
metric.set(reporterId, o);
@@ -165,6 +161,24 @@ public class MetricsCollectorHandler extends RequestHandlerBase {
});
}
+ private AggregateMetric getOrRegister(MetricRegistry registry, String name, AggregateMetric add) {
+ AggregateMetric existing = (AggregateMetric)registry.getMetrics().get(name);
+ if (existing != null) {
+ return existing;
+ }
+ try {
+ registry.register(name, add);
+ return add;
+ } catch (IllegalArgumentException e) {
+ // someone added before us
+ existing = (AggregateMetric)registry.getMetrics().get(name);
+ if (existing == null) { // now, that is weird...
+ throw new IllegalArgumentException("Inconsistent metric status, " + name);
+ }
+ return existing;
+ }
+ }
+
@Override
public void processDelete(DeleteUpdateCommand cmd) throws IOException {
throw new UnsupportedOperationException("processDelete");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cba0956f/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java
index 912380c..da901e2 100644
--- a/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java
+++ b/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java
@@ -77,7 +77,7 @@ public class SolrCoreMetricManager implements Closeable {
PluginInfo[] pluginInfos = nodeConfig.getMetricReporterPlugins();
metricManager.loadReporters(pluginInfos, core.getResourceLoader(), SolrInfoMBean.Group.core, registryName);
if (cloudMode) {
- metricManager.loadReplicaReporter(core, leaderRegistryName, registryName);
+ metricManager.loadReplicaReporters(pluginInfos, core, leaderRegistryName, registryName);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cba0956f/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
index e807925..f34ef17 100644
--- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
+++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
@@ -46,17 +46,14 @@ import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.MetricSet;
import com.codahale.metrics.SharedMetricRegistries;
import com.codahale.metrics.Timer;
-import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoMBean;
import org.apache.solr.core.SolrResourceLoader;
-import org.apache.solr.metrics.reporters.solr.SolrNodeReporter;
+import org.apache.solr.metrics.reporters.solr.SolrOverseerReporter;
import org.apache.solr.metrics.reporters.solr.SolrReplicaReporter;
-import org.apache.solr.metrics.reporters.solr.SolrReporter;
-import org.apache.solr.util.plugin.SolrCoreAware;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -802,45 +799,114 @@ public class SolrMetricManager {
}
}
- public void loadReplicaReporter(SolrCore core, String leaderRegistryName, String registryName) {
+ private List<PluginInfo> prepareCloudPlugins(PluginInfo[] pluginInfos, String group, String className,
+ Map<String, String> defaultAttributes,
+ Map<String, Object> defaultInitArgs,
+ PluginInfo defaultPlugin) {
+ List<PluginInfo> result = new ArrayList<>();
+ if (pluginInfos == null) {
+ pluginInfos = new PluginInfo[0];
+ }
+ for (PluginInfo info : pluginInfos) {
+ String groupAttr = info.attributes.get("group");
+ if (!group.equals(groupAttr)) {
+ continue;
+ }
+ info = preparePlugin(info, className, defaultAttributes, defaultInitArgs);
+ if (info != null) {
+ result.add(info);
+ }
+ }
+ if (result.isEmpty()) {
+ defaultPlugin = preparePlugin(defaultPlugin, className, defaultAttributes, defaultInitArgs);
+ if (defaultPlugin != null) {
+ result.add(defaultPlugin);
+ }
+ }
+ return result;
+ }
+
+ private PluginInfo preparePlugin(PluginInfo info, String className, Map<String, String> defaultAttributes,
+ Map<String, Object> defaultInitArgs) {
+ if (info == null) {
+ return null;
+ }
+ String classNameAttr = info.attributes.get("class");
+ if (className != null) {
+ if (classNameAttr != null && !className.equals(classNameAttr)) {
+ log.warn("Conflicting class name attributes, expected " + className + " but was " + classNameAttr + ", skipping " + info);
+ return null;
+ }
+ }
+
+ Map<String, String> attrs = new HashMap<>(info.attributes);
+ defaultAttributes.forEach((k, v) -> {
+ if (!attrs.containsKey(k)) {
+ attrs.put(k, v);
+ }
+ });
+ attrs.put("class", className);
+ Map<String, Object> initArgs = new HashMap<>();
+ if (info.initArgs != null) {
+ initArgs.putAll(info.initArgs.asMap(10));
+ }
+ defaultInitArgs.forEach((k, v) -> {
+ if (!initArgs.containsKey(k)) {
+ initArgs.put(k, v);
+ }
+ });
+ return new PluginInfo(info.type, attrs, new NamedList(initArgs), null);
+ }
+
+ public void loadReplicaReporters(PluginInfo[] pluginInfos, SolrCore core, String leaderRegistryName, String registryName) {
// don't load for non-cloud cores
if (leaderRegistryName == null) {
return;
}
- // load even for non-leader replicas, as their status may change unexpectedly
+ // prepare default plugin if none present in the config
Map<String, String> attrs = new HashMap<>();
- attrs.put("name", "replica");
- attrs.put("class", SolrReplicaReporter.class.getName());
- NamedList initArgs = new NamedList();
- initArgs.add("groupId", leaderRegistryName);
- initArgs.add("period", 5);
- PluginInfo pluginInfo = new PluginInfo("reporter", attrs, initArgs, null);
- try {
- SolrMetricReporter reporter = loadReporter(registryName, core.getResourceLoader(), pluginInfo);
- ((SolrReplicaReporter)reporter).setCore(core);
- } catch (Exception e) {
- log.warn("Could not load shard reporter, pluginInfo=" + pluginInfo, e);
+ attrs.put("name", "replicaDefault");
+ attrs.put("group", "replica");
+ Map<String, Object> initArgs = new HashMap<>();
+ initArgs.put("groupId", leaderRegistryName);
+ initArgs.put("period", 30);
+ PluginInfo defaultPlugin = new PluginInfo("reporter", attrs, new NamedList(), null);
+
+ // collect infos and normalize
+ List<PluginInfo> infos = prepareCloudPlugins(pluginInfos, "replica", SolrReplicaReporter.class.getName(),
+ attrs, initArgs, defaultPlugin);
+ for (PluginInfo info : infos) {
+ try {
+ SolrMetricReporter reporter = loadReporter(registryName, core.getResourceLoader(), info);
+ ((SolrReplicaReporter)reporter).setCore(core);
+ } catch (Exception e) {
+ log.warn("Could not load shard reporter, pluginInfo=" + info, e);
+ }
}
}
- public void loadNodeReporter(CoreContainer cc, String nodeName) {
+ public void loadOverseerReporters(PluginInfo[] pluginInfos, CoreContainer cc) {
// don't load for non-cloud instances
if (!cc.isZooKeeperAware()) {
return;
}
// load even for non-leader replicas, as their status may change unexpectedly
Map<String, String> attrs = new HashMap<>();
- attrs.put("name", "node");
- attrs.put("class", SolrNodeReporter.class.getName());
- NamedList initArgs = new NamedList();
- initArgs.add("period", 5);
- PluginInfo pluginInfo = new PluginInfo("reporter", attrs, initArgs, null);
- String registryName = overridableRegistryName(nodeName);
- try {
- SolrMetricReporter reporter = loadReporter(registryName, cc.getResourceLoader(), pluginInfo);
- ((SolrNodeReporter)reporter).setCoreContainer(cc);
- } catch (Exception e) {
- log.warn("Could not load node reporter, pluginInfo=" + pluginInfo, e);
+ attrs.put("name", "overseerDefault");
+ attrs.put("group", "overseer");
+ Map<String, Object> initArgs = new HashMap<>();
+ initArgs.put("period", 30);
+ PluginInfo defaultPlugin = new PluginInfo("reporter", attrs, new NamedList(), null);
+ List<PluginInfo> infos = prepareCloudPlugins(pluginInfos, "overseer", SolrOverseerReporter.class.getName(),
+ attrs, initArgs, defaultPlugin);
+ String registryName = getRegistryName(SolrInfoMBean.Group.overseer);
+ for (PluginInfo info : infos) {
+ try {
+ SolrMetricReporter reporter = loadReporter(registryName, cc.getResourceLoader(), info);
+ ((SolrOverseerReporter)reporter).setCoreContainer(cc);
+ } catch (Exception e) {
+ log.warn("Could not load node reporter, pluginInfo=" + info, e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cba0956f/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrNodeReporter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrNodeReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrNodeReporter.java
deleted file mode 100644
index fd0b008..0000000
--- a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrNodeReporter.java
+++ /dev/null
@@ -1,178 +0,0 @@
-package org.apache.solr.metrics.reporters.solr;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.net.MalformedURLException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Supplier;
-
-import org.apache.http.client.HttpClient;
-import org.apache.solr.cloud.Overseer;
-import org.apache.solr.cloud.ZkController;
-import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.common.cloud.ZkNodeProps;
-import org.apache.solr.core.CoreContainer;
-import org.apache.solr.core.SolrInfoMBean;
-import org.apache.solr.handler.admin.MetricsCollectorHandler;
-import org.apache.solr.metrics.SolrMetricManager;
-import org.apache.solr.metrics.SolrMetricReporter;
-import org.apache.zookeeper.KeeperException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- */
-public class SolrNodeReporter extends SolrMetricReporter {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- public static final String OVERSEER_GROUP = SolrMetricManager.overridableRegistryName(SolrInfoMBean.Group.overseer.toString());
-
- public static final List<SolrReporter.Specification> DEFAULT_METRICS = new ArrayList<SolrReporter.Specification>() {{
- add(new SolrReporter.Specification(OVERSEER_GROUP, "jetty",
- SolrMetricManager.overridableRegistryName(SolrInfoMBean.Group.jetty.toString()),
- Collections.singleton(".*\\.mean"))); // all metrics
- add(new SolrReporter.Specification(OVERSEER_GROUP, "jvm",
- SolrMetricManager.overridableRegistryName(SolrInfoMBean.Group.jvm.toString()),
- Collections.emptySet())); // all metrics
- add(new SolrReporter.Specification(OVERSEER_GROUP, "node", SolrMetricManager.overridableRegistryName(SolrInfoMBean.Group.node.toString()),
- Collections.emptySet())); // all metrics
- add(new SolrReporter.Specification(OVERSEER_GROUP, "leader.$1", "solr\\.core\\.(.*)\\.leader",
- new HashSet<String>(){{
- add("UPDATE\\./update/.*");
- add("QUERY\\./select.*");
- add("INDEX\\..*");
- add("TLOG\\..*");
- }}));
- }};
-
- private String handler = MetricsCollectorHandler.HANDLER_PATH;
- private int period = 60;
- private List<SolrReporter.Specification> metrics = DEFAULT_METRICS;
-
- private SolrReporter reporter;
-
- /**
- * Create a reporter for metrics managed in a named registry.
- *
- * @param metricManager metric manager
- * @param registryName unlike in other reporters, this is the node id
- */
- public SolrNodeReporter(SolrMetricManager metricManager, String registryName) {
- super(metricManager, registryName);
- }
-
- public void setHandler(String handler) {
- this.handler = handler;
- }
-
- public void setPeriod(int period) {
- this.period = period;
- }
-
- @Override
- protected void validate() throws IllegalStateException {
- if (period < 1) {
- throw new IllegalStateException("Period must be greater than 0");
- }
- // start in setCoreContainer(...)
- }
-
- @Override
- public void close() throws IOException {
- if (reporter != null) {
- reporter.close();;
- }
- }
-
- public void setCoreContainer(CoreContainer cc) {
- // start reporter only in cloud mode
- if (!cc.isZooKeeperAware()) {
- return;
- }
- HttpClient httpClient = cc.getUpdateShardHandler().getHttpClient();
- ZkController zk = cc.getZkController();
- String reporterId = zk.getNodeName();
- reporter = SolrReporter.Builder.forRegistries(metricManager, metrics)
- .convertRatesTo(TimeUnit.SECONDS)
- .convertDurationsTo(TimeUnit.MILLISECONDS)
- .withHandler(handler)
- .withReporterId(reporterId)
- .cloudClient(false) // we want to send reports specifically to a selected leader instance
- .skipAggregateValues(true) // we don't want to transport details of aggregates
- .skipHistograms(true) // we don't want to transport histograms
- .build(httpClient, new OverseerUrlSupplier(zk));
-
- reporter.start(period, TimeUnit.SECONDS);
-
- }
-
- // TODO: fix this when there is an elegant way to retrieve URL of a node that runs Overseer leader.
- private static class OverseerUrlSupplier implements Supplier<String> {
- private static final long DEFAULT_INTERVAL = 30000; // 30s
- private ZkController zk;
- private String lastKnownUrl = null;
- private long lastCheckTime = 0;
- private long interval = DEFAULT_INTERVAL;
-
- OverseerUrlSupplier(ZkController zk) {
- this.zk = zk;
- }
-
- @Override
- public String get() {
- if (zk == null) {
- return null;
- }
- // primitive caching for interval
- long now = System.currentTimeMillis();
- if (lastKnownUrl != null && (now - lastCheckTime) < interval) {
- return lastKnownUrl;
- }
- if (!zk.isConnected()) {
- return lastKnownUrl;
- }
- lastCheckTime = now;
- SolrZkClient zkClient = zk.getZkClient();
- ZkNodeProps props;
- try {
- props = ZkNodeProps.load(zkClient.getData(
- Overseer.OVERSEER_ELECT + "/leader", null, null, true));
- } catch (KeeperException e) {
- log.warn("Could not obtain overseer's address, skipping.", e);
- return lastKnownUrl;
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- return lastKnownUrl;
- }
- if (props == null) {
- return lastKnownUrl;
- }
- String oid = props.getStr("id");
- if (oid == null) {
- return lastKnownUrl;
- }
- String[] ids = oid.split("-");
- if (ids.length != 3) { // unknown format
- log.warn("Unknown format of leader id, skipping: " + oid);
- return lastKnownUrl;
- }
- // convert nodeName back to URL
- String url = zk.getZkStateReader().getBaseUrlForNodeName(ids[1]);
- // check that it's parseable
- try {
- new java.net.URL(url);
- } catch (MalformedURLException mue) {
- log.warn("Malformed Overseer's leader URL: url", mue);
- return lastKnownUrl;
- }
- lastKnownUrl = url;
- return url;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cba0956f/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrOverseerReporter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrOverseerReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrOverseerReporter.java
new file mode 100644
index 0000000..f3bb4d5
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrOverseerReporter.java
@@ -0,0 +1,186 @@
+package org.apache.solr.metrics.reporters.solr;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.MalformedURLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
+import org.apache.http.client.HttpClient;
+import org.apache.solr.cloud.Overseer;
+import org.apache.solr.cloud.ZkController;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrInfoMBean;
+import org.apache.solr.handler.admin.MetricsCollectorHandler;
+import org.apache.solr.metrics.SolrMetricManager;
+import org.apache.solr.metrics.SolrMetricReporter;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class SolrOverseerReporter extends SolrMetricReporter {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public static final String OVERSEER_GROUP = SolrMetricManager.overridableRegistryName(SolrInfoMBean.Group.overseer.toString());
+
+ public static final List<SolrReporter.Specification> DEFAULT_METRICS = new ArrayList<SolrReporter.Specification>() {{
+ add(new SolrReporter.Specification(OVERSEER_GROUP, "jetty",
+ SolrMetricManager.overridableRegistryName(SolrInfoMBean.Group.jetty.toString()),
+ Collections.singleton(".*\\.mean"))); // all metrics
+ add(new SolrReporter.Specification(OVERSEER_GROUP, "jvm",
+ SolrMetricManager.overridableRegistryName(SolrInfoMBean.Group.jvm.toString()),
+ Collections.emptySet())); // all metrics
+ add(new SolrReporter.Specification(OVERSEER_GROUP, "node", SolrMetricManager.overridableRegistryName(SolrInfoMBean.Group.node.toString()),
+ Collections.emptySet())); // all metrics
+ add(new SolrReporter.Specification(OVERSEER_GROUP, "leader.$1", "solr\\.core\\.(.*)\\.leader",
+ new HashSet<String>(){{
+ add("UPDATE\\./update/.*");
+ add("QUERY\\./select.*");
+ add("INDEX\\..*");
+ add("TLOG\\..*");
+ }}));
+ }};
+
+ private String handler = MetricsCollectorHandler.HANDLER_PATH;
+ private int period = 60;
+ private List<SolrReporter.Specification> metrics = DEFAULT_METRICS;
+
+ private SolrReporter reporter;
+
+ /**
+ * Create a reporter for metrics managed in a named registry.
+ *
+ * @param metricManager metric manager
+ * @param registryName unlike in other reporters, this is the node id
+ */
+ public SolrOverseerReporter(SolrMetricManager metricManager, String registryName) {
+ super(metricManager, registryName);
+ }
+
+ public void setHandler(String handler) {
+ this.handler = handler;
+ }
+
+ public void setPeriod(int period) {
+ this.period = period;
+ }
+
+ // for unit tests
+ public int getPeriod() {
+ return period;
+ }
+
+ @Override
+ protected void validate() throws IllegalStateException {
+ if (period < 1) {
+ log.info("Turning off node reporter, period=" + period);
+ }
+ // start in setCoreContainer(...)
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (reporter != null) {
+ reporter.close();;
+ }
+ }
+
+ public void setCoreContainer(CoreContainer cc) {
+ // start reporter only in cloud mode
+ if (!cc.isZooKeeperAware()) {
+ return;
+ }
+ if (period < 1) { // don't start it
+ return;
+ }
+ HttpClient httpClient = cc.getUpdateShardHandler().getHttpClient();
+ ZkController zk = cc.getZkController();
+ String reporterId = zk.getNodeName();
+ reporter = SolrReporter.Builder.forRegistries(metricManager, metrics)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .withHandler(handler)
+ .withReporterId(reporterId)
+ .cloudClient(false) // we want to send reports specifically to a selected leader instance
+ .skipAggregateValues(true) // we don't want to transport details of aggregates
+ .skipHistograms(true) // we don't want to transport histograms
+ .build(httpClient, new OverseerUrlSupplier(zk));
+
+ reporter.start(period, TimeUnit.SECONDS);
+
+ }
+
+ // TODO: fix this when there is an elegant way to retrieve URL of a node that runs Overseer leader.
+ private static class OverseerUrlSupplier implements Supplier<String> {
+ private static final long DEFAULT_INTERVAL = 30000; // 30s
+ private ZkController zk;
+ private String lastKnownUrl = null;
+ private long lastCheckTime = 0;
+ private long interval = DEFAULT_INTERVAL;
+
+ OverseerUrlSupplier(ZkController zk) {
+ this.zk = zk;
+ }
+
+ @Override
+ public String get() {
+ if (zk == null) {
+ return null;
+ }
+ // primitive caching for interval
+ long now = System.currentTimeMillis();
+ if (lastKnownUrl != null && (now - lastCheckTime) < interval) {
+ return lastKnownUrl;
+ }
+ if (!zk.isConnected()) {
+ return lastKnownUrl;
+ }
+ lastCheckTime = now;
+ SolrZkClient zkClient = zk.getZkClient();
+ ZkNodeProps props;
+ try {
+ props = ZkNodeProps.load(zkClient.getData(
+ Overseer.OVERSEER_ELECT + "/leader", null, null, true));
+ } catch (KeeperException e) {
+ log.warn("Could not obtain overseer's address, skipping.", e);
+ return lastKnownUrl;
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return lastKnownUrl;
+ }
+ if (props == null) {
+ return lastKnownUrl;
+ }
+ String oid = props.getStr("id");
+ if (oid == null) {
+ return lastKnownUrl;
+ }
+ String[] ids = oid.split("-");
+ if (ids.length != 3) { // unknown format
+ log.warn("Unknown format of leader id, skipping: " + oid);
+ return lastKnownUrl;
+ }
+ // convert nodeName back to URL
+ String url = zk.getZkStateReader().getBaseUrlForNodeName(ids[1]);
+ // check that it's parseable
+ try {
+ new java.net.URL(url);
+ } catch (MalformedURLException mue) {
+ log.warn("Malformed Overseer's leader URL: url", mue);
+ return lastKnownUrl;
+ }
+ lastKnownUrl = url;
+ return url;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cba0956f/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReplicaReporter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReplicaReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReplicaReporter.java
index c058591..5d2d584 100644
--- a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReplicaReporter.java
+++ b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReplicaReporter.java
@@ -74,6 +74,11 @@ public class SolrReplicaReporter extends SolrMetricReporter {
this.period = period;
}
+ // for unit tests
+ public int getPeriod() {
+ return period;
+ }
+
public void setMetrics(String prefixList) {
if (prefixList == null || prefixList.isEmpty()) {
return;
@@ -87,7 +92,7 @@ public class SolrReplicaReporter extends SolrMetricReporter {
@Override
protected void validate() throws IllegalStateException {
if (period < 1) {
- throw new IllegalStateException("Period must be greater than 0");
+ log.info("Turning off replica reporter, period=" + period);
}
// start in inform(...) only when core is available
}
@@ -95,7 +100,7 @@ public class SolrReplicaReporter extends SolrMetricReporter {
@Override
public void close() throws IOException {
if (reporter != null) {
- reporter.close();;
+ reporter.close();
}
}
@@ -108,6 +113,9 @@ public class SolrReplicaReporter extends SolrMetricReporter {
log.warn("Not initializing replica reporter for non-cloud core " + core.getName());
return;
}
+ if (period < 1) { // don't start it
+ return;
+ }
// our id is coreNodeName
String id = core.getCoreDescriptor().getCloudDescriptor().getCoreNodeName();
SolrReporter.Specification spec = new SolrReporter.Specification(groupId, null, registryName, Arrays.asList(metrics));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cba0956f/solr/core/src/test-files/solr/solr-solrreporter.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/solr-solrreporter.xml b/solr/core/src/test-files/solr/solr-solrreporter.xml
new file mode 100644
index 0000000..1351e38
--- /dev/null
+++ b/solr/core/src/test-files/solr/solr-solrreporter.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<solr>
+ <shardHandlerFactory name="shardHandlerFactory" class="HttpShardHandlerFactory">
+ <str name="urlScheme">${urlScheme:}</str>
+ <int name="socketTimeout">${socketTimeout:90000}</int>
+ <int name="connTimeout">${connTimeout:15000}</int>
+ </shardHandlerFactory>
+
+ <solrcloud>
+ <str name="host">127.0.0.1</str>
+ <int name="hostPort">${hostPort:8983}</int>
+ <str name="hostContext">${hostContext:solr}</str>
+ <int name="zkClientTimeout">${solr.zkclienttimeout:30000}</int>
+ <bool name="genericCoreNodeNames">${genericCoreNodeNames:true}</bool>
+ <int name="leaderVoteWait">${leaderVoteWait:10000}</int>
+ <int name="distribUpdateConnTimeout">${distribUpdateConnTimeout:45000}</int>
+ <int name="distribUpdateSoTimeout">${distribUpdateSoTimeout:340000}</int>
+ <int name="autoReplicaFailoverWaitAfterExpiration">${autoReplicaFailoverWaitAfterExpiration:10000}</int>
+ <int name="autoReplicaFailoverWorkLoopDelay">${autoReplicaFailoverWorkLoopDelay:10000}</int>
+ <int name="autoReplicaFailoverBadNodeExpiration">${autoReplicaFailoverBadNodeExpiration:60000}</int>
+ </solrcloud>
+
+ <metrics>
+ <reporter name="test" group="replica">
+ <int name="period">111</int>
+ </reporter>
+ <reporter name="test" group="overseer">
+ <int name="period">111</int>
+ </reporter>
+ </metrics>
+</solr>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cba0956f/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java b/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java
new file mode 100644
index 0000000..261d186
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.metrics.reporters.solr;
+
+import java.nio.file.Paths;
+import java.util.Map;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.metrics.SolrMetricManager;
+import org.apache.solr.metrics.SolrMetricReporter;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class SolrCloudReportersTest extends SolrCloudTestCase {
+
+ @BeforeClass
+ public static void configureDummyCluster() throws Exception {
+ configureCluster(0).configure();
+ }
+
+ @Before
+ public void closePreviousCluster() throws Exception {
+ shutdownCluster();
+ }
+
+ @Test
+ public void testExplicitConfiguration() throws Exception {
+ String solrXml = IOUtils.toString(SolrCloudReportersTest.class.getResourceAsStream("/solr/solr-solrreporter.xml"), "UTF-8");
+ configureCluster(2)
+ .withSolrXml(solrXml).configure();
+ cluster.uploadConfigSet(Paths.get(TEST_PATH().toString(), "configsets", "minimal", "conf"), "test");
+ System.out.println("ZK: " + cluster.getZkServer().getZkAddress());
+ CollectionAdminRequest.createCollection("test_collection", "test", 2, 2)
+ .setMaxShardsPerNode(4)
+ .process(cluster.getSolrClient());
+ waitForState("Expected test_collection with 2 shards and 2 replicas", "test_collection", clusterShape(2, 2));
+ cluster.getJettySolrRunners().forEach(jetty -> {
+ CoreContainer cc = jetty.getCoreContainer();
+ SolrMetricManager metricManager = cc.getMetricManager();
+ Map<String, SolrMetricReporter> reporters = metricManager.getReporters("solr.overseer");
+ assertEquals(reporters.toString(), 1, reporters.size());
+ SolrMetricReporter reporter = reporters.get("test");
+ assertNotNull(reporter);
+ assertTrue(reporter.toString(), reporter instanceof SolrOverseerReporter);
+ SolrOverseerReporter sor = (SolrOverseerReporter)reporter;
+ assertEquals(111, sor.getPeriod());
+ for (String registryName : metricManager.registryNames(".*\\.shard[0-9]\\.core.*")) {
+ reporters = metricManager.getReporters(registryName);
+ assertEquals(reporters.toString(), 1, reporters.size());
+ reporter = reporters.get("test");
+ assertNotNull(reporter);
+ assertTrue(reporter.toString(), reporter instanceof SolrReplicaReporter);
+ SolrReplicaReporter srr = (SolrReplicaReporter)reporter;
+ assertEquals(111, srr.getPeriod());
+ }
+ for (String registryName : metricManager.registryNames(".*\\.leader")) {
+ reporters = metricManager.getReporters(registryName);
+ // no reporters registered for leader registry
+ assertEquals(reporters.toString(), 0, reporters.size());
+ }
+ });
+ }
+
+ @Test
+ public void testDefaultPlugins() throws Exception {
+ String solrXml = IOUtils.toString(SolrCloudReportersTest.class.getResourceAsStream("/solr/solr.xml"), "UTF-8");
+ configureCluster(2)
+ .withSolrXml(solrXml).configure();
+ cluster.uploadConfigSet(Paths.get(TEST_PATH().toString(), "configsets", "minimal", "conf"), "test");
+ System.out.println("ZK: " + cluster.getZkServer().getZkAddress());
+ CollectionAdminRequest.createCollection("test_collection", "test", 2, 2)
+ .setMaxShardsPerNode(4)
+ .process(cluster.getSolrClient());
+ waitForState("Expected test_collection with 2 shards and 2 replicas", "test_collection", clusterShape(2, 2));
+ cluster.getJettySolrRunners().forEach(jetty -> {
+ CoreContainer cc = jetty.getCoreContainer();
+ SolrMetricManager metricManager = cc.getMetricManager();
+ Map<String, SolrMetricReporter> reporters = metricManager.getReporters("solr.overseer");
+ assertEquals(reporters.toString(), 1, reporters.size());
+ SolrMetricReporter reporter = reporters.get("overseerDefault");
+ assertNotNull(reporter);
+ assertTrue(reporter.toString(), reporter instanceof SolrOverseerReporter);
+ SolrOverseerReporter sor = (SolrOverseerReporter)reporter;
+ assertEquals(30, sor.getPeriod());
+ for (String registryName : metricManager.registryNames(".*\\.shard[0-9]\\.core.*")) {
+ reporters = metricManager.getReporters(registryName);
+ assertEquals(reporters.toString(), 1, reporters.size());
+ reporter = reporters.get("replicaDefault");
+ assertNotNull(reporter);
+ assertTrue(reporter.toString(), reporter instanceof SolrReplicaReporter);
+ SolrReplicaReporter srr = (SolrReplicaReporter)reporter;
+ assertEquals(30, srr.getPeriod());
+ }
+ for (String registryName : metricManager.registryNames(".*\\.leader")) {
+ reporters = metricManager.getReporters(registryName);
+ // no reporters registered for leader registry
+ assertEquals(reporters.toString(), 0, reporters.size());
+ }
+ });
+ }
+}