You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2017/03/03 20:38:23 UTC
lucene-solr:jira/solr-9858: SOLR-9858 Fix issues from review.
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-9858 d5bf3506d -> e05112ee4
SOLR-9858 Fix issues from review.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e05112ee
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e05112ee
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e05112ee
Branch: refs/heads/jira/solr-9858
Commit: e05112ee43b72840aa1d1b12dcdf6bdacaeaaf21
Parents: d5bf350
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Fri Mar 3 21:37:51 2017 +0100
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Fri Mar 3 21:37:51 2017 +0100
----------------------------------------------------------------------
.../org/apache/solr/core/CoreContainer.java | 6 +-
.../org/apache/solr/core/JmxMonitoredMap.java | 3 +
.../src/java/org/apache/solr/core/SolrCore.java | 2 +-
.../org/apache/solr/core/SolrInfoMBean.java | 2 +-
.../handler/admin/MetricsCollectorHandler.java | 5 +-
.../solr/metrics/SolrCoreMetricManager.java | 20 +-
.../apache/solr/metrics/SolrMetricManager.java | 96 +++++++---
.../reporters/solr/SolrOverseerReporter.java | 4 +-
.../metrics/reporters/solr/SolrReporter.java | 14 ++
.../reporters/solr/SolrShardReporter.java | 188 +++++++++++++++++++
.../src/test-files/solr/solr-solrreporter.xml | 2 +-
.../solr/metrics/SolrCoreMetricManagerTest.java | 32 ++--
.../solr/metrics/SolrMetricManagerTest.java | 30 +--
.../metrics/SolrMetricsIntegrationTest.java | 14 +-
.../metrics/reporters/SolrJmxReporterTest.java | 13 +-
.../reporters/solr/SolrCloudReportersTest.java | 75 +++++++-
.../reporters/solr/SolrShardReporterTest.java | 98 ++++++++++
17 files changed, 515 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e05112ee/solr/core/src/java/org/apache/solr/core/CoreContainer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 3ba82cd..558f5ee 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -523,9 +523,9 @@ public class CoreContainer {
if(pkiAuthenticationPlugin != null)
containerHandlers.put(PKIAuthenticationPlugin.PATH, pkiAuthenticationPlugin.getRequestHandler());
- metricManager.loadReporters(cfg.getMetricReporterPlugins(), loader, SolrInfoMBean.Group.node);
- metricManager.loadReporters(cfg.getMetricReporterPlugins(), loader, SolrInfoMBean.Group.jvm);
- metricManager.loadReporters(cfg.getMetricReporterPlugins(), loader, SolrInfoMBean.Group.jetty);
+ metricManager.loadReporters(cfg.getMetricReporterPlugins(), loader, null, SolrInfoMBean.Group.node);
+ metricManager.loadReporters(cfg.getMetricReporterPlugins(), loader, null, SolrInfoMBean.Group.jvm);
+ metricManager.loadReporters(cfg.getMetricReporterPlugins(), loader, null, SolrInfoMBean.Group.jetty);
coreConfigService = ConfigSetService.createConfigSetService(cfg, loader, zkSys.zkController);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e05112ee/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java b/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java
index d650ec9..10f0f58 100644
--- a/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java
+++ b/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java
@@ -20,6 +20,7 @@ import javax.management.Attribute;
import javax.management.AttributeList;
import javax.management.AttributeNotFoundException;
import javax.management.DynamicMBean;
+import javax.management.InstanceNotFoundException;
import javax.management.InvalidAttributeValueException;
import javax.management.MBeanAttributeInfo;
import javax.management.MBeanException;
@@ -165,6 +166,8 @@ public class JmxMonitoredMap<K, V> extends
for (ObjectName name : objectNames) {
try {
server.unregisterMBean(name);
+ } catch (InstanceNotFoundException ie) {
+ // ignore - someone else already deleted this one
} catch (Exception e) {
log.warn("Exception un-registering mbean {}", name, e);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e05112ee/solr/core/src/java/org/apache/solr/core/SolrCore.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index b871c53..051e7ca 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -860,6 +860,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
this.configSetProperties = configSetProperties;
// Initialize the metrics manager
this.coreMetricManager = initCoreMetricManager(config);
+ this.coreMetricManager.loadReporters();
if (updateHandler == null) {
directoryFactory = initDirectoryFactory();
@@ -1098,7 +1099,6 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
*/
private SolrCoreMetricManager initCoreMetricManager(SolrConfig config) {
SolrCoreMetricManager coreMetricManager = new SolrCoreMetricManager(this);
- coreMetricManager.loadReporters();
return coreMetricManager;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e05112ee/solr/core/src/java/org/apache/solr/core/SolrInfoMBean.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrInfoMBean.java b/solr/core/src/java/org/apache/solr/core/SolrInfoMBean.java
index e083621..6bcdb32 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrInfoMBean.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrInfoMBean.java
@@ -38,7 +38,7 @@ public interface SolrInfoMBean {
/**
* Top-level group of beans for a subsystem.
*/
- enum Group { jvm, jetty, node, core, replica, overseer }
+ enum Group { jvm, jetty, node, core, shard, overseer }
/**
* Simple common usage name, e.g. BasicQueryHandler,
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e05112ee/solr/core/src/java/org/apache/solr/handler/admin/MetricsCollectorHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsCollectorHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsCollectorHandler.java
index bf1e657..de39a61 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsCollectorHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsCollectorHandler.java
@@ -52,7 +52,7 @@ import org.slf4j.LoggerFactory;
/**
* Handler to collect and aggregate metric reports. Each report indicates the target registry where
- * metrics values should be collected and aggregated. Mtrics with the same names are
+ * metrics values should be collected and aggregated. Metrics with the same names are
* aggregated using {@link AggregateMetric} instances, which track the source of updates and
* their count, as well as providing simple statistics over collected values.
*
@@ -116,6 +116,9 @@ public class MetricsCollectorHandler extends RequestHandlerBase {
return;
}
//log.info("#### " + req.toString());
+ if (req.getContentStreams() == null) { // no content
+ return;
+ }
for (ContentStream cs : req.getContentStreams()) {
if (cs.getContentType() == null) {
log.warn("Missing content type - ignoring");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e05112ee/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java
index 0fed8c3..cdd086f 100644
--- a/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java
+++ b/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java
@@ -37,6 +37,7 @@ public class SolrCoreMetricManager implements Closeable {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final SolrCore core;
+ private final String tag;
private final SolrMetricManager metricManager;
private String registryName;
private String collectionName;
@@ -52,6 +53,7 @@ public class SolrCoreMetricManager implements Closeable {
*/
public SolrCoreMetricManager(SolrCore core) {
this.core = core;
+ this.tag = String.valueOf(core.hashCode());
this.metricManager = core.getCoreDescriptor().getCoreContainer().getMetricManager();
initCloudMode();
registryName = createRegistryName(cloudMode, collectionName, shardName, replicaName, core.getName());
@@ -75,9 +77,10 @@ public class SolrCoreMetricManager implements Closeable {
public void loadReporters() {
NodeConfig nodeConfig = core.getCoreDescriptor().getCoreContainer().getConfig();
PluginInfo[] pluginInfos = nodeConfig.getMetricReporterPlugins();
- metricManager.loadReporters(pluginInfos, core.getResourceLoader(), SolrInfoMBean.Group.core, registryName);
+ metricManager.loadReporters(pluginInfos, core.getResourceLoader(), tag,
+ SolrInfoMBean.Group.core, registryName);
if (cloudMode) {
- metricManager.loadReplicaReporters(pluginInfos, core, leaderRegistryName, registryName);
+ metricManager.loadShardReporters(pluginInfos, core);
}
}
@@ -96,9 +99,9 @@ public class SolrCoreMetricManager implements Closeable {
return;
}
// close old reporters
- metricManager.closeReporters(oldRegistryName);
+ metricManager.closeReporters(oldRegistryName, tag);
if (oldLeaderRegistryName != null) {
- metricManager.closeReporters(oldLeaderRegistryName);
+ metricManager.closeReporters(oldLeaderRegistryName, tag);
}
// load reporters again, using the new core name
loadReporters();
@@ -123,7 +126,7 @@ public class SolrCoreMetricManager implements Closeable {
*/
@Override
public void close() throws IOException {
- metricManager.closeReporters(getRegistryName());
+ metricManager.closeReporters(getRegistryName(), tag);
}
public SolrCore getCore() {
@@ -159,6 +162,13 @@ public class SolrCoreMetricManager implements Closeable {
return leaderRegistryName;
}
+ /**
+ * Return a tag specific to this instance.
+ */
+ public String getTag() {
+ return tag;
+ }
+
public static String createRegistryName(boolean cloud, String collectionName, String shardName, String replicaName, String coreName) {
if (collectionName == null) {
// single core
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e05112ee/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
index da1ee96..40155dd 100644
--- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
+++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
@@ -53,7 +53,7 @@ import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoMBean;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.metrics.reporters.solr.SolrOverseerReporter;
-import org.apache.solr.metrics.reporters.solr.SolrReplicaReporter;
+import org.apache.solr.metrics.reporters.solr.SolrShardReporter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -99,6 +99,8 @@ public class SolrMetricManager {
private final Lock reportersLock = new ReentrantLock();
private final Lock swapLock = new ReentrantLock();
+ public static final int DEFAULT_CLOUD_REPORTER_PERIOD = 60;
+
public SolrMetricManager() { }
/**
@@ -163,6 +165,13 @@ public class SolrMetricManager {
public void reset() {
matched.clear();
}
+
+ @Override
+ public String toString() {
+ return "PrefixFilter{" +
+ "prefixes=" + prefixes +
+ '}';
+ }
}
/**
@@ -228,6 +237,13 @@ public class SolrMetricManager {
public void reset() {
matched.clear();
}
+
+ @Override
+ public String toString() {
+ return "RegexFilter{" +
+ "compiledPatterns=" + compiledPatterns +
+ '}';
+ }
}
/**
@@ -329,7 +345,7 @@ public class SolrMetricManager {
*/
public void removeRegistry(String registry) {
// close any reporters for this registry first
- closeReporters(registry);
+ closeReporters(registry, null);
// make sure we use a name with prefix, with overrides
registry = overridableRegistryName(registry);
if (isSharedRegistry(registry)) {
@@ -610,10 +626,12 @@ public class SolrMetricManager {
* the list. If both attributes are present then only "group" attribute will be processed.
* @param pluginInfos plugin configurations
* @param loader resource loader
+ * @param tag optional tag for the reporters, to distinguish reporters logically created for different parent
+ * component instances.
* @param group selected group, not null
* @param registryNames optional child registry name elements
*/
- public void loadReporters(PluginInfo[] pluginInfos, SolrResourceLoader loader, SolrInfoMBean.Group group, String... registryNames) {
+ public void loadReporters(PluginInfo[] pluginInfos, SolrResourceLoader loader, String tag, SolrInfoMBean.Group group, String... registryNames) {
if (pluginInfos == null || pluginInfos.length == 0) {
return;
}
@@ -653,7 +671,7 @@ public class SolrMetricManager {
}
}
try {
- loadReporter(registryName, loader, info);
+ loadReporter(registryName, loader, info, tag);
} catch (Exception e) {
log.warn("Error loading metrics reporter, plugin info: " + info, e);
}
@@ -665,10 +683,12 @@ public class SolrMetricManager {
* @param registry reporter is associated with this registry
* @param loader loader to use when creating an instance of the reporter
* @param pluginInfo plugin configuration. Plugin "name" and "class" attributes are required.
+ * @param tag optional tag for the reporter, to distinguish reporters logically created for different parent
+ * component instances.
* @return instance of newly created and registered reporter
* @throws Exception if any argument is missing or invalid
*/
- public SolrMetricReporter loadReporter(String registry, SolrResourceLoader loader, PluginInfo pluginInfo) throws Exception {
+ public SolrMetricReporter loadReporter(String registry, SolrResourceLoader loader, PluginInfo pluginInfo, String tag) throws Exception {
if (registry == null || pluginInfo == null || pluginInfo.name == null || pluginInfo.className == null) {
throw new IllegalArgumentException("loadReporter called with missing arguments: " +
"registry=" + registry + ", loader=" + loader + ", pluginInfo=" + pluginInfo);
@@ -687,11 +707,11 @@ public class SolrMetricManager {
} catch (IllegalStateException e) {
throw new IllegalArgumentException("reporter init failed: " + pluginInfo, e);
}
- registerReporter(registry, pluginInfo.name, reporter);
+ registerReporter(registry, pluginInfo.name, tag, reporter);
return reporter;
}
- private void registerReporter(String registry, String name, SolrMetricReporter reporter) throws Exception {
+ private void registerReporter(String registry, String name, String tag, SolrMetricReporter reporter) throws Exception {
try {
if (!reportersLock.tryLock(10, TimeUnit.SECONDS)) {
throw new Exception("Could not obtain lock to modify reporters registry: " + registry);
@@ -705,6 +725,9 @@ public class SolrMetricManager {
perRegistry = new HashMap<>();
reporters.put(registry, perRegistry);
}
+ if (tag != null && !tag.isEmpty()) {
+ name = name + "@" + tag;
+ }
SolrMetricReporter oldReporter = perRegistry.get(name);
if (oldReporter != null) { // close it
log.info("Replacing existing reporter '" + name + "' in registry '" + registry + "': " + oldReporter.toString());
@@ -721,9 +744,11 @@ public class SolrMetricManager {
* Close and unregister a named {@link SolrMetricReporter} for a registry.
* @param registry registry name
* @param name reporter name
+ * @param tag optional tag for the reporter, to distinguish reporters logically created for different parent
+ * component instances.
* @return true if a named reporter existed and was closed.
*/
- public boolean closeReporter(String registry, String name) {
+ public boolean closeReporter(String registry, String name, String tag) {
// make sure we use a name with prefix, with overrides
registry = overridableRegistryName(registry);
try {
@@ -740,6 +765,9 @@ public class SolrMetricManager {
if (perRegistry == null) {
return false;
}
+ if (tag != null && !tag.isEmpty()) {
+ name = name + "@" + tag;
+ }
SolrMetricReporter reporter = perRegistry.remove(name);
if (reporter == null) {
return false;
@@ -761,6 +789,17 @@ public class SolrMetricManager {
* @return names of closed reporters
*/
public Set<String> closeReporters(String registry) {
+ return closeReporters(registry, null);
+ }
+
+ /**
+ * Close and unregister all {@link SolrMetricReporter}-s for a registry.
+ * @param registry registry name
+ * @param tag optional tag for the reporter, to distinguish reporters logically created for different parent
+ * component instances.
+ * @return names of closed reporters
+ */
+ public Set<String> closeReporters(String registry, String tag) {
// make sure we use a name with prefix, with overrides
registry = overridableRegistryName(registry);
try {
@@ -772,18 +811,28 @@ public class SolrMetricManager {
log.warn("Interrupted while trying to obtain lock to modify reporters registry: " + registry);
return Collections.emptySet();
}
- log.info("Closing metric reporters for: " + registry);
+ log.info("Closing metric reporters for registry=" + registry + ", tag=" + tag);
try {
- Map<String, SolrMetricReporter> perRegistry = reporters.remove(registry);
+ Map<String, SolrMetricReporter> perRegistry = reporters.get(registry);
if (perRegistry != null) {
- for (SolrMetricReporter reporter : perRegistry.values()) {
+ Set<String> names = new HashSet<>(perRegistry.keySet());
+ Set<String> removed = new HashSet<>();
+ names.forEach(name -> {
+ if (tag != null && !tag.isEmpty() && !name.endsWith("@" + tag)) {
+ return;
+ }
+ SolrMetricReporter reporter = perRegistry.remove(name);
try {
reporter.close();
} catch (IOException ioe) {
log.warn("Exception closing reporter " + reporter, ioe);
}
+ removed.add(name);
+ });
+ if (removed.size() == names.size()) {
+ reporters.remove(registry);
}
- return perRegistry.keySet();
+ return removed;
} else {
return Collections.emptySet();
}
@@ -881,27 +930,28 @@ public class SolrMetricManager {
return new PluginInfo(info.type, attrs, new NamedList(initArgs), null);
}
- public void loadReplicaReporters(PluginInfo[] pluginInfos, SolrCore core, String leaderRegistryName, String registryName) {
+ public void loadShardReporters(PluginInfo[] pluginInfos, SolrCore core) {
// don't load for non-cloud cores
- if (leaderRegistryName == null) {
+ if (core.getCoreDescriptor().getCloudDescriptor() == null) {
return;
}
// prepare default plugin if none present in the config
Map<String, String> attrs = new HashMap<>();
- attrs.put("name", "replicaDefault");
- attrs.put("group", "replica");
+ attrs.put("name", "shardDefault");
+ attrs.put("group", "shard");
Map<String, Object> initArgs = new HashMap<>();
- initArgs.put("groupId", leaderRegistryName);
- initArgs.put("period", 60);
+ initArgs.put("period", DEFAULT_CLOUD_REPORTER_PERIOD);
PluginInfo defaultPlugin = new PluginInfo("reporter", attrs, new NamedList(), null);
+ String registryName = core.getCoreMetricManager().getRegistryName();
// collect infos and normalize
- List<PluginInfo> infos = prepareCloudPlugins(pluginInfos, "replica", SolrReplicaReporter.class.getName(),
+ List<PluginInfo> infos = prepareCloudPlugins(pluginInfos, "shard", SolrShardReporter.class.getName(),
attrs, initArgs, defaultPlugin);
for (PluginInfo info : infos) {
try {
- SolrMetricReporter reporter = loadReporter(registryName, core.getResourceLoader(), info);
- ((SolrReplicaReporter)reporter).setCore(core);
+ SolrMetricReporter reporter = loadReporter(registryName, core.getResourceLoader(), info,
+ String.valueOf(core.hashCode()));
+ ((SolrShardReporter)reporter).setCore(core);
} catch (Exception e) {
log.warn("Could not load shard reporter, pluginInfo=" + info, e);
}
@@ -917,14 +967,14 @@ public class SolrMetricManager {
attrs.put("name", "overseerDefault");
attrs.put("group", "overseer");
Map<String, Object> initArgs = new HashMap<>();
- initArgs.put("period", 60);
+ initArgs.put("period", DEFAULT_CLOUD_REPORTER_PERIOD);
PluginInfo defaultPlugin = new PluginInfo("reporter", attrs, new NamedList(), null);
List<PluginInfo> infos = prepareCloudPlugins(pluginInfos, "overseer", SolrOverseerReporter.class.getName(),
attrs, initArgs, defaultPlugin);
String registryName = getRegistryName(SolrInfoMBean.Group.overseer);
for (PluginInfo info : infos) {
try {
- SolrMetricReporter reporter = loadReporter(registryName, cc.getResourceLoader(), info);
+ SolrMetricReporter reporter = loadReporter(registryName, cc.getResourceLoader(), info, null);
((SolrOverseerReporter)reporter).setCoreContainer(cc);
} catch (Exception e) {
log.warn("Could not load node reporter, pluginInfo=" + info, e);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e05112ee/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrOverseerReporter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrOverseerReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrOverseerReporter.java
index f05d6c1..ace5810 100644
--- a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrOverseerReporter.java
+++ b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrOverseerReporter.java
@@ -104,7 +104,7 @@ public class SolrOverseerReporter extends SolrMetricReporter {
}};
private String handler = MetricsCollectorHandler.HANDLER_PATH;
- private int period = 60;
+ private int period = SolrMetricManager.DEFAULT_CLOUD_REPORTER_PERIOD;
private List<SolrReporter.Report> reports = new ArrayList<>();
private SolrReporter reporter;
@@ -171,6 +171,7 @@ public class SolrOverseerReporter extends SolrMetricReporter {
}
// start reporter only in cloud mode
if (!cc.isZooKeeperAware()) {
+ log.warn("Not ZK-aware, not starting...");
return;
}
if (period < 1) { // don't start it
@@ -190,7 +191,6 @@ public class SolrOverseerReporter extends SolrMetricReporter {
.build(httpClient, new OverseerUrlSupplier(zk));
reporter.start(period, TimeUnit.SECONDS);
-
}
// TODO: fix this when there is an elegant way to retrieve URL of a node that runs Overseer leader.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e05112ee/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java
index d3e8759..9e3854e 100644
--- a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java
+++ b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java
@@ -272,6 +272,16 @@ public class SolrReporter extends ScheduledReporter {
this.registryPattern = Pattern.compile(report.registryPattern);
this.filter = new SolrMetricManager.RegexFilter(report.metricFilters);
}
+
+ @Override
+ public String toString() {
+ return "CompiledReport{" +
+ "group='" + group + '\'' +
+ ", label='" + label + '\'' +
+ ", registryPattern=" + registryPattern +
+ ", filter=" + filter +
+ '}';
+ }
}
public SolrReporter(HttpClient httpClient, Supplier<String> urlProvider, SolrMetricManager metricManager,
@@ -361,6 +371,10 @@ public class SolrReporter extends ScheduledReporter {
});
});
+ // if no docs added then don't send a report
+ if (req.getDocuments() == null || req.getDocuments().isEmpty()) {
+ return;
+ }
try {
//log.info("%%% sending to " + url + ": " + req.getParams());
solr.request(req);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e05112ee/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrShardReporter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrShardReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrShardReporter.java
new file mode 100644
index 0000000..2b20274
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrShardReporter.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.metrics.reporters.solr;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
+import org.apache.solr.cloud.CloudDescriptor;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.admin.MetricsCollectorHandler;
+import org.apache.solr.metrics.SolrMetricManager;
+import org.apache.solr.metrics.SolrMetricReporter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class reports selected metrics from replicas to shard leader.
+ * <p>The following configuration properties are supported:</p>
+ * <ul>
+ * <li>handler - (optional str) handler path where reports are sent. Default is
+ * {@link MetricsCollectorHandler#HANDLER_PATH}.</li>
+ * <li>period - (optional int) how often reports are sent, in seconds. Default is 60. Setting this
+ * to 0 disables the reporter.</li>
+ * <li>filter - (optional multiple str) regex expression(s) matching selected metrics to be reported.</li>
+ * </ul>
+ * NOTE: this reporter uses predefined "replica" group, and it's always created even if explicit configuration
+ * is missing. Default configuration uses filters defined in {@link #DEFAULT_FILTERS}.
+ * <p>Example configuration:</p>
+ * <pre>
+ * <reporter name="test" group="replica">
+ * <int name="period">11</int>
+ * <str name="filter">UPDATE\./update/.*requests</str>
+ * <str name="filter">QUERY\./select.*requests</str>
+ * </reporter>
+ * </pre>
+ */
+public class SolrShardReporter extends SolrMetricReporter {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public static final List<String> DEFAULT_FILTERS = new ArrayList(){{
+ add("TLOG.*");
+ add("REPLICATION.*");
+ add("INDEX.flush.*");
+ add("INDEX.merge.major.*");
+ add("UPDATE\\./update/.*requests");
+ add("QUERY\\./select.*requests");
+ }};
+
+ private String handler = MetricsCollectorHandler.HANDLER_PATH;
+ private int period = SolrMetricManager.DEFAULT_CLOUD_REPORTER_PERIOD;
+ private List<String> filters = new ArrayList<>();
+
+ private SolrReporter reporter;
+
+ /**
+ * Create a reporter for metrics managed in a named registry.
+ *
+ * @param metricManager metric manager
+ * @param registryName registry to use, one of registries managed by
+ * {@link SolrMetricManager}
+ */
+ public SolrShardReporter(SolrMetricManager metricManager, String registryName) {
+ super(metricManager, registryName);
+ }
+
+ public void setHandler(String handler) {
+ this.handler = handler;
+ }
+
+ public void setPeriod(int period) {
+ this.period = period;
+ }
+
+ public void setFilter(List<String> filterConfig) {
+ if (filterConfig == null || filterConfig.isEmpty()) {
+ return;
+ }
+ filters = filterConfig;
+ }
+
+ // for unit tests
+ int getPeriod() {
+ return period;
+ }
+
+ @Override
+ protected void validate() throws IllegalStateException {
+ if (period < 1) {
+ log.info("Turning off shard reporter, period=" + period);
+ }
+ if (filters.isEmpty()) {
+ filters = DEFAULT_FILTERS;
+ }
+ // start in inform(...) only when core is available
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (reporter != null) {
+ reporter.close();
+ }
+ }
+
+ public void setCore(SolrCore core) {
+ if (reporter != null) {
+ reporter.close();
+ }
+ if (core.getCoreDescriptor().getCloudDescriptor() == null) {
+ // not a cloud core
+ log.warn("Not initializing shard reporter for non-cloud core " + core.getName());
+ return;
+ }
+ if (period < 1) { // don't start it
+ log.warn("Not starting shard reporter ");
+ return;
+ }
+ // our id is coreNodeName
+ String id = core.getCoreDescriptor().getCloudDescriptor().getCoreNodeName();
+ // target registry is the leaderRegistryName
+ String groupId = core.getCoreMetricManager().getLeaderRegistryName();
+ if (groupId == null) {
+ log.warn("No leaderRegistryName for core " + core + ", not starting the reporter...");
+ return;
+ }
+ SolrReporter.Report spec = new SolrReporter.Report(groupId, null, registryName, filters);
+ reporter = SolrReporter.Builder.forReports(metricManager, Collections.singletonList(spec))
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .withHandler(handler)
+ .withReporterId(id)
+ .cloudClient(false) // we want to send reports specifically to a selected leader instance
+ .skipAggregateValues(true) // we don't want to transport details of aggregates
+ .skipHistograms(true) // we don't want to transport histograms
+ .build(core.getCoreDescriptor().getCoreContainer().getUpdateShardHandler().getHttpClient(), new LeaderUrlSupplier(core));
+
+ reporter.start(period, TimeUnit.SECONDS);
+ }
+
+ private static class LeaderUrlSupplier implements Supplier<String> {
+ private SolrCore core;
+
+ LeaderUrlSupplier(SolrCore core) {
+ this.core = core;
+ }
+
+ @Override
+ public String get() {
+ CloudDescriptor cd = core.getCoreDescriptor().getCloudDescriptor();
+ if (cd == null) {
+ return null;
+ }
+ ClusterState state = core.getCoreDescriptor().getCoreContainer().getZkController().getClusterState();
+ DocCollection collection = state.getCollection(core.getCoreDescriptor().getCollectionName());
+ Replica replica = collection.getLeader(core.getCoreDescriptor().getCloudDescriptor().getShardId());
+ if (replica == null) {
+ log.warn("No leader for " + collection.getName() + "/" + core.getCoreDescriptor().getCloudDescriptor().getShardId());
+ return null;
+ }
+ String baseUrl = replica.getStr("base_url");
+ if (baseUrl == null) {
+ log.warn("No base_url for replica " + replica);
+ }
+ return baseUrl;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e05112ee/solr/core/src/test-files/solr/solr-solrreporter.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/solr-solrreporter.xml b/solr/core/src/test-files/solr/solr-solrreporter.xml
index d341d45..d3074a4 100644
--- a/solr/core/src/test-files/solr/solr-solrreporter.xml
+++ b/solr/core/src/test-files/solr/solr-solrreporter.xml
@@ -38,7 +38,7 @@
</solrcloud>
<metrics>
- <reporter name="test" group="replica">
+ <reporter name="test" group="shard">
<int name="period">5</int>
<str name="filter">UPDATE\./update/.*requests</str>
<str name="filter">QUERY\./select.*requests</str>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e05112ee/solr/core/src/test/org/apache/solr/metrics/SolrCoreMetricManagerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/metrics/SolrCoreMetricManagerTest.java b/solr/core/src/test/org/apache/solr/metrics/SolrCoreMetricManagerTest.java
index c2f0c23..6e8e1e5 100644
--- a/solr/core/src/test/org/apache/solr/metrics/SolrCoreMetricManagerTest.java
+++ b/solr/core/src/test/org/apache/solr/metrics/SolrCoreMetricManagerTest.java
@@ -103,6 +103,7 @@ public class SolrCoreMetricManagerTest extends SolrTestCaseJ4 {
String className = MockMetricReporter.class.getName();
String reporterName = TestUtil.randomUnicodeString(random);
+ String taggedName = reporterName + "@" + coreMetricManager.getTag();
Map<String, Object> attrs = new HashMap<>();
attrs.put(FieldType.CLASS_NAME, className);
@@ -116,15 +117,16 @@ public class SolrCoreMetricManagerTest extends SolrTestCaseJ4 {
PluginInfo pluginInfo = shouldDefinePlugin ? new PluginInfo(TestUtil.randomUnicodeString(random), attrs) : null;
try {
- metricManager.loadReporter(coreMetricManager.getRegistryName(), coreMetricManager.getCore().getResourceLoader(), pluginInfo);
+ metricManager.loadReporter(coreMetricManager.getRegistryName(), coreMetricManager.getCore().getResourceLoader(),
+ pluginInfo, String.valueOf(coreMetricManager.getCore().hashCode()));
assertNotNull(pluginInfo);
Map<String, SolrMetricReporter> reporters = metricManager.getReporters(coreMetricManager.getRegistryName());
assertTrue("reporters.size should be > 0, but was + " + reporters.size(), reporters.size() > 0);
- assertNotNull("reporter " + reporterName + " not present among " + reporters, reporters.get(reporterName));
- assertTrue("wrong reporter class: " + reporters.get(reporterName), reporters.get(reporterName) instanceof MockMetricReporter);
+ assertNotNull("reporter " + reporterName + " not present among " + reporters, reporters.get(taggedName));
+ assertTrue("wrong reporter class: " + reporters.get(taggedName), reporters.get(taggedName) instanceof MockMetricReporter);
} catch (IllegalArgumentException e) {
assertTrue(pluginInfo == null || attrs.get("configurable") == null);
- assertNull(metricManager.getReporters(coreMetricManager.getRegistryName()).get(reporterName));
+ assertNull(metricManager.getReporters(coreMetricManager.getRegistryName()).get(taggedName));
}
}
@@ -152,21 +154,11 @@ public class SolrCoreMetricManagerTest extends SolrTestCaseJ4 {
}
@Test
- public void testRegistryName() throws Exception {
- String collectionName = "my_collection_";
- String cloudCoreName = "my_collection__shard1_0_replica0";
- String simpleCoreName = "collection_1_replica0";
- String simpleRegistryName = "solr.core." + simpleCoreName;
- String cloudRegistryName = "solr.core." + cloudCoreName;
- String nestedRegistryName = "solr.core.my_collection_.shard1_0.replica0";
- /*
- // pass through
- assertEquals(cloudRegistryName, coreMetricManager.createRegistryName(null, cloudCoreName));
- assertEquals(simpleRegistryName, coreMetricManager.createRegistryName(null, simpleCoreName));
- // unknown naming scheme -> pass through
- assertEquals(simpleRegistryName, coreMetricManager.createRegistryName(collectionName, simpleCoreName));
- // cloud collection
- assertEquals(nestedRegistryName, coreMetricManager.createRegistryName(collectionName, cloudCoreName));
-*/
+ public void testNonCloudRegistryName() throws Exception {
+ String registryName = h.getCore().getCoreMetricManager().getRegistryName();
+ String leaderRegistryName = h.getCore().getCoreMetricManager().getLeaderRegistryName();
+ assertNotNull(registryName);
+ assertEquals("solr.core.collection1", registryName);
+ assertNull(leaderRegistryName);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e05112ee/solr/core/src/test/org/apache/solr/metrics/SolrMetricManagerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/metrics/SolrMetricManagerTest.java b/solr/core/src/test/org/apache/solr/metrics/SolrMetricManagerTest.java
index 3813488..8cd03b1 100644
--- a/solr/core/src/test/org/apache/solr/metrics/SolrMetricManagerTest.java
+++ b/solr/core/src/test/org/apache/solr/metrics/SolrMetricManagerTest.java
@@ -205,32 +205,32 @@ public class SolrMetricManagerTest extends SolrTestCaseJ4 {
createPluginInfo("node_foo", "node", null),
createPluginInfo("core_foo", "core", null)
};
-
- metricManager.loadReporters(plugins, loader, SolrInfoMBean.Group.node);
+ String tag = "xyz";
+ metricManager.loadReporters(plugins, loader, tag, SolrInfoMBean.Group.node);
Map<String, SolrMetricReporter> reporters = metricManager.getReporters(
SolrMetricManager.getRegistryName(SolrInfoMBean.Group.node));
assertEquals(4, reporters.size());
- assertTrue(reporters.containsKey("universal_foo"));
- assertTrue(reporters.containsKey("multigroup_foo"));
- assertTrue(reporters.containsKey("node_foo"));
- assertTrue(reporters.containsKey("multiregistry_foo"));
+ assertTrue(reporters.containsKey("universal_foo@" + tag));
+ assertTrue(reporters.containsKey("multigroup_foo@" + tag));
+ assertTrue(reporters.containsKey("node_foo@" + tag));
+ assertTrue(reporters.containsKey("multiregistry_foo@" + tag));
- metricManager.loadReporters(plugins, loader, SolrInfoMBean.Group.core, "collection1");
+ metricManager.loadReporters(plugins, loader, tag, SolrInfoMBean.Group.core, "collection1");
reporters = metricManager.getReporters(
SolrMetricManager.getRegistryName(SolrInfoMBean.Group.core, "collection1"));
assertEquals(5, reporters.size());
- assertTrue(reporters.containsKey("universal_foo"));
- assertTrue(reporters.containsKey("multigroup_foo"));
- assertTrue(reporters.containsKey("specific_foo"));
- assertTrue(reporters.containsKey("core_foo"));
- assertTrue(reporters.containsKey("multiregistry_foo"));
+ assertTrue(reporters.containsKey("universal_foo@" + tag));
+ assertTrue(reporters.containsKey("multigroup_foo@" + tag));
+ assertTrue(reporters.containsKey("specific_foo@" + tag));
+ assertTrue(reporters.containsKey("core_foo@" + tag));
+ assertTrue(reporters.containsKey("multiregistry_foo@" + tag));
- metricManager.loadReporters(plugins, loader, SolrInfoMBean.Group.jvm);
+ metricManager.loadReporters(plugins, loader, tag, SolrInfoMBean.Group.jvm);
reporters = metricManager.getReporters(
SolrMetricManager.getRegistryName(SolrInfoMBean.Group.jvm));
assertEquals(2, reporters.size());
- assertTrue(reporters.containsKey("universal_foo"));
- assertTrue(reporters.containsKey("multigroup_foo"));
+ assertTrue(reporters.containsKey("universal_foo@" + tag));
+ assertTrue(reporters.containsKey("multigroup_foo@" + tag));
metricManager.removeRegistry("solr.jvm");
reporters = metricManager.getReporters(
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e05112ee/solr/core/src/test/org/apache/solr/metrics/SolrMetricsIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/metrics/SolrMetricsIntegrationTest.java b/solr/core/src/test/org/apache/solr/metrics/SolrMetricsIntegrationTest.java
index 27c038b..9aed62b 100644
--- a/solr/core/src/test/org/apache/solr/metrics/SolrMetricsIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/metrics/SolrMetricsIntegrationTest.java
@@ -55,6 +55,11 @@ public class SolrMetricsIntegrationTest extends SolrTestCaseJ4 {
private CoreContainer cc;
private SolrMetricManager metricManager;
+ private String tag;
+
+ private void assertTagged(Map<String, SolrMetricReporter> reporters, String name) {
+ assertTrue("Reporter '" + name + "' missing in " + reporters, reporters.containsKey(name + "@" + tag));
+ }
@Before
public void beforeTest() throws Exception {
@@ -68,10 +73,13 @@ public class SolrMetricsIntegrationTest extends SolrTestCaseJ4 {
new TestHarness.TestCoresLocator(DEFAULT_TEST_CORENAME, initCoreDataDir.getAbsolutePath(), "solrconfig.xml", "schema.xml"));
h.coreName = DEFAULT_TEST_CORENAME;
metricManager = cc.getMetricManager();
+ tag = h.getCore().getCoreMetricManager().getTag();
// initially there are more reporters, because two of them are added via a matching collection name
Map<String, SolrMetricReporter> reporters = metricManager.getReporters("solr.core." + DEFAULT_TEST_CORENAME);
assertEquals(INITIAL_REPORTERS.length, reporters.size());
- assertTrue(reporters.keySet().containsAll(Arrays.asList(INITIAL_REPORTERS)));
+ for (String r : INITIAL_REPORTERS) {
+ assertTagged(reporters, r);
+ }
// test rename operation
cc.rename(DEFAULT_TEST_CORENAME, CORE_NAME);
h.coreName = CORE_NAME;
@@ -101,7 +109,7 @@ public class SolrMetricsIntegrationTest extends SolrTestCaseJ4 {
deleteCore();
for (String reporterName : RENAMED_REPORTERS) {
- SolrMetricReporter reporter = reporters.get(reporterName);
+ SolrMetricReporter reporter = reporters.get(reporterName + "@" + tag);
MockMetricReporter mockReporter = (MockMetricReporter) reporter;
assertTrue("Reporter " + reporterName + " was not closed: " + mockReporter, mockReporter.didClose);
}
@@ -130,7 +138,7 @@ public class SolrMetricsIntegrationTest extends SolrTestCaseJ4 {
// SPECIFIC and MULTIREGISTRY were skipped because they were
// specific to collection1
for (String reporterName : RENAMED_REPORTERS) {
- SolrMetricReporter reporter = reporters.get(reporterName);
+ SolrMetricReporter reporter = reporters.get(reporterName + "@" + tag);
assertNotNull("Reporter " + reporterName + " was not found.", reporter);
assertTrue(reporter instanceof MockMetricReporter);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e05112ee/solr/core/src/test/org/apache/solr/metrics/reporters/SolrJmxReporterTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/metrics/reporters/SolrJmxReporterTest.java b/solr/core/src/test/org/apache/solr/metrics/reporters/SolrJmxReporterTest.java
index ea452b2..82b9d58 100644
--- a/solr/core/src/test/org/apache/solr/metrics/reporters/SolrJmxReporterTest.java
+++ b/solr/core/src/test/org/apache/solr/metrics/reporters/SolrJmxReporterTest.java
@@ -64,15 +64,17 @@ public class SolrJmxReporterTest extends SolrTestCaseJ4 {
coreMetricManager = core.getCoreMetricManager();
metricManager = core.getCoreDescriptor().getCoreContainer().getMetricManager();
PluginInfo pluginInfo = createReporterPluginInfo();
- metricManager.loadReporter(coreMetricManager.getRegistryName(), coreMetricManager.getCore().getResourceLoader(), pluginInfo);
+ metricManager.loadReporter(coreMetricManager.getRegistryName(), coreMetricManager.getCore().getResourceLoader(),
+ pluginInfo, coreMetricManager.getTag());
Map<String, SolrMetricReporter> reporters = metricManager.getReporters(coreMetricManager.getRegistryName());
assertTrue("reporters.size should be > 0, but was + " + reporters.size(), reporters.size() > 0);
reporterName = pluginInfo.name;
- assertNotNull("reporter " + reporterName + " not present among " + reporters, reporters.get(reporterName));
- assertTrue("wrong reporter class: " + reporters.get(reporterName), reporters.get(reporterName) instanceof SolrJmxReporter);
+ String taggedName = reporterName + "@" + coreMetricManager.getTag();
+ assertNotNull("reporter " + taggedName + " not present among " + reporters, reporters.get(taggedName));
+ assertTrue("wrong reporter class: " + reporters.get(taggedName), reporters.get(taggedName) instanceof SolrJmxReporter);
- reporter = (SolrJmxReporter) reporters.get(reporterName);
+ reporter = (SolrJmxReporter) reporters.get(taggedName);
mBeanServer = reporter.getMBeanServer();
assertNotNull("MBean server not found.", mBeanServer);
}
@@ -144,7 +146,8 @@ public class SolrJmxReporterTest extends SolrTestCaseJ4 {
h.getCoreContainer().reload(h.getCore().getName());
PluginInfo pluginInfo = createReporterPluginInfo();
- metricManager.loadReporter(coreMetricManager.getRegistryName(), coreMetricManager.getCore().getResourceLoader(), pluginInfo);
+ metricManager.loadReporter(coreMetricManager.getRegistryName(), coreMetricManager.getCore().getResourceLoader(),
+ pluginInfo, String.valueOf(coreMetricManager.getCore().hashCode()));
coreMetricManager.registerMetricProducer(scope, producer);
objects = mBeanServer.queryMBeans(null, null);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e05112ee/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java b/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java
index f7431df..bf5ffa1 100644
--- a/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java
+++ b/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java
@@ -24,6 +24,7 @@ import org.apache.commons.io.IOUtils;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
import org.apache.solr.metrics.AggregateMetric;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricReporter;
@@ -35,6 +36,9 @@ import org.junit.Test;
*
*/
public class SolrCloudReportersTest extends SolrCloudTestCase {
+ int leaderRegistries;
+ int overseerRegistries;
+
@BeforeClass
public static void configureDummyCluster() throws Exception {
@@ -44,6 +48,8 @@ public class SolrCloudReportersTest extends SolrCloudTestCase {
@Before
public void closePreviousCluster() throws Exception {
shutdownCluster();
+ leaderRegistries = 0;
+ overseerRegistries = 0;
}
@Test
@@ -57,7 +63,7 @@ public class SolrCloudReportersTest extends SolrCloudTestCase {
.setMaxShardsPerNode(4)
.process(cluster.getSolrClient());
waitForState("Expected test_collection with 2 shards and 2 replicas", "test_collection", clusterShape(2, 2));
- Thread.sleep(10000);
+ Thread.sleep(15000);
cluster.getJettySolrRunners().forEach(jetty -> {
CoreContainer cc = jetty.getCoreContainer();
SolrMetricManager metricManager = cc.getMetricManager();
@@ -71,13 +77,19 @@ public class SolrCloudReportersTest extends SolrCloudTestCase {
for (String registryName : metricManager.registryNames(".*\\.shard[0-9]\\.core.*")) {
reporters = metricManager.getReporters(registryName);
assertEquals(reporters.toString(), 1, reporters.size());
- reporter = reporters.get("test");
+ reporter = null;
+ for (String name : reporters.keySet()) {
+ if (name.startsWith("test")) {
+ reporter = reporters.get(name);
+ }
+ }
assertNotNull(reporter);
- assertTrue(reporter.toString(), reporter instanceof SolrReplicaReporter);
- SolrReplicaReporter srr = (SolrReplicaReporter)reporter;
+ assertTrue(reporter.toString(), reporter instanceof SolrShardReporter);
+ SolrShardReporter srr = (SolrShardReporter)reporter;
assertEquals(5, srr.getPeriod());
}
for (String registryName : metricManager.registryNames(".*\\.leader")) {
+ leaderRegistries++;
reporters = metricManager.getReporters(registryName);
// no reporters registered for leader registry
assertEquals(reporters.toString(), 0, reporters.size());
@@ -91,6 +103,7 @@ public class SolrCloudReportersTest extends SolrCloudTestCase {
assertTrue(key, metrics.get(key) instanceof AggregateMetric);
}
if (metricManager.registryNames().contains("solr.overseer")) {
+ overseerRegistries++;
Map<String,Metric> metrics = metricManager.registry("solr.overseer").getMetrics();
String key = "jvm.memory.heap.init.value";
assertTrue(key, metrics.containsKey(key));
@@ -100,6 +113,8 @@ public class SolrCloudReportersTest extends SolrCloudTestCase {
assertTrue(key, metrics.get(key) instanceof AggregateMetric);
}
});
+ assertEquals("leaderRegistries", 2, leaderRegistries);
+ assertEquals("overseerRegistries", 1, overseerRegistries);
}
@Test
@@ -113,8 +128,32 @@ public class SolrCloudReportersTest extends SolrCloudTestCase {
.setMaxShardsPerNode(4)
.process(cluster.getSolrClient());
waitForState("Expected test_collection with 2 shards and 2 replicas", "test_collection", clusterShape(2, 2));
+ // has to wait at least twice the SolrMetricManager.DEFAULT_CLOUD_REPORTER_PERIOD for the first
+ // report to aggregate into a *.leader registry, and for the second report to aggregate from *.leader
+ // into solr.overseer
+ Thread.sleep(SolrMetricManager.DEFAULT_CLOUD_REPORTER_PERIOD * 3 * 1000);
cluster.getJettySolrRunners().forEach(jetty -> {
CoreContainer cc = jetty.getCoreContainer();
+ // verify registry names
+ for (String name : cc.getCoreNames()) {
+ SolrCore core = cc.getCore(name);
+ try {
+ String registryName = core.getCoreMetricManager().getRegistryName();
+ String leaderRegistryName = core.getCoreMetricManager().getLeaderRegistryName();
+ String coreName = core.getName();
+ String collectionName = core.getCoreDescriptor().getCollectionName();
+ String coreNodeName = core.getCoreDescriptor().getCloudDescriptor().getCoreNodeName();
+ String shardId = core.getCoreDescriptor().getCloudDescriptor().getShardId();
+
+ assertEquals("solr.core." + collectionName + "." + shardId + "." + coreNodeName, registryName);
+ assertEquals("solr.core." + collectionName + "." + shardId + ".leader", leaderRegistryName);
+
+ } finally {
+ if (core != null) {
+ core.close();
+ }
+ }
+ }
SolrMetricManager metricManager = cc.getMetricManager();
Map<String, SolrMetricReporter> reporters = metricManager.getReporters("solr.overseer");
assertEquals(reporters.toString(), 1, reporters.size());
@@ -122,21 +161,39 @@ public class SolrCloudReportersTest extends SolrCloudTestCase {
assertNotNull(reporter);
assertTrue(reporter.toString(), reporter instanceof SolrOverseerReporter);
SolrOverseerReporter sor = (SolrOverseerReporter)reporter;
- assertEquals(60, sor.getPeriod());
+ assertEquals(SolrMetricManager.DEFAULT_CLOUD_REPORTER_PERIOD, sor.getPeriod());
for (String registryName : metricManager.registryNames(".*\\.shard[0-9]\\.core.*")) {
reporters = metricManager.getReporters(registryName);
assertEquals(reporters.toString(), 1, reporters.size());
- reporter = reporters.get("replicaDefault");
+ reporter = null;
+ for (String name : reporters.keySet()) {
+ if (name.startsWith("shardDefault")) {
+ reporter = reporters.get(name);
+ }
+ }
assertNotNull(reporter);
- assertTrue(reporter.toString(), reporter instanceof SolrReplicaReporter);
- SolrReplicaReporter srr = (SolrReplicaReporter)reporter;
- assertEquals(60, srr.getPeriod());
+ assertTrue(reporter.toString(), reporter instanceof SolrShardReporter);
+ SolrShardReporter srr = (SolrShardReporter)reporter;
+ assertEquals(SolrMetricManager.DEFAULT_CLOUD_REPORTER_PERIOD, srr.getPeriod());
}
for (String registryName : metricManager.registryNames(".*\\.leader")) {
+ leaderRegistries++;
reporters = metricManager.getReporters(registryName);
// no reporters registered for leader registry
assertEquals(reporters.toString(), 0, reporters.size());
}
+ if (metricManager.registryNames().contains("solr.overseer")) {
+ overseerRegistries++;
+ Map<String,Metric> metrics = metricManager.registry("solr.overseer").getMetrics();
+ String key = "jvm.memory.heap.init.value";
+ assertTrue(key, metrics.containsKey(key));
+ assertTrue(key, metrics.get(key) instanceof AggregateMetric);
+ key = "leader.test_collection.shard1.UPDATE./update/json.requests.count.max";
+ assertTrue(key, metrics.containsKey(key));
+ assertTrue(key, metrics.get(key) instanceof AggregateMetric);
+ }
});
+ assertEquals("leaderRegistries", 2, leaderRegistries);
+ assertEquals("overseerRegistries", 1, overseerRegistries);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e05112ee/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrShardReporterTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrShardReporterTest.java b/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrShardReporterTest.java
new file mode 100644
index 0000000..df1f9f2
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrShardReporterTest.java
@@ -0,0 +1,98 @@
+package org.apache.solr.metrics.reporters.solr;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Map;
+
+import com.codahale.metrics.Metric;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
+import org.apache.solr.cloud.CloudDescriptor;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.CoreDescriptor;
+import org.apache.solr.metrics.AggregateMetric;
+import org.apache.solr.metrics.SolrCoreMetricManager;
+import org.apache.solr.metrics.SolrMetricManager;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class SolrShardReporterTest extends AbstractFullDistribZkTestBase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public SolrShardReporterTest() {
+ schemaString = "schema15.xml"; // we need a string id
+ }
+
+ @Override
+ public String getSolrXml() {
+ return "solr-solrreporter.xml";
+ }
+
+ @Test
+ public void test() throws Exception {
+ waitForRecoveriesToFinish("control_collection",
+ jettys.get(0).getCoreContainer().getZkController().getZkStateReader(), false);
+ waitForRecoveriesToFinish("collection1",
+ jettys.get(0).getCoreContainer().getZkController().getZkStateReader(), false);
+ printLayout();
+ // wait for at least two reports
+ Thread.sleep(10000);
+ ClusterState state = jettys.get(0).getCoreContainer().getZkController().getClusterState();
+ for (JettySolrRunner jetty : jettys) {
+ CoreContainer cc = jetty.getCoreContainer();
+ SolrMetricManager metricManager = cc.getMetricManager();
+ for (final String coreName : cc.getCoreNames()) {
+ CoreDescriptor cd = cc.getCoreDescriptor(coreName);
+ if (cd.getCloudDescriptor() == null) { // not a cloud collection
+ continue;
+ }
+ CloudDescriptor cloudDesc = cd.getCloudDescriptor();
+ DocCollection docCollection = state.getCollection(cloudDesc.getCollectionName());
+ String registryName = SolrCoreMetricManager.createRegistryName(true,
+ cloudDesc.getCollectionName(), cloudDesc.getShardId(), cloudDesc.getCoreNodeName(), null);
+ String leaderRegistryName = SolrCoreMetricManager.createLeaderRegistryName(true,
+ cloudDesc.getCollectionName(), cloudDesc.getShardId());
+ boolean leader = cloudDesc.isLeader();
+ Slice slice = docCollection.getSlice(cloudDesc.getShardId());
+ int numReplicas = slice.getReplicas().size();
+ if (leader) {
+ assertTrue(metricManager.registryNames() + " doesn't contain " + leaderRegistryName,
+ metricManager.registryNames().contains(leaderRegistryName));
+ Map<String, Metric> metrics = metricManager.registry(leaderRegistryName).getMetrics();
+ metrics.forEach((k, v) -> {
+ assertTrue("Unexpected type of " + k + ": " + v.getClass().getName() + ", " + v,
+ v instanceof AggregateMetric);
+ AggregateMetric am = (AggregateMetric)v;
+ if (!k.startsWith("REPLICATION.peerSync")) {
+ assertEquals(coreName + "::" + registryName + "::" + k + ": " + am.toString(), numReplicas, am.size());
+ }
+ });
+ } else {
+ assertFalse(metricManager.registryNames() + " contains " + leaderRegistryName +
+ " but it's not a leader!",
+ metricManager.registryNames().contains(leaderRegistryName));
+ Map<String, Metric> metrics = metricManager.registry(leaderRegistryName).getMetrics();
+ metrics.forEach((k, v) -> {
+ assertTrue("Unexpected type of " + k + ": " + v.getClass().getName() + ", " + v,
+ v instanceof AggregateMetric);
+ AggregateMetric am = (AggregateMetric)v;
+ if (!k.startsWith("REPLICATION.peerSync")) {
+ assertEquals(coreName + "::" + registryName + "::" + k + ": " + am.toString(), 1, am.size());
+ }
+ });
+ }
+ assertTrue(metricManager.registryNames() + " doesn't contain " + registryName,
+ metricManager.registryNames().contains(registryName));
+ }
+ }
+ SolrMetricManager metricManager = controlJetty.getCoreContainer().getMetricManager();
+ assertTrue(metricManager.registryNames().contains("solr.overseer"));
+ Map<String, Metric> metrics = metricManager.registry("solr.overseer").getMetrics();
+ }
+}