You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2018/05/29 09:28:07 UTC
[3/3] lucene-solr:master: SOLR-11779: Basic long-term collection of
aggregated metrics.
SOLR-11779: Basic long-term collection of aggregated metrics.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6bbce38b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6bbce38b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6bbce38b
Branch: refs/heads/master
Commit: 6bbce38b77d5850f2d62d62fe87254e2ac8bd447
Parents: 44015e2
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Tue May 29 10:33:56 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Tue May 29 10:35:39 2018 +0200
----------------------------------------------------------------------
lucene/ivy-versions.properties | 2 +
solr/CHANGES.txt | 5 +
solr/core/ivy.xml | 2 +
.../api/collections/DeleteCollectionCmd.java | 15 +-
.../cloud/api/collections/DeleteReplicaCmd.java | 1 +
.../org/apache/solr/core/CoreContainer.java | 40 +-
.../org/apache/solr/core/MetricsConfig.java | 17 +-
.../org/apache/solr/core/SolrXmlConfig.java | 4 +
.../solr/handler/admin/CollectionsHandler.java | 5 +-
.../solr/handler/admin/CoreAdminOperation.java | 24 +
.../solr/handler/admin/MetricsHandler.java | 72 +-
.../handler/admin/MetricsHistoryHandler.java | 858 +++++++++++++++++++
.../apache/solr/metrics/SolrMetricManager.java | 4 +-
.../apache/solr/metrics/rrd/SolrRrdBackend.java | 118 +++
.../solr/metrics/rrd/SolrRrdBackendFactory.java | 418 +++++++++
.../apache/solr/metrics/rrd/package-info.java | 22 +
.../solr/security/PermissionNameProvider.java | 1 +
.../cloud/MetricsHistoryIntegrationTest.java | 193 +++++
.../cloud/autoscaling/sim/LiveNodesSet.java | 8 +-
.../cloud/autoscaling/sim/SimCloudManager.java | 221 +++--
.../sim/SimClusterStateProvider.java | 88 +-
.../autoscaling/sim/TestComputePlanAction.java | 6 +-
.../solr/handler/admin/MetricsHandlerTest.java | 8 +-
.../admin/MetricsHistoryHandlerTest.java | 126 +++
.../metrics/rrd/SolrRrdBackendFactoryTest.java | 182 ++++
.../solr/util/MockSearchableSolrClient.java | 117 +++
solr/licenses/rrd4j-3.2.jar.sha1 | 1 +
solr/licenses/rrd4j-LICENSE-ASL.txt | 202 +++++
solr/licenses/rrd4j-NOTICE.txt | 2 +
.../src/images/metrics-history/loadAvg-60s.png | Bin 0 -> 7101 bytes
.../src/images/metrics-history/memHeap-60s.png | Bin 0 -> 7559 bytes
.../metrics-history/query-graph-10min.png | Bin 0 -> 5678 bytes
.../images/metrics-history/query-graph-60s.png | Bin 0 -> 6916 bytes
.../images/metrics-history/update-graph-60s.png | Bin 0 -> 6473 bytes
solr/solr-ref-guide/src/metrics-history.adoc | 411 +++++++++
solr/solr-ref-guide/src/monitoring-solr.adoc | 4 +-
.../apache/solr/common/params/CommonParams.java | 2 +
.../solr/common/params/CoreAdminParams.java | 2 +
.../src/resources/apispec/metrics.history.json | 23 +
39 files changed, 3095 insertions(+), 109 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6bbce38b/lucene/ivy-versions.properties
----------------------------------------------------------------------
diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties
index 3579720..981bbe6 100644
--- a/lucene/ivy-versions.properties
+++ b/lucene/ivy-versions.properties
@@ -297,6 +297,8 @@ org.restlet.jee.version = 2.3.0
/org.restlet.jee/org.restlet = ${org.restlet.jee.version}
/org.restlet.jee/org.restlet.ext.servlet = ${org.restlet.jee.version}
+/org.rrd4j/rrd4j = 3.2
+
/org.simpleframework/simple-xml = 2.7.1
org.slf4j.version = 1.7.24
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6bbce38b/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index d56bf03..fcdcbaa 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -143,6 +143,11 @@ New Features
* SOLR-11453: Configuring slowQueryThresholdMillis logs slow requests to a separate file - solr_slow_requests.log .
(Shawn Heisey, Remko Popma, Varun Thacker)
+* SOLR-11779: Basic long-term collection of aggregated metrics. Historical data is
+ maintained as multi-resolution time series using round-robin databases in the '.system'
+ collection. New /admin/metrics/history API allows retrieval of this data in numeric
+ or graph formats. (ab)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6bbce38b/solr/core/ivy.xml
----------------------------------------------------------------------
diff --git a/solr/core/ivy.xml b/solr/core/ivy.xml
index e47d5b8..ee6fe80 100644
--- a/solr/core/ivy.xml
+++ b/solr/core/ivy.xml
@@ -155,6 +155,8 @@
<dependency org="org.codehaus.janino" name="commons-compiler" rev="${/org.codehaus.janino/commons-compiler}" conf="compile"/>
<dependency org="com.google.protobuf" name="protobuf-java" rev="${/com.google.protobuf/protobuf-java}" conf="compile"/>
+ <dependency org="org.rrd4j" name="rrd4j" rev="${/org.rrd4j/rrd4j}" conf="compile"/>
+
<exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
</dependencies>
</ivy-module>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6bbce38b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
index c8143ba..c676cf3 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
@@ -40,7 +40,10 @@ import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.TimeSource;
import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.core.snapshots.SolrSnapshotManager;
+import org.apache.solr.handler.admin.MetricsHistoryHandler;
+import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
@@ -67,6 +70,8 @@ public class DeleteCollectionCmd implements OverseerCollectionMessageHandler.Cmd
checkNotReferencedByAlias(zkStateReader, collection);
+ final boolean deleteHistory = message.getBool(CoreAdminParams.DELETE_METRICS_HISTORY, true);
+
boolean removeCounterNode = true;
try {
// Remove the snapshots meta-data for this collection in ZK. Deleting actual index files
@@ -82,10 +87,19 @@ public class DeleteCollectionCmd implements OverseerCollectionMessageHandler.Cmd
return;
}
}
+ // remove collection-level metrics history
+ if (deleteHistory) {
+ MetricsHistoryHandler historyHandler = ocmh.overseer.getCoreContainer().getMetricsHistoryHandler();
+ if (historyHandler != null) {
+ String registry = SolrMetricManager.getRegistryName(SolrInfoBean.Group.collection, collection);
+ historyHandler.removeHistory(registry);
+ }
+ }
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.UNLOAD.toString());
params.set(CoreAdminParams.DELETE_INSTANCE_DIR, true);
params.set(CoreAdminParams.DELETE_DATA_DIR, true);
+ params.set(CoreAdminParams.DELETE_METRICS_HISTORY, deleteHistory);
String asyncId = message.getStr(ASYNC);
Map<String, String> requestMap = null;
@@ -126,7 +140,6 @@ public class DeleteCollectionCmd implements OverseerCollectionMessageHandler.Cmd
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Could not fully remove collection: " + collection);
}
-
} finally {
try {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6bbce38b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java
index eefe903..4dbc059 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java
@@ -233,6 +233,7 @@ public class DeleteReplicaCmd implements Cmd {
params.set(CoreAdminParams.DELETE_INDEX, message.getBool(CoreAdminParams.DELETE_INDEX, true));
params.set(CoreAdminParams.DELETE_INSTANCE_DIR, message.getBool(CoreAdminParams.DELETE_INSTANCE_DIR, true));
params.set(CoreAdminParams.DELETE_DATA_DIR, message.getBool(CoreAdminParams.DELETE_DATA_DIR, true));
+ params.set(CoreAdminParams.DELETE_METRICS_HISTORY, message.getBool(CoreAdminParams.DELETE_METRICS_HISTORY, true));
boolean isLive = ocmh.zkStateReader.getClusterState().getLiveNodes().contains(replica.getNodeName());
if (isLive) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6bbce38b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 74b718c..cc54620 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -25,6 +25,7 @@ import static org.apache.solr.common.params.CommonParams.CONFIGSETS_HANDLER_PATH
import static org.apache.solr.common.params.CommonParams.CORES_HANDLER_PATH;
import static org.apache.solr.common.params.CommonParams.HEALTH_CHECK_HANDLER_PATH;
import static org.apache.solr.common.params.CommonParams.INFO_HANDLER_PATH;
+import static org.apache.solr.common.params.CommonParams.METRICS_HISTORY_PATH;
import static org.apache.solr.common.params.CommonParams.METRICS_PATH;
import static org.apache.solr.common.params.CommonParams.ZK_PATH;
import static org.apache.solr.security.AuthenticationPlugin.AUTHENTICATION_PLUGIN_PROP;
@@ -56,6 +57,7 @@ import org.apache.http.client.CredentialsProvider;
import org.apache.http.config.Lookup;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.store.Directory;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder;
import org.apache.solr.client.solrj.impl.SolrHttpClientContextBuilder;
@@ -87,6 +89,7 @@ import org.apache.solr.handler.admin.HealthCheckHandler;
import org.apache.solr.handler.admin.InfoHandler;
import org.apache.solr.handler.admin.MetricsCollectorHandler;
import org.apache.solr.handler.admin.MetricsHandler;
+import org.apache.solr.handler.admin.MetricsHistoryHandler;
import org.apache.solr.handler.admin.SecurityConfHandler;
import org.apache.solr.handler.admin.SecurityConfHandlerLocal;
import org.apache.solr.handler.admin.SecurityConfHandlerZk;
@@ -190,6 +193,8 @@ public class CoreContainer {
protected MetricsHandler metricsHandler;
+ protected MetricsHistoryHandler metricsHistoryHandler;
+
protected MetricsCollectorHandler metricsCollectorHandler;
protected AutoscalingHistoryHandler autoscalingHistoryHandler;
@@ -474,6 +479,14 @@ public class CoreContainer {
return metricManager;
}
+ public MetricsHandler getMetricsHandler() {
+ return metricsHandler;
+ }
+
+ public MetricsHistoryHandler getMetricsHistoryHandler() {
+ return metricsHistoryHandler;
+ }
+
//-------------------------------------------------------------------
// Initialization / Cleanup
//-------------------------------------------------------------------
@@ -536,7 +549,28 @@ public class CoreContainer {
infoHandler = createHandler(INFO_HANDLER_PATH, cfg.getInfoHandlerClass(), InfoHandler.class);
coreAdminHandler = createHandler(CORES_HANDLER_PATH, cfg.getCoreAdminHandlerClass(), CoreAdminHandler.class);
configSetsHandler = createHandler(CONFIGSETS_HANDLER_PATH, cfg.getConfigSetsHandlerClass(), ConfigSetsHandler.class);
- metricsHandler = createHandler(METRICS_PATH, MetricsHandler.class.getName(), MetricsHandler.class);
+
+ // metricsHistoryHandler uses metricsHandler, so create it first
+ metricsHandler = new MetricsHandler(metricManager);
+ containerHandlers.put(METRICS_PATH, metricsHandler);
+ metricsHandler.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), metricTag, METRICS_PATH);
+
+ if (isZooKeeperAware()) {
+ PluginInfo plugin = cfg.getMetricsConfig().getHistoryHandler();
+ Map<String, Object> initArgs;
+ if (plugin != null && plugin.initArgs != null) {
+ initArgs = plugin.initArgs.asMap(5);
+ initArgs.put(MetricsHistoryHandler.ENABLE_PROP, plugin.isEnabled());
+ } else {
+ initArgs = Collections.emptyMap();
+ }
+ metricsHistoryHandler = new MetricsHistoryHandler(getZkController().getNodeName(), metricsHandler,
+ new CloudSolrClient.Builder(Collections.singletonList(getZkController().getZkServerAddress()), Optional.empty())
+ .withHttpClient(updateShardHandler.getDefaultHttpClient()).build(), getZkController().getSolrCloudManager(), initArgs);
+ containerHandlers.put(METRICS_HISTORY_PATH, metricsHistoryHandler);
+ metricsHistoryHandler.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), metricTag, METRICS_HISTORY_PATH);
+ }
+
autoscalingHistoryHandler = createHandler(AUTOSCALING_HISTORY_PATH, AutoscalingHistoryHandler.class.getName(), AutoscalingHistoryHandler.class);
metricsCollectorHandler = createHandler(MetricsCollectorHandler.HANDLER_PATH, MetricsCollectorHandler.class.getName(), MetricsCollectorHandler.class);
// may want to add some configuration here in the future
@@ -761,6 +795,10 @@ public class CoreContainer {
} catch (Exception e) {
log.warn("Error removing live node. Continuing to close CoreContainer", e);
}
+ if (metricsHistoryHandler != null) {
+ IOUtils.closeQuietly(metricsHistoryHandler.getSolrClient());
+ metricsHistoryHandler.close();
+ }
if (metricManager != null) {
metricManager.closeReporters(SolrMetricManager.getRegistryName(SolrInfoBean.Group.cluster));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6bbce38b/solr/core/src/java/org/apache/solr/core/MetricsConfig.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/MetricsConfig.java b/solr/core/src/java/org/apache/solr/core/MetricsConfig.java
index 796483b..fab2553 100644
--- a/solr/core/src/java/org/apache/solr/core/MetricsConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/MetricsConfig.java
@@ -30,16 +30,19 @@ public class MetricsConfig {
private final PluginInfo meterSupplier;
private final PluginInfo timerSupplier;
private final PluginInfo histogramSupplier;
+ private final PluginInfo historyHandler;
private MetricsConfig(PluginInfo[] metricReporters, Set<String> hiddenSysProps,
PluginInfo counterSupplier, PluginInfo meterSupplier,
- PluginInfo timerSupplier, PluginInfo histogramSupplier) {
+ PluginInfo timerSupplier, PluginInfo histogramSupplier,
+ PluginInfo historyHandler) {
this.metricReporters = metricReporters;
this.hiddenSysProps = hiddenSysProps;
this.counterSupplier = counterSupplier;
this.meterSupplier = meterSupplier;
this.timerSupplier = timerSupplier;
this.histogramSupplier = histogramSupplier;
+ this.historyHandler = historyHandler;
}
public PluginInfo[] getMetricReporters() {
@@ -66,6 +69,10 @@ public class MetricsConfig {
return histogramSupplier;
}
+ public PluginInfo getHistoryHandler() {
+ return historyHandler;
+ }
+
public static class MetricsConfigBuilder {
private PluginInfo[] metricReporterPlugins = new PluginInfo[0];
private Set<String> hiddenSysProps = new HashSet<>();
@@ -73,6 +80,7 @@ public class MetricsConfig {
private PluginInfo meterSupplier;
private PluginInfo timerSupplier;
private PluginInfo histogramSupplier;
+ private PluginInfo historyHandler;
public MetricsConfigBuilder() {
@@ -111,9 +119,14 @@ public class MetricsConfig {
return this;
}
+ public MetricsConfigBuilder setHistoryHandler(PluginInfo info) {
+ this.historyHandler = info;
+ return this;
+ }
+
public MetricsConfig build() {
return new MetricsConfig(metricReporterPlugins, hiddenSysProps, counterSupplier, meterSupplier,
- timerSupplier, histogramSupplier);
+ timerSupplier, histogramSupplier, historyHandler);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6bbce38b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
index 9737c09..64fe731 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
@@ -483,6 +483,10 @@ public class SolrXmlConfig {
if (node != null) {
builder = builder.setHistogramSupplier(new PluginInfo(node, "histogramSupplier", false, false));
}
+ node = config.getNode("solr/metrics/history", false);
+ if (node != null) {
+ builder = builder.setHistoryHandler(new PluginInfo(node, "history", false, false));
+ }
PluginInfo[] reporterPlugins = getMetricReporterPluginInfos(config);
Set<String> hiddenSysProps = getHiddenSysProps(config);
return builder
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6bbce38b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index e724167..f7f6172 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -147,6 +147,7 @@ import static org.apache.solr.common.params.CoreAdminParams.DATA_DIR;
import static org.apache.solr.common.params.CoreAdminParams.DELETE_DATA_DIR;
import static org.apache.solr.common.params.CoreAdminParams.DELETE_INDEX;
import static org.apache.solr.common.params.CoreAdminParams.DELETE_INSTANCE_DIR;
+import static org.apache.solr.common.params.CoreAdminParams.DELETE_METRICS_HISTORY;
import static org.apache.solr.common.params.CoreAdminParams.INSTANCE_DIR;
import static org.apache.solr.common.params.CoreAdminParams.ULOG_DIR;
import static org.apache.solr.common.params.ShardParams._ROUTE_;
@@ -642,7 +643,8 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
copy(req.getParams(), map,
DELETE_INDEX,
DELETE_DATA_DIR,
- DELETE_INSTANCE_DIR);
+ DELETE_INSTANCE_DIR,
+ DELETE_METRICS_HISTORY);
return map;
}),
FORCELEADER_OP(FORCELEADER, (req, rsp, h) -> {
@@ -671,6 +673,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
DELETE_INDEX,
DELETE_DATA_DIR,
DELETE_INSTANCE_DIR,
+ DELETE_METRICS_HISTORY,
COUNT_PROP, REPLICA_PROP,
SHARD_ID_PROP,
ONLY_IF_DOWN);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6bbce38b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
index 075fb8d..8cf2bcb 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
@@ -25,6 +25,7 @@ import java.util.Optional;
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.store.Directory;
+import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
@@ -32,14 +33,17 @@ import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.core.snapshots.SolrSnapshotManager;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData;
import org.apache.solr.handler.admin.CoreAdminHandler.CoreAdminOp;
+import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.update.UpdateLog;
import org.apache.solr.util.NumberUtils;
@@ -98,7 +102,27 @@ enum CoreAdminOperation implements CoreAdminOp {
boolean deleteIndexDir = params.getBool(CoreAdminParams.DELETE_INDEX, false);
boolean deleteDataDir = params.getBool(CoreAdminParams.DELETE_DATA_DIR, false);
boolean deleteInstanceDir = params.getBool(CoreAdminParams.DELETE_INSTANCE_DIR, false);
+ boolean deleteMetricsHistory = params.getBool(CoreAdminParams.DELETE_METRICS_HISTORY, false);
+ CoreDescriptor cdescr = it.handler.coreContainer.getCoreDescriptor(cname);
it.handler.coreContainer.unload(cname, deleteIndexDir, deleteDataDir, deleteInstanceDir);
+ if (deleteMetricsHistory) {
+ MetricsHistoryHandler historyHandler = it.handler.coreContainer.getMetricsHistoryHandler();
+ if (historyHandler != null) {
+ CloudDescriptor cd = cdescr != null ? cdescr.getCloudDescriptor() : null;
+ String registry;
+ if (cd == null) {
+ registry = SolrMetricManager.getRegistryName(SolrInfoBean.Group.core, cname);
+ } else {
+ String replicaName = Utils.parseMetricsReplicaName(cd.getCollectionName(), cname);
+ registry = SolrMetricManager.getRegistryName(SolrInfoBean.Group.core,
+ cd.getCollectionName(),
+ cd.getShardId(),
+ replicaName);
+ }
+ historyHandler.checkSystemCollection();
+ historyHandler.removeHistory(registry);
+ }
+ }
assert TestInjection.injectNonExistentCoreExceptionAfterUnload(cname);
}),
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6bbce38b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
index ed1e474..ca291e8 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
@@ -24,6 +24,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.function.BiConsumer;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -36,10 +37,10 @@ import com.codahale.metrics.MetricFilter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.core.CoreContainer;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.request.SolrQueryRequest;
@@ -52,7 +53,6 @@ import org.apache.solr.util.stats.MetricUtils;
* Request handler to return metrics
*/
public class MetricsHandler extends RequestHandlerBase implements PermissionNameProvider {
- final CoreContainer container;
final SolrMetricManager metricManager;
public static final String COMPACT_PARAM = "compact";
@@ -69,13 +69,11 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
private static final Pattern KEY_REGEX = Pattern.compile("(?<!" + Pattern.quote("\\") + ")" + Pattern.quote(":"));
public MetricsHandler() {
- this.container = null;
this.metricManager = null;
}
- public MetricsHandler(CoreContainer container) {
- this.container = container;
- this.metricManager = this.container.getMetricManager();
+ public MetricsHandler(SolrMetricManager metricManager) {
+ this.metricManager = metricManager;
}
@Override
@@ -85,21 +83,25 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
- if (container == null) {
- throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "Core container instance not initialized");
+ if (metricManager == null) {
+ throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "SolrMetricManager instance not initialized");
}
- boolean compact = req.getParams().getBool(COMPACT_PARAM, true);
- String[] keys = req.getParams().getParams(KEY_PARAM);
+ handleRequest(req.getParams(), (k, v) -> rsp.add(k, v));
+ }
+
+ public void handleRequest(SolrParams params, BiConsumer<String, Object> consumer) throws Exception {
+ boolean compact = params.getBool(COMPACT_PARAM, true);
+ String[] keys = params.getParams(KEY_PARAM);
if (keys != null && keys.length > 0) {
- handleKeyRequest(keys, req, rsp);
+ handleKeyRequest(keys, consumer);
return;
}
- MetricFilter mustMatchFilter = parseMustMatchFilter(req);
- MetricUtils.PropertyFilter propertyFilter = parsePropertyFilter(req);
- List<MetricType> metricTypes = parseMetricTypes(req);
+ MetricFilter mustMatchFilter = parseMustMatchFilter(params);
+ MetricUtils.PropertyFilter propertyFilter = parsePropertyFilter(params);
+ List<MetricType> metricTypes = parseMetricTypes(params);
List<MetricFilter> metricFilters = metricTypes.stream().map(MetricType::asMetricFilter).collect(Collectors.toList());
- Set<String> requestedRegistries = parseRegistries(req);
+ Set<String> requestedRegistries = parseRegistries(params);
NamedList response = new SimpleOrderedMap();
for (String registryName : requestedRegistries) {
@@ -111,10 +113,10 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
response.add(registryName, result);
}
}
- rsp.getValues().add("metrics", response);
+ consumer.accept("metrics", response);
}
- private void handleKeyRequest(String[] keys, SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
+ public void handleKeyRequest(String[] keys, BiConsumer<String, Object> consumer) throws Exception {
SimpleOrderedMap result = new SimpleOrderedMap();
SimpleOrderedMap errors = new SimpleOrderedMap();
for (String key : keys) {
@@ -153,9 +155,9 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
}
});
}
- rsp.getValues().add("metrics", result);
+ consumer.accept("metrics", result);
if (errors.size() > 0) {
- rsp.getValues().add("errors", errors);
+ consumer.accept("errors", errors);
}
}
@@ -174,8 +176,8 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
return sb.toString();
}
- private MetricFilter parseMustMatchFilter(SolrQueryRequest req) {
- String[] prefixes = req.getParams().getParams(PREFIX_PARAM);
+ private MetricFilter parseMustMatchFilter(SolrParams params) {
+ String[] prefixes = params.getParams(PREFIX_PARAM);
MetricFilter prefixFilter = null;
if (prefixes != null && prefixes.length > 0) {
Set<String> prefixSet = new HashSet<>();
@@ -184,7 +186,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
}
prefixFilter = new SolrMetricManager.PrefixFilter(prefixSet);
}
- String[] regexes = req.getParams().getParams(REGEX_PARAM);
+ String[] regexes = params.getParams(REGEX_PARAM);
MetricFilter regexFilter = null;
if (regexes != null && regexes.length > 0) {
regexFilter = new SolrMetricManager.RegexFilter(regexes);
@@ -204,8 +206,8 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
return mustMatchFilter;
}
- private MetricUtils.PropertyFilter parsePropertyFilter(SolrQueryRequest req) {
- String[] props = req.getParams().getParams(PROPERTY_PARAM);
+ private MetricUtils.PropertyFilter parsePropertyFilter(SolrParams params) {
+ String[] props = params.getParams(PROPERTY_PARAM);
if (props == null || props.length == 0) {
return MetricUtils.PropertyFilter.ALL;
}
@@ -222,12 +224,16 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
}
}
- private Set<String> parseRegistries(SolrQueryRequest req) {
- String[] groupStr = req.getParams().getParams(GROUP_PARAM);
- String[] registryStr = req.getParams().getParams(REGISTRY_PARAM);
+ private Set<String> parseRegistries(SolrParams params) {
+ String[] groupStr = params.getParams(GROUP_PARAM);
+ String[] registryStr = params.getParams(REGISTRY_PARAM);
+ return parseRegistries(groupStr, registryStr);
+ }
+
+ public Set<String> parseRegistries(String[] groupStr, String[] registryStr) {
if ((groupStr == null || groupStr.length == 0) && (registryStr == null || registryStr.length == 0)) {
// return all registries
- return container.getMetricManager().registryNames();
+ return metricManager.registryNames();
}
boolean allRegistries = false;
Set<String> initialPrefixes = Collections.emptySet();
@@ -243,7 +249,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
initialPrefixes.add(SolrMetricManager.overridableRegistryName(s.trim()));
}
if (allRegistries) {
- return container.getMetricManager().registryNames();
+ return metricManager.registryNames();
}
}
}
@@ -262,12 +268,12 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
initialPrefixes.add(SolrMetricManager.overridableRegistryName(s.trim()));
}
if (allRegistries) {
- return container.getMetricManager().registryNames();
+ return metricManager.registryNames();
}
}
}
Set<String> validRegistries = new HashSet<>();
- for (String r : container.getMetricManager().registryNames()) {
+ for (String r : metricManager.registryNames()) {
for (String prefix : initialPrefixes) {
if (r.startsWith(prefix)) {
validRegistries.add(r);
@@ -278,8 +284,8 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
return validRegistries;
}
- private List<MetricType> parseMetricTypes(SolrQueryRequest req) {
- String[] typeStr = req.getParams().getParams(TYPE_PARAM);
+ private List<MetricType> parseMetricTypes(SolrParams params) {
+ String[] typeStr = params.getParams(TYPE_PARAM);
List<String> types = Collections.emptyList();
if (typeStr != null && typeStr.length > 0) {
types = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6bbce38b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
new file mode 100644
index 0000000..f6abcdb
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
@@ -0,0 +1,858 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.admin;
+
+import javax.imageio.ImageIO;
+import java.awt.Color;
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.DoubleAdder;
+import java.util.function.Function;
+import java.util.stream.Stream;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.solr.api.Api;
+import org.apache.solr.api.ApiBag;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.cloud.NodeStateProvider;
+import org.apache.solr.client.solrj.cloud.SolrCloudManager;
+import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
+import org.apache.solr.client.solrj.cloud.autoscaling.Suggestion;
+import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData;
+import org.apache.solr.cloud.Overseer;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.params.CollectionAdminParams;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.Base64;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.common.util.TimeSource;
+import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.metrics.SolrMetricManager;
+import org.apache.solr.metrics.rrd.SolrRrdBackendFactory;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.security.AuthorizationContext;
+import org.apache.solr.security.PermissionNameProvider;
+import org.apache.solr.util.DefaultSolrThreadFactory;
+import org.apache.zookeeper.KeeperException;
+import org.rrd4j.ConsolFun;
+import org.rrd4j.DsType;
+import org.rrd4j.core.ArcDef;
+import org.rrd4j.core.Archive;
+import org.rrd4j.core.Datasource;
+import org.rrd4j.core.DsDef;
+import org.rrd4j.core.FetchData;
+import org.rrd4j.core.FetchRequest;
+import org.rrd4j.core.RrdDb;
+import org.rrd4j.core.RrdDef;
+import org.rrd4j.core.Sample;
+import org.rrd4j.graph.RrdGraph;
+import org.rrd4j.graph.RrdGraphDef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static java.util.stream.Collectors.toMap;
+import static org.apache.solr.common.params.CommonParams.ID;
+
+/**
+ *
+ */
+public class MetricsHistoryHandler extends RequestHandlerBase implements PermissionNameProvider, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public static final List<String> DEFAULT_CORE_COUNTERS = new ArrayList<String>() {{
+ add("QUERY./select.requests");
+ add("UPDATE./update.requests");
+ }};
+ public static final List<String> DEFAULT_CORE_GAUGES = new ArrayList<String>() {{
+ add("INDEX.sizeInBytes");
+ }};
+ public static final List<String> DEFAULT_NODE_GAUGES = new ArrayList<String>() {{
+ add("CONTAINER.fs.coreRoot.usableSpace");
+ }};
+ public static final List<String> DEFAULT_JVM_GAUGES = new ArrayList<String>() {{
+ add("memory.heap.used");
+ add("os.processCpuLoad");
+ add("os.systemLoadAverage");
+ }};
+
+ public static final String NUM_SHARDS_KEY = "numShards";
+ public static final String NUM_REPLICAS_KEY = "numReplicas";
+ public static final String NUM_NODES_KEY = "numNodes";
+
+ public static final List<String> DEFAULT_COLLECTION_GAUGES = new ArrayList<String>() {{
+ add(NUM_SHARDS_KEY);
+ add(NUM_REPLICAS_KEY);
+ }};
+
+ public static final String COLLECT_PERIOD_PROP = "collectPeriod";
+ public static final String SYNC_PERIOD_PROP = "syncPeriod";
+ public static final String ENABLE_PROP = "enable";
+ public static final String ENABLE_REPLICAS_PROP = "enableReplicas";
+ public static final String ENABLE_NODES_PROP = "enableNodes";
+
+ public static final int DEFAULT_COLLECT_PERIOD = 60;
+ public static final String URI_PREFIX = "solr:";
+
+ private final SolrRrdBackendFactory factory;
+ private final String nodeName;
+ private final SolrClient solrClient;
+ private final MetricsHandler metricsHandler;
+ private final SolrCloudManager cloudManager;
+ private final TimeSource timeSource;
+ private final int collectPeriod;
+ private final Map<String, List<String>> counters = new HashMap<>();
+ private final Map<String, List<String>> gauges = new HashMap<>();
+
+ private final Map<String, RrdDb> knownDbs = new ConcurrentHashMap<>();
+
+ private ScheduledThreadPoolExecutor collectService;
+ private boolean logMissingCollection = true;
+ private boolean enable;
+ private boolean enableReplicas;
+ private boolean enableNodes;
+ private String versionString;
+
+ public MetricsHistoryHandler(String nodeName, MetricsHandler metricsHandler,
+ SolrClient solrClient, SolrCloudManager cloudManager, Map<String, Object> pluginArgs) {
+
+ Map<String, Object> args = new HashMap<>();
+ // init from optional solr.xml config
+ if (pluginArgs != null) {
+ args.putAll(pluginArgs);
+ }
+ // override from ZK
+ Map<String, Object> props = (Map<String, Object>)cloudManager.getClusterStateProvider()
+ .getClusterProperty("metrics", Collections.emptyMap())
+ .getOrDefault("history", Collections.emptyMap());
+ args.putAll(props);
+
+ this.nodeName = nodeName;
+ this.enable = Boolean.parseBoolean(String.valueOf(args.getOrDefault(ENABLE_PROP, "true")));
+ // default to false - don't collect local per-replica metrics
+ this.enableReplicas = Boolean.parseBoolean(String.valueOf(args.getOrDefault(ENABLE_REPLICAS_PROP, "false")));
+ this.enableNodes = Boolean.parseBoolean(String.valueOf(args.getOrDefault(ENABLE_NODES_PROP, "false")));
+ this.collectPeriod = Integer.parseInt(String.valueOf(args.getOrDefault(COLLECT_PERIOD_PROP, DEFAULT_COLLECT_PERIOD)));
+ int syncPeriod = Integer.parseInt(String.valueOf(args.getOrDefault(SYNC_PERIOD_PROP, SolrRrdBackendFactory.DEFAULT_SYNC_PERIOD)));
+
+ factory = new SolrRrdBackendFactory(solrClient, CollectionAdminParams.SYSTEM_COLL,
+ syncPeriod, cloudManager.getTimeSource());
+ this.solrClient = solrClient;
+ this.metricsHandler = metricsHandler;
+ this.cloudManager = cloudManager;
+ this.timeSource = cloudManager.getTimeSource();
+
+ counters.put(Group.core.toString(), DEFAULT_CORE_COUNTERS);
+ counters.put(Group.node.toString(), Collections.emptyList());
+ counters.put(Group.jvm.toString(), Collections.emptyList());
+ counters.put(Group.collection.toString(), Collections.emptyList());
+ gauges.put(Group.core.toString(), DEFAULT_CORE_GAUGES);
+ gauges.put(Group.node.toString(), DEFAULT_NODE_GAUGES);
+ gauges.put(Group.jvm.toString(), DEFAULT_JVM_GAUGES);
+ gauges.put(Group.collection.toString(), DEFAULT_COLLECTION_GAUGES);
+
+ versionString = this.getClass().getPackage().getImplementationVersion();
+ if (versionString == null) {
+ versionString = "?.?.?";
+ }
+ if (versionString.length() > 24) {
+ versionString = versionString.substring(0, 24) + "...";
+ }
+
+ if (enable) {
+ collectService = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(1,
+ new DefaultSolrThreadFactory("MetricsHistoryHandler"));
+ collectService.setRemoveOnCancelPolicy(true);
+ collectService.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
+ collectService.scheduleWithFixedDelay(() -> collectMetrics(),
+ timeSource.convertDelay(TimeUnit.SECONDS, collectPeriod, TimeUnit.MILLISECONDS),
+ timeSource.convertDelay(TimeUnit.SECONDS, collectPeriod, TimeUnit.MILLISECONDS),
+ TimeUnit.MILLISECONDS);
+ checkSystemCollection();
+ }
+ }
+
+ public void checkSystemCollection() {
+ // check that .system exists
+ try {
+ if (cloudManager.isClosed() || Thread.interrupted()) {
+ factory.setPersistent(false);
+ return;
+ }
+ ClusterState clusterState = cloudManager.getClusterStateProvider().getClusterState();
+ DocCollection systemColl = clusterState.getCollectionOrNull(CollectionAdminParams.SYSTEM_COLL);
+ if (systemColl == null) {
+ if (logMissingCollection) {
+ log.warn("Missing " + CollectionAdminParams.SYSTEM_COLL + ", keeping metrics history in memory");
+ logMissingCollection = false;
+ }
+ factory.setPersistent(false);
+ return;
+ } else {
+ boolean ready = false;
+ for (Replica r : systemColl.getReplicas()) {
+ if (r.isActive(clusterState.getLiveNodes())) {
+ ready = true;
+ break;
+ }
+ }
+ if (!ready) {
+ log.debug(CollectionAdminParams.SYSTEM_COLL + " not ready yet, keeping metrics history in memory");
+ factory.setPersistent(false);
+ return;
+ }
+ }
+ } catch (Exception e) {
+ log.warn("Error getting cluster state, keeping metrics history in memory", e);
+ factory.setPersistent(false);
+ return;
+ }
+ logMissingCollection = true;
+ factory.setPersistent(true);
+ }
+
+ public SolrClient getSolrClient() {
+ return solrClient;
+ }
+
+ public void removeHistory(String registry) throws IOException {
+ registry = SolrMetricManager.overridableRegistryName(registry);
+ knownDbs.remove(registry);
+ factory.remove(registry);
+ }
+
+ @VisibleForTesting
+ public SolrRrdBackendFactory getFactory() {
+ return factory;
+ }
+
+ private boolean isOverseerLeader() {
+ ZkNodeProps props = null;
+ try {
+ VersionedData data = cloudManager.getDistribStateManager().getData(
+ Overseer.OVERSEER_ELECT + "/leader");
+ if (data != null && data.getData() != null) {
+ props = ZkNodeProps.load(data.getData());
+ }
+ } catch (KeeperException | IOException | NoSuchElementException e) {
+ log.warn("Could not obtain overseer's address, skipping.", e);
+ return false;
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return false;
+ }
+ if (props == null) {
+ return false;
+ }
+ String oid = props.getStr(ID);
+ if (oid == null) {
+ return false;
+ }
+ String[] ids = oid.split("-");
+ if (ids.length != 3) { // unknown format
+ log.warn("Unknown format of leader id, skipping: " + oid);
+ return false;
+ }
+ return nodeName.equals(ids[1]);
+ }
+
+ private void collectMetrics() {
+ log.debug("-- collectMetrics");
+ try {
+ checkSystemCollection();
+ } catch (Exception e) {
+ log.warn("Error checking for .system collection, keeping metrics history in memory", e);
+ factory.setPersistent(false);
+ }
+ // get metrics
+ collectLocalReplicaMetrics();
+ collectGlobalMetrics();
+ }
+
+ private void collectLocalReplicaMetrics() {
+ List<Group> groups = new ArrayList<>();
+ if (enableNodes) {
+ groups.add(Group.jvm);
+ groups.add(Group.node);
+ }
+ if (enableReplicas) {
+ groups.add(Group.core);
+ }
+ for (Group group : groups) {
+ if (Thread.interrupted()) {
+ return;
+ }
+ log.debug("-- collecting local " + group + "...");
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add(MetricsHandler.GROUP_PARAM, group.toString());
+ params.add(MetricsHandler.COMPACT_PARAM, "true");
+ counters.get(group.toString()).forEach(c -> params.add(MetricsHandler.PREFIX_PARAM, c));
+ gauges.get(group.toString()).forEach(c -> params.add(MetricsHandler.PREFIX_PARAM, c));
+ AtomicReference<Object> result = new AtomicReference<>();
+ try {
+ metricsHandler.handleRequest(params, (k, v) -> {
+ if (k.equals("metrics")) {
+ result.set(v);
+ }
+ });
+ NamedList nl = (NamedList)result.get();
+ if (nl != null) {
+ for (Iterator<Map.Entry<String, Object>> it = nl.iterator(); it.hasNext(); ) {
+ Map.Entry<String, Object> entry = it.next();
+ String registry = entry.getKey();
+ if (group != Group.core) { // add nodeName suffix
+ registry = registry + "." + nodeName;
+ }
+
+ RrdDb db = getOrCreateDb(registry, group);
+ if (db == null) {
+ continue;
+ }
+ // set the timestamp
+ Sample s = db.createSample(TimeUnit.SECONDS.convert(timeSource.getEpochTimeNs(), TimeUnit.NANOSECONDS));
+ NamedList<Object> values = (NamedList<Object>)entry.getValue();
+ AtomicBoolean dirty = new AtomicBoolean(false);
+ counters.get(group.toString()).forEach(c -> {
+ Number val = (Number)values.get(c);
+ if (val != null) {
+ dirty.set(true);
+ s.setValue(c, val.doubleValue());
+ }
+ });
+ gauges.get(group.toString()).forEach(c -> {
+ Number val = (Number)values.get(c);
+ if (val != null) {
+ dirty.set(true);
+ s.setValue(c, val.doubleValue());
+ }
+ });
+ if (dirty.get()) {
+ s.update();
+ }
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ private void collectGlobalMetrics() {
+ if (!isOverseerLeader()) {
+ return;
+ }
+ Set<String> nodes = new HashSet<>(cloudManager.getClusterStateProvider().getLiveNodes());
+ NodeStateProvider nodeStateProvider = cloudManager.getNodeStateProvider();
+ Set<String> collTags = new HashSet<>();
+ collTags.addAll(counters.get(Group.core.toString()));
+ collTags.addAll(gauges.get(Group.core.toString()));
+
+ Set<String> nodeTags = new HashSet<>();
+ String nodePrefix = "metrics:" + SolrMetricManager.getRegistryName(Group.node) + ":";
+ counters.get(Group.node.toString()).forEach(name -> {
+ nodeTags.add(nodePrefix + name);
+ });
+ gauges.get(Group.node.toString()).forEach(name -> {
+ nodeTags.add(nodePrefix + name);
+ });
+ String jvmPrefix = "metrics:" + SolrMetricManager.getRegistryName(Group.jvm) + ":";
+ counters.get(Group.jvm.toString()).forEach(name -> {
+ nodeTags.add(jvmPrefix + name);
+ });
+ gauges.get(Group.jvm.toString()).forEach(name -> {
+ nodeTags.add(jvmPrefix + name);
+ });
+
+ // per-registry totals
+ // XXX at the moment the type of metrics that we collect allows
+ // adding all partial values. At some point it may be necessary to implement
+ // other aggregation functions.
+ // group : registry : name : value
+ Map<Group, Map<String, Map<String, Number>>> totals = new HashMap<>();
+
+ // collect and aggregate per-collection totals
+ for (String node : nodes) {
+ if (cloudManager.isClosed() || Thread.interrupted()) {
+ return;
+ }
+ // add core-level stats
+ Map<String, Map<String, List<ReplicaInfo>>> infos = nodeStateProvider.getReplicaInfo(node, collTags);
+ infos.forEach((coll, shards) -> {
+ shards.forEach((sh, replicas) -> {
+ String registry = SolrMetricManager.getRegistryName(Group.collection, coll);
+ Map<String, Number> perReg = totals
+ .computeIfAbsent(Group.collection, g -> new HashMap<>())
+ .computeIfAbsent(registry, r -> new HashMap<>());
+ replicas.forEach(ri -> {
+ collTags.forEach(tag -> {
+ double value = ((Number)ri.getVariable(tag, 0.0)).doubleValue();
+ // TODO: fix this when Suggestion.Condition.DISK_IDX uses proper conversion
+ if (tag.contains(Suggestion.coreidxsize)) {
+ value = value * 1024.0 * 1024.0 * 1024.0;
+ }
+ DoubleAdder adder = (DoubleAdder)perReg.computeIfAbsent(tag, t -> new DoubleAdder());
+ adder.add(value);
+ });
+ });
+ });
+ });
+ // add node-level stats
+ Map<String, Object> nodeValues = nodeStateProvider.getNodeValues(node, nodeTags);
+ for (Group g : Arrays.asList(Group.node, Group.jvm)) {
+ String registry = SolrMetricManager.getRegistryName(g);
+ Map<String, Number> perReg = totals
+ .computeIfAbsent(g, gr -> new HashMap<>())
+ .computeIfAbsent(registry, r -> new HashMap<>());
+ Set<String> names = new HashSet<>();
+ names.addAll(counters.get(g.toString()));
+ names.addAll(gauges.get(g.toString()));
+ names.forEach(name -> {
+ String tag = "metrics:" + registry + ":" + name;
+ double value = ((Number)nodeValues.getOrDefault(tag, 0.0)).doubleValue();
+ DoubleAdder adder = (DoubleAdder)perReg.computeIfAbsent(name, t -> new DoubleAdder());
+ adder.add(value);
+ });
+ }
+ }
+
+ // add numNodes
+ String nodeReg = SolrMetricManager.getRegistryName(Group.node);
+ Map<String, Number> perNodeReg = totals
+ .computeIfAbsent(Group.node, gr -> new HashMap<>())
+ .computeIfAbsent(nodeReg, r -> new HashMap<>());
+ perNodeReg.put(NUM_NODES_KEY, nodes.size());
+
+ // add some global collection-level stats
+ try {
+ ClusterState state = cloudManager.getClusterStateProvider().getClusterState();
+ state.forEachCollection(coll -> {
+ String registry = SolrMetricManager.getRegistryName(Group.collection, coll.getName());
+ Map<String, Number> perReg = totals
+ .computeIfAbsent(Group.collection, g -> new HashMap<>())
+ .computeIfAbsent(registry, r -> new HashMap<>());
+ Collection<Slice> slices = coll.getActiveSlices();
+ perReg.put(NUM_SHARDS_KEY, slices.size());
+ DoubleAdder numActiveReplicas = new DoubleAdder();
+ slices.forEach(s -> {
+ s.forEach(r -> {
+ if (r.isActive(state.getLiveNodes())) {
+ numActiveReplicas.add(1.0);
+ }
+ });
+ });
+ perReg.put(NUM_REPLICAS_KEY, numActiveReplicas);
+ });
+ } catch (IOException e) {
+ log.warn("Exception getting cluster state", e);
+ }
+
+ // now update the db-s
+ totals.forEach((group, perGroup) -> {
+ perGroup.forEach((reg, perReg) -> {
+ RrdDb db = getOrCreateDb(reg, group);
+ if (db == null) {
+ return;
+ }
+ try {
+ // set the timestamp
+ Sample s = db.createSample(TimeUnit.SECONDS.convert(timeSource.getEpochTimeNs(), TimeUnit.NANOSECONDS));
+ AtomicBoolean dirty = new AtomicBoolean(false);
+ List<Group> groups = new ArrayList<>();
+ groups.add(group);
+ if (group == Group.collection) {
+ groups.add(Group.core);
+ }
+ for (Group g : groups) {
+ counters.get(g.toString()).forEach(c -> {
+ Number val = perReg.get(c);
+ if (val != null) {
+ dirty.set(true);
+ s.setValue(c, val.doubleValue());
+ }
+ });
+ gauges.get(g.toString()).forEach(c -> {
+ Number val = perReg.get(c);
+ if (val != null) {
+ dirty.set(true);
+ s.setValue(c, val.doubleValue());
+ }
+ });
+ }
+ if (dirty.get()) {
+ s.update();
+ }
+ } catch (Exception e) {
+ }
+ });
+ });
+ }
+
+ private RrdDef createDef(String registry, Group group) {
+ registry = SolrMetricManager.overridableRegistryName(registry);
+
+ // base sampling period is collectPeriod - samples more frequent than
+ // that will be dropped, samples less frequent will be interpolated
+ RrdDef def = new RrdDef(URI_PREFIX + registry, collectPeriod);
+ // set the start time early enough so that the first sample is always later
+ // than the start of the archive
+ def.setStartTime(TimeUnit.SECONDS.convert(timeSource.getEpochTimeNs(), TimeUnit.NANOSECONDS) - def.getStep());
+
+ // add datasources
+ List<Group> groups = new ArrayList<>();
+ groups.add(group);
+ if (group == Group.collection) {
+ groups.add(Group.core);
+ }
+ for (Group g : groups) {
+ // use NaN when more than 1 sample is missing
+ counters.get(g.toString()).forEach(name ->
+ def.addDatasource(name, DsType.COUNTER, collectPeriod * 2, Double.NaN, Double.NaN));
+ gauges.get(g.toString()).forEach(name ->
+ def.addDatasource(name, DsType.GAUGE, collectPeriod * 2, Double.NaN, Double.NaN));
+ }
+ if (groups.contains(Group.node)) {
+ // add nomNodes gauge
+ def.addDatasource(NUM_NODES_KEY, DsType.GAUGE, collectPeriod * 2, Double.NaN, Double.NaN);
+ }
+
+ // add archives
+
+ // use AVERAGE consolidation,
+ // use NaN when >50% samples are missing
+ def.addArchive(ConsolFun.AVERAGE, 0.5, 1, 240); // 4 hours
+ def.addArchive(ConsolFun.AVERAGE, 0.5, 10, 288); // 48 hours
+ def.addArchive(ConsolFun.AVERAGE, 0.5, 60, 336); // 2 weeks
+ def.addArchive(ConsolFun.AVERAGE, 0.5, 240, 180); // 2 months
+ def.addArchive(ConsolFun.AVERAGE, 0.5, 1440, 365); // 1 year
+ return def;
+ }
+
+ private RrdDb getOrCreateDb(String registry, Group group) {
+ RrdDb db = knownDbs.computeIfAbsent(registry, r -> {
+ RrdDef def = createDef(r, group);
+ try {
+ RrdDb newDb = new RrdDb(def, factory);
+ return newDb;
+ } catch (IOException e) {
+ return null;
+ }
+ });
+ return db;
+ }
+
+ @Override
+ public void close() {
+ log.debug("Closing " + hashCode());
+ if (collectService != null) {
+ collectService.shutdownNow();
+ }
+ if (factory != null) {
+ factory.close();
+ }
+ knownDbs.clear();
+ }
+
+ public enum Cmd {
+ LIST, STATUS, GET, DELETE;
+
+ static final Map<String, Cmd> actions = Collections.unmodifiableMap(
+ Stream.of(Cmd.values())
+ .collect(toMap(Cmd::toLower, Function.identity())));
+
+ public static Cmd get(String p) {
+ return p == null ? null : actions.get(p.toLowerCase(Locale.ROOT));
+ }
+
+ public String toLower() {
+ return toString().toLowerCase(Locale.ROOT);
+ }
+ }
+
+ public enum Format {
+ LIST, STRING, GRAPH;
+
+ static final Map<String, Format> formats = Collections.unmodifiableMap(
+ Stream.of(Format.values())
+ .collect(toMap(Format::toLower, Function.identity())));
+
+ public static Format get(String p) {
+ return p == null ? null : formats.get(p.toLowerCase(Locale.ROOT));
+ }
+
+ public String toLower() {
+ return toString().toLowerCase(Locale.ROOT);
+ }
+ }
+
+
+ @Override
+ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
+ String actionStr = req.getParams().get(CommonParams.ACTION);
+ if (actionStr == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'action' is a required param");
+ }
+ Cmd cmd = Cmd.get(actionStr);
+ if (cmd == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "unknown 'action' param '" + actionStr + "', supported actions: " + Cmd.actions);
+ }
+ Object res = null;
+ switch (cmd) {
+ case LIST:
+ int rows = req.getParams().getInt(CommonParams.ROWS, SolrRrdBackendFactory.DEFAULT_MAX_DBS);
+ res = factory.list(rows);
+ break;
+ case GET:
+ String name = req.getParams().get(CommonParams.NAME);
+ if (name == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'name' is a required param");
+ }
+ String[] dsNames = req.getParams().getParams("ds");
+ String formatStr = req.getParams().get("format", Format.LIST.toString());
+ Format format = Format.get(formatStr);
+ if (format == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "unknown 'format' param '" + formatStr + "', supported formats: " + Format.formats);
+ }
+ if (!factory.exists(name)) {
+ rsp.add("error", "'" + name + "' doesn't exist");
+ } else {
+ // get a throwaway copy (safe to close and discard)
+ RrdDb db = new RrdDb(URI_PREFIX + name, true, factory);
+ res = new NamedList<>();
+ NamedList<Object> data = new NamedList<>();
+ data.add("data", getDbData(db, dsNames, format, req.getParams()));
+ ((NamedList)res).add(name, data);
+ db.close();
+ }
+ break;
+ case STATUS:
+ name = req.getParams().get(CommonParams.NAME);
+ if (name == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'name' is a required param");
+ }
+ if (!factory.exists(name)) {
+ rsp.add("error", "'" + name + "' doesn't exist");
+ } else {
+ // get a throwaway copy (safe to close and discard)
+ RrdDb db = new RrdDb(URI_PREFIX + name, true, factory);
+ NamedList<Object> map = new NamedList<>();
+ NamedList<Object> status = new NamedList<>();
+ status.add("status", getDbStatus(db));
+ map.add(name, status);
+ db.close();
+ res = map;
+ }
+ break;
+ case DELETE:
+ name = req.getParams().get(CommonParams.NAME);
+ if (name == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'name' is a required param");
+ }
+ if (name.equalsIgnoreCase("all") || name.equals("*")) {
+ factory.removeAll();
+ } else {
+ factory.remove(name);
+ }
+ rsp.add("success", "ok");
+ break;
+ }
+ if (res != null) {
+ rsp.add("metrics", res);
+ }
+ }
+
+ private NamedList<Object> getDbStatus(RrdDb db) throws IOException {
+ NamedList<Object> res = new SimpleOrderedMap<>();
+ res.add("lastModified", db.getLastUpdateTime());
+ RrdDef def = db.getRrdDef();
+ res.add("step", def.getStep());
+ res.add("datasourceCount", db.getDsCount());
+ res.add("archiveCount", db.getArcCount());
+ res.add("datasourceNames", Arrays.asList(db.getDsNames()));
+ List<Object> dss = new ArrayList<>(db.getDsCount());
+ res.add("datasources", dss);
+ for (DsDef dsDef : def.getDsDefs()) {
+ Map<String, Object> map = new LinkedHashMap<>();
+ map.put("datasource", dsDef.dump());
+ Datasource ds = db.getDatasource(dsDef.getDsName());
+ map.put("lastValue", ds.getLastValue());
+ dss.add(map);
+ }
+ List<Object> archives = new ArrayList<>(db.getArcCount());
+ res.add("archives", archives);
+ ArcDef[] arcDefs = def.getArcDefs();
+ for (int i = 0; i < db.getArcCount(); i++) {
+ Archive a = db.getArchive(i);
+ Map<String, Object> map = new LinkedHashMap<>();
+ map.put("archive", arcDefs[i].dump());
+ map.put("steps", a.getSteps());
+ map.put("consolFun", a.getConsolFun().name());
+ map.put("xff", a.getXff());
+ map.put("startTime", a.getStartTime());
+ map.put("endTime", a.getEndTime());
+ map.put("rows", a.getRows());
+ archives.add(map);
+ }
+
+ return res;
+ }
+
+ private NamedList<Object> getDbData(RrdDb db, String[] dsNames, Format format, SolrParams params) throws IOException {
+ NamedList<Object> res = new SimpleOrderedMap<>();
+ if (dsNames == null || dsNames.length == 0) {
+ dsNames = db.getDsNames();
+ }
+ StringBuilder str = new StringBuilder();
+ RrdDef def = db.getRrdDef();
+ ArcDef[] arcDefs = def.getArcDefs();
+ for (ArcDef arcDef : arcDefs) {
+ SimpleOrderedMap map = new SimpleOrderedMap();
+ res.add(arcDef.dump(), map);
+ Archive a = db.getArchive(arcDef.getConsolFun(), arcDef.getSteps());
+ // startTime / endTime, arcStep are in seconds
+ FetchRequest fr = db.createFetchRequest(arcDef.getConsolFun(),
+ a.getStartTime() - a.getArcStep(),
+ a.getEndTime() + a.getArcStep());
+ FetchData fd = fr.fetchData();
+ if (format != Format.GRAPH) {
+ // add timestamps separately from values
+ long[] timestamps = fd.getTimestamps();
+ str.setLength(0);
+ for (int i = 0; i < timestamps.length; i++) {
+ if (format == Format.LIST) {
+ map.add("timestamps", timestamps[i]);
+ } else {
+ if (i > 0) {
+ str.append('\n');
+ }
+ str.append(String.valueOf(timestamps[i]));
+ }
+ }
+ if (format == Format.STRING) {
+ map.add("timestamps", str.toString());
+ }
+ }
+ SimpleOrderedMap values = new SimpleOrderedMap();
+ map.add("values", values);
+ for (String name : dsNames) {
+ double[] vals = fd.getValues(name);
+ switch (format) {
+ case GRAPH:
+ RrdGraphDef graphDef = new RrdGraphDef();
+ graphDef.setTitle(name);
+ graphDef.datasource(name, fd);
+ graphDef.setStartTime(a.getStartTime() - a.getArcStep());
+ graphDef.setEndTime(a.getEndTime() + a.getArcStep());
+ graphDef.setPoolUsed(false);
+ graphDef.setAltAutoscale(true);
+ graphDef.setAltYGrid(true);
+ graphDef.setAltYMrtg(true);
+ graphDef.setSignature("Apache Solr " + versionString);
+ graphDef.setNoLegend(true);
+ graphDef.setAntiAliasing(true);
+ graphDef.setTextAntiAliasing(true);
+ graphDef.setWidth(500);
+ graphDef.setHeight(175);
+ graphDef.setTimeZone(TimeZone.getDefault());
+ graphDef.setLocale(Locale.getDefault());
+ // redraw immediately
+ graphDef.setLazy(false);
+ // area with a border
+ graphDef.area(name, new Color(0xffb860), null);
+ graphDef.line(name, Color.RED, null, 1.0f);
+ RrdGraph graph = new RrdGraph(graphDef);
+ BufferedImage bi = new BufferedImage(
+ graph.getRrdGraphInfo().getWidth(),
+ graph.getRrdGraphInfo().getHeight(),
+ BufferedImage.TYPE_INT_RGB);
+ graph.render(bi.getGraphics());
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ ImageIO.write(bi, "png", baos);
+ values.add(name, Base64.byteArrayToBase64(baos.toByteArray()));
+ break;
+ case STRING:
+ str.setLength(0);
+ for (int i = 0; i < vals.length; i++) {
+ if (i > 0) {
+ str.append('\n');
+ }
+ str.append(String.valueOf(vals[i]));
+ }
+ values.add(name, str.toString());
+ break;
+ case LIST:
+ for (int i = 0; i < vals.length; i++) {
+ values.add(name, vals[i]);
+ }
+ break;
+ }
+ }
+ }
+ return res;
+ }
+
+ @Override
+ public String getDescription() {
+ return "A handler for metrics history";
+ }
+
+ @Override
+ public Name getPermissionName(AuthorizationContext request) {
+ return Name.METRICS_HISTORY_READ_PERM;
+ }
+
+ @Override
+ public Boolean registerV2() {
+ return Boolean.TRUE;
+ }
+
+ @Override
+ public Collection<Api> getApis() {
+ return ApiBag.wrapRequestHandlers(this, "metrics.history");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6bbce38b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
index d5b8864..5fa3659 100644
--- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
+++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
@@ -423,14 +423,14 @@ public class SolrMetricManager {
} else {
swapLock.lock();
try {
- return getOrCreate(registries, registry);
+ return getOrCreateRegistry(registries, registry);
} finally {
swapLock.unlock();
}
}
}
- private static MetricRegistry getOrCreate(ConcurrentMap<String, MetricRegistry> map, String registry) {
+ private static MetricRegistry getOrCreateRegistry(ConcurrentMap<String, MetricRegistry> map, String registry) {
final MetricRegistry existing = map.get(registry);
if (existing == null) {
final MetricRegistry created = new MetricRegistry();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6bbce38b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java
new file mode 100644
index 0000000..956aabb
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.metrics.rrd;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.rrd4j.core.RrdByteArrayBackend;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class SolrRrdBackend extends RrdByteArrayBackend implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private final SolrRrdBackendFactory factory;
+ private final boolean readOnly;
+ private final ReentrantLock lock = new ReentrantLock();
+ private volatile boolean dirty = false;
+ private volatile boolean closed = false;
+
+ public SolrRrdBackend(String path, boolean readOnly, SolrRrdBackendFactory factory) {
+ super(path);
+ this.factory = factory;
+ try {
+ byte[] data = factory.getData(path);
+ if (data != null) {
+ this.buffer = data;
+ }
+ } catch (IOException e) {
+ log.warn("Exception retrieving data from " + path + ", store will be readOnly", e);
+ readOnly = true;
+ }
+ this.readOnly = readOnly;
+ }
+
+ /**
+ * Open an unregistered (throwaway) read-only clone of another backend.
+ * @param other other backend
+ */
+ public SolrRrdBackend(SolrRrdBackend other) {
+ super(other.getPath());
+ readOnly = true;
+ factory = null;
+ byte[] otherBuffer = other.buffer;
+ buffer = new byte[otherBuffer.length];
+ System.arraycopy(otherBuffer, 0, buffer, 0, otherBuffer.length);
+ }
+
+ public boolean isReadOnly() {
+ return readOnly;
+ }
+
+ @Override
+ protected void write(long offset, byte[] bytes) throws IOException {
+ if (readOnly || closed) {
+ return;
+ }
+ lock.lock();
+ try {
+ super.write(offset, bytes);
+ dirty = true;
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ public byte[] getSyncData() {
+ if (readOnly || closed) {
+ return null;
+ }
+ if (!dirty) {
+ return null;
+ }
+ // hold a lock to block writes so that we get consistent data
+ lock.lock();
+ try {
+ byte[] bufferCopy = new byte[buffer.length];
+ System.arraycopy(buffer, 0, bufferCopy, 0, buffer.length);
+ return bufferCopy;
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ public void markClean() {
+ dirty = false;
+ }
+
+ @Override
+ public void close() throws IOException {
+ super.close();
+ closed = true;
+ if (factory != null) {
+ // unregister myself from the factory
+ factory.unregisterBackend(getPath());
+ }
+ // close
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6bbce38b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java
new file mode 100644
index 0000000..06ab5fe
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java
@@ -0,0 +1,418 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.metrics.rrd;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrCloseable;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CollectionAdminParams;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.IOUtils;
+import org.apache.solr.common.util.TimeSource;
+import org.apache.solr.util.DefaultSolrThreadFactory;
+import org.rrd4j.core.RrdBackend;
+import org.rrd4j.core.RrdBackendFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * RRD backend factory using Solr documents as underlying storage.
+ * <p>RRD databases are identified by paths in the format <code>solr:dbName</code>.
+ * Typically the path will correspond to the name of metric or a group of metrics, eg:
+ * <code>solr:QUERY./select.requests</code></p>
+ * <p>NOTE: Solr doesn't register instances of this factory in the static
+ * registry {@link RrdBackendFactory#registerFactory(RrdBackendFactory)} because
+ * it's then impossible to manage its life-cycle.</p>
+ */
+public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrCloseable {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public static final int DEFAULT_SYNC_PERIOD = 60;
+ public static final int DEFAULT_MAX_DBS = 500;
+
+ public static final String NAME = "SOLR";
+ public static final String URI_PREFIX = "solr:";
+ public static final String ID_SEP = "|";
+ public static final String ID_PREFIX = "rrd";
+ public static final String DOC_TYPE = "metrics_rrd";
+
+ public static final String DATA_FIELD = "data_bin";
+
+ private final SolrClient solrClient;
+ private final TimeSource timeSource;
+ private final String collection;
+ private final int syncPeriod;
+ private final int idPrefixLength;
+ private ScheduledThreadPoolExecutor syncService;
+ private volatile boolean closed = false;
+ private volatile boolean persistent = true;
+
+ private final Map<String, SolrRrdBackend> backends = new ConcurrentHashMap<>();
+
+ /**
+ * Create a factory.
+ * @param solrClient SolrClient to use
+ * @param collection collection name where documents are stored (typically this is
+ * {@link CollectionAdminParams#SYSTEM_COLL})
+ * @param syncPeriod synchronization period in seconds - how often modified
+ * databases are stored as updated Solr documents
+ * @param timeSource time source
+ */
+ public SolrRrdBackendFactory(SolrClient solrClient, String collection, int syncPeriod, TimeSource timeSource) {
+ this.solrClient = solrClient;
+ this.timeSource = timeSource;
+ this.collection = collection;
+ this.syncPeriod = syncPeriod;
+ log.debug("Created " + hashCode());
+ this.idPrefixLength = ID_PREFIX.length() + ID_SEP.length();
+ syncService = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(2,
+ new DefaultSolrThreadFactory("SolrRrdBackendFactory"));
+ syncService.setRemoveOnCancelPolicy(true);
+ syncService.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
+ syncService.scheduleWithFixedDelay(() -> maybeSyncBackends(),
+ timeSource.convertDelay(TimeUnit.SECONDS, syncPeriod, TimeUnit.MILLISECONDS),
+ timeSource.convertDelay(TimeUnit.SECONDS, syncPeriod, TimeUnit.MILLISECONDS),
+ TimeUnit.MILLISECONDS);
+ }
+
+ private void ensureOpen() throws IOException {
+ if (closed) {
+ throw new IOException("Factory already closed");
+ }
+ }
+
+ @Override
+ public boolean canStore(URI uri) {
+ if (uri == null) {
+ return false;
+ }
+ if (uri.getScheme().toUpperCase(Locale.ROOT).equals(getName())) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public String getPath(URI uri) {
+ return uri.getSchemeSpecificPart();
+ }
+
+ @Override
+ public URI getUri(String path) {
+ if (!path.startsWith(URI_PREFIX)) {
+ path = URI_PREFIX + path;
+ }
+ try {
+ return new URI(path);
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException("Invalid path: " + path);
+ }
+ }
+
+ /**
+ * Open (or get) a backend.
+ * @param path backend path (without URI scheme)
+ * @param readOnly if true then the backend will never be synchronized to Solr,
+ * and updates will be silently ignored. Read-only backends can
+ * be safely closed and discarded after use.
+ * @return an instance of Solr backend.
+ * @throws IOException on Solr error when retrieving existing data
+ */
+ @Override
+ protected synchronized RrdBackend open(String path, boolean readOnly) throws IOException {
+ ensureOpen();
+ SolrRrdBackend backend = backends.computeIfAbsent(path, p -> new SolrRrdBackend(p, readOnly, this));
+ if (backend.isReadOnly()) {
+ if (readOnly) {
+ return backend;
+ } else {
+ // replace it with a writable one
+ backend = new SolrRrdBackend(path, readOnly, this);
+ backends.put(path, backend);
+ return backend;
+ }
+ } else {
+ if (readOnly) {
+ // return a throwaway unregistered read-only copy
+ return new SolrRrdBackend(backend);
+ } else {
+ return backend;
+ }
+ }
+ }
+
+ byte[] getData(String path) throws IOException {
+ if (!persistent) {
+ return null;
+ }
+ try {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add(CommonParams.Q, "{!term f=id}" + ID_PREFIX + ID_SEP + path);
+ params.add(CommonParams.FQ, CommonParams.TYPE + ":" + DOC_TYPE);
+ QueryResponse rsp = solrClient.query(collection, params);
+ SolrDocumentList docs = rsp.getResults();
+ if (docs == null || docs.isEmpty()) {
+ return null;
+ }
+ if (docs.size() > 1) {
+ throw new SolrServerException("Expected at most 1 doc with id '" + path + "' but got " + docs);
+ }
+ SolrDocument doc = docs.get(0);
+ Object o = doc.getFieldValue(DATA_FIELD);
+ if (o == null) {
+ return null;
+ }
+ if (o instanceof byte[]) {
+ return (byte[])o;
+ } else {
+ throw new SolrServerException("Unexpected value of '" + DATA_FIELD + "' field: " + o.getClass().getName() + ": " + o);
+ }
+ } catch (SolrServerException e) {
+ throw new IOException(e);
+ }
+ }
+
+ void unregisterBackend(String path) {
+ backends.remove(path);
+ }
+
+ /**
+ * List all available databases created by this node name
+ * @param maxLength maximum number of results to return
+ * @return list of database names, or empty
+ * @throws IOException on server errors
+ */
+ public List<String> list(int maxLength) throws IOException {
+ Set<String> names = new HashSet<>();
+ if (persistent) {
+ try {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add(CommonParams.Q, "*:*");
+ params.add(CommonParams.FQ, CommonParams.TYPE + ":" + DOC_TYPE);
+ params.add(CommonParams.FL, "id");
+ params.add(CommonParams.ROWS, String.valueOf(maxLength));
+ QueryResponse rsp = solrClient.query(collection, params);
+ SolrDocumentList docs = rsp.getResults();
+ if (docs != null) {
+ docs.forEach(d -> names.add(((String)d.getFieldValue("id")).substring(idPrefixLength)));
+ }
+ } catch (SolrServerException e) {
+ log.warn("Error retrieving RRD list", e);
+ }
+ }
+ // add in-memory backends not yet stored
+ names.addAll(backends.keySet());
+ ArrayList<String> list = new ArrayList<>(names);
+ Collections.sort(list);
+ return list;
+ }
+
+ /**
+ * Remove all databases created by this node name.
+ * @throws IOException on server error
+ */
+ public void removeAll() throws IOException {
+ for (Iterator<SolrRrdBackend> it = backends.values().iterator(); it.hasNext(); ) {
+ SolrRrdBackend backend = it.next();
+ it.remove();
+ IOUtils.closeQuietly(backend);
+ }
+ if (!persistent) {
+ return;
+ }
+ // remove all Solr docs
+ try {
+ solrClient.deleteByQuery(collection,
+ "{!term f=" + CommonParams.TYPE + "}:" + DOC_TYPE, syncPeriod * 1000);
+ } catch (SolrServerException e) {
+ log.warn("Error deleting RRDs", e);
+ }
+ }
+
+ /**
+ * Remove a database.
+ * @param path database path.
+ * @throws IOException on Solr exception
+ */
+ public void remove(String path) throws IOException {
+ SolrRrdBackend backend = backends.remove(path);
+ if (backend != null) {
+ IOUtils.closeQuietly(backend);
+ }
+ if (!persistent) {
+ return;
+ }
+ // remove Solr doc
+ try {
+ solrClient.deleteByQuery(collection, "{!term f=id}" + ID_PREFIX + ID_SEP + path);
+ } catch (SolrServerException | SolrException e) {
+ log.warn("Error deleting RRD for path " + path, e);
+ }
+ }
+
+ synchronized void maybeSyncBackends() {
+ if (closed) {
+ return;
+ }
+ if (!persistent) {
+ return;
+ }
+ if (Thread.interrupted()) {
+ return;
+ }
+ log.debug("-- maybe sync backends: " + backends.keySet());
+ Map<String, byte[]> syncData = new HashMap<>();
+ backends.forEach((path, backend) -> {
+ byte[] data = backend.getSyncData();
+ if (data != null) {
+ syncData.put(backend.getPath(), data);
+ }
+ });
+ if (syncData.isEmpty()) {
+ return;
+ }
+ log.debug("-- syncing " + syncData.keySet());
+ // write updates
+ try {
+ syncData.forEach((path, data) -> {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.setField("id", ID_PREFIX + ID_SEP + path);
+ doc.addField(CommonParams.TYPE, DOC_TYPE);
+ doc.addField(DATA_FIELD, data);
+ doc.setField("timestamp", new Date(TimeUnit.MILLISECONDS.convert(timeSource.getEpochTimeNs(), TimeUnit.NANOSECONDS)));
+ try {
+ solrClient.add(collection, doc);
+ } catch (SolrServerException | IOException e) {
+ log.warn("Error updating RRD data for " + path, e);
+ }
+ });
+ if (Thread.interrupted()) {
+ return;
+ }
+ try {
+ solrClient.commit(collection);
+ } catch (SolrServerException e) {
+ log.warn("Error committing RRD data updates", e);
+ }
+ syncData.forEach((path, data) -> {
+ SolrRrdBackend backend = backends.get(path);
+ if (backend != null) {
+ backend.markClean();
+ }
+ });
+ } catch (IOException e) {
+ log.warn("Error sending RRD data updates", e);
+ }
+ }
+
+ /**
+ * Check for existence of a backend.
+ * @param path backend path, without the URI scheme
+ * @return true when a backend exists. Note that a backend may exist only
+ * in memory if it was created recently within {@link #syncPeriod}.
+ * @throws IOException on Solr exception
+ */
+ @Override
+ public boolean exists(String path) throws IOException {
+ // check in-memory backends first
+ if (backends.containsKey(path)) {
+ return true;
+ }
+ if (!persistent) {
+ return false;
+ }
+ try {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add(CommonParams.Q, "{!term f=id}" + ID_PREFIX + ID_SEP + path);
+ params.add(CommonParams.FQ, CommonParams.TYPE + ":" + DOC_TYPE);
+ params.add(CommonParams.FL, "id");
+ QueryResponse rsp = solrClient.query(collection, params);
+ SolrDocumentList docs = rsp.getResults();
+ if (docs == null || docs.isEmpty()) {
+ return false;
+ }
+ if (docs.size() > 1) {
+ throw new SolrServerException("Expected at most 1 doc with id '" + path + "' but got " + docs);
+ }
+ return true;
+ } catch (SolrServerException e) {
+ throw new IOException(e);
+ }
+ }
+
+ public boolean isPersistent() {
+ return persistent;
+ }
+
+ public void setPersistent(boolean persistent) {
+ this.persistent = persistent;
+ }
+
+ @Override
+ protected boolean shouldValidateHeader(String path) throws IOException {
+ return false;
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public boolean isClosed() {
+ return closed;
+ }
+
+ @Override
+ public void close() {
+ if (closed) {
+ return;
+ }
+ log.debug("Closing " + hashCode());
+ closed = true;
+ backends.forEach((p, b) -> IOUtils.closeQuietly(b));
+ backends.clear();
+ syncService.shutdownNow();
+ syncService = null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6bbce38b/solr/core/src/java/org/apache/solr/metrics/rrd/package-info.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/rrd/package-info.java b/solr/core/src/java/org/apache/solr/metrics/rrd/package-info.java
new file mode 100644
index 0000000..2fd018c
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/metrics/rrd/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains {@link org.rrd4j.core.RrdBackendFactory} implementation
+ * that supports storing metrics history in Solr.
+ */
+package org.apache.solr.metrics.rrd;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6bbce38b/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java b/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java
index 4073947..79b4d29 100644
--- a/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java
+++ b/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java
@@ -50,6 +50,7 @@ public interface PermissionNameProvider {
AUTOSCALING_READ_PERM("autoscaling-read", null),
AUTOSCALING_WRITE_PERM("autoscaling-write", null),
AUTOSCALING_HISTORY_READ_PERM("autoscaling-history-read", null),
+ METRICS_HISTORY_READ_PERM("metrics-history-read", null),
ALL("all", unmodifiableSet(new HashSet<>(asList("*", null))))
;
final String name;