You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ja...@apache.org on 2022/09/11 22:40:00 UTC
[solr] branch branch_9x updated: SOLR-16340 Provide a cluster_id label for all metrics exposed by the Solr Prometheus Exporter (#984)
This is an automated email from the ASF dual-hosted git repository.
janhoy pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new 1a8ab18f777 SOLR-16340 Provide a cluster_id label for all metrics exposed by the Solr Prometheus Exporter (#984)
1a8ab18f777 is described below
commit 1a8ab18f77706c150aadc21877a004baf6d87abc
Author: Jan Høydahl <ja...@users.noreply.github.com>
AuthorDate: Mon Sep 12 00:29:56 2022 +0200
SOLR-16340 Provide a cluster_id label for all metrics exposed by the Solr Prometheus Exporter (#984)
Co-authored-by: Houston Putman <ho...@apache.org>
---
solr/CHANGES.txt | 3 +
.../docker/tests/cases/prometheus-exporter/test.sh | 3 +-
solr/prometheus-exporter/bin/solr-exporter | 4 +
solr/prometheus-exporter/bin/solr-exporter.cmd | 1 +
solr/prometheus-exporter/build.gradle | 1 +
.../conf/grafana-solr-dashboard.json | 215 ++++++++++++---------
.../solr/prometheus/exporter/SolrExporter.java | 59 +++++-
.../solr/prometheus/scraper/SolrCloudScraper.java | 7 +-
.../solr/prometheus/scraper/SolrScraper.java | 35 +++-
.../prometheus/scraper/SolrStandaloneScraper.java | 5 +-
.../solr/prometheus/exporter/SolrExporterTest.java | 29 +++
.../prometheus/exporter/SolrExporterTestBase.java | 3 +-
.../prometheus/scraper/SolrCloudScraperTest.java | 12 +-
.../scraper/SolrStandaloneScraperTest.java | 12 +-
.../monitoring-with-prometheus-and-grafana.adoc | 9 +
15 files changed, 271 insertions(+), 127 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 6225c153eea..3ae7048748c 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -49,6 +49,9 @@ Improvements
* SOLR-16323: The Docker image now uses the Solr User ID instead of the User Name, helps with non-root checks (Houston Putman)
+* SOLR-16340: Provide a cluster_id label for all metrics exposed by the Solr Prometheus Exporter, making it possible to
+ filter Grafana board by cluster (janhoy, Houston Putman)
+
* SOLR-16337: implement Zk metrics (noble)
* SOLR-16353: Make SplitShardCmd#checkDiskSpace disableable through a system property. (Haythem Khiri)
diff --git a/solr/docker/tests/cases/prometheus-exporter/test.sh b/solr/docker/tests/cases/prometheus-exporter/test.sh
index 76a972fcfc8..d964453170b 100755
--- a/solr/docker/tests/cases/prometheus-exporter/test.sh
+++ b/solr/docker/tests/cases/prometheus-exporter/test.sh
@@ -31,6 +31,7 @@ solr_ip=$(docker inspect --format="{{range .NetworkSettings.Networks}}{{.IPAddre
docker run --name "$container_name" --add-host "solr-host:${solr_ip}" -d \
--env "SOLR_URL=http://solr-host:8983/solr" \
--env "SCRAPE_INTERVAL=1" \
+ --env "CLUSTER_ID=myCluster" \
"$tag" "solr-exporter"
wait_for_container_and_solr_exporter "${container_name}"
@@ -41,7 +42,7 @@ docker exec --user=solr "${container_name}-solr" wget -q -O - 'http://localhost:
echo "Checking prometheus data"
data=$(docker exec --user=solr "$container_name" wget -q -O - 'http://localhost:8989/metrics')
-if ! grep -E -q 'solr_metrics_core_query_requests_total{category="QUERY",searchHandler="/select",internal="false",core="demo",base_url="http://solr-host:8983/solr",} [0-9]+.0' <<<"$data"; then
+if ! grep -E -q 'solr_metrics_core_query_requests_total{category="QUERY",searchHandler="/select",internal="false",core="demo",base_url="http://solr-host:8983/solr",cluster_id="myCluster",} [0-9]+.0' <<<"$data"; then
echo "Test $TEST_NAME $tag failed; did not find correct data"
echo "$data"
exit 1
diff --git a/solr/prometheus-exporter/bin/solr-exporter b/solr/prometheus-exporter/bin/solr-exporter
index f378e3d4258..06504457431 100755
--- a/solr/prometheus-exporter/bin/solr-exporter
+++ b/solr/prometheus-exporter/bin/solr-exporter
@@ -125,6 +125,10 @@ elif [[ -n "$SOLR_URL" ]]; then
EXPORTER_ARGS+=(-b "$SOLR_URL")
fi
+if [[ -n "$CLUSTER_ID" ]]; then
+ EXPORTER_ARGS+=(-i "$CLUSTER_ID")
+fi
+
exec "$JAVACMD" \
$JAVA_MEM_OPTS \
$GC_TUNE \
diff --git a/solr/prometheus-exporter/bin/solr-exporter.cmd b/solr/prometheus-exporter/bin/solr-exporter.cmd
index cbed311e596..e89b4042ae2 100644
--- a/solr/prometheus-exporter/bin/solr-exporter.cmd
+++ b/solr/prometheus-exporter/bin/solr-exporter.cmd
@@ -82,6 +82,7 @@ IF NOT "%SCRAPE_INTERVAL%"=="" set EXPORTER_ARGS=%EXPORTER_ARGS% -s %SCRAPE_INTE
IF NOT "%NUM_THREADS%"=="" set EXPORTER_ARGS=%EXPORTER_ARGS% -n %NUM_THREADS%
IF NOT "%ZK_HOST%"=="" set EXPORTER_ARGS=%EXPORTER_ARGS% -z %ZK_HOST%
IF NOT "%SOLR_URL%"=="" set EXPORTER_ARGS=%EXPORTER_ARGS% -b %SOLR_URL%
+IF NOT "%CLUSTER_ID%"=="" set EXPORTER_ARGS=%EXPORTER_ARGS% -i "%CLUSTER_ID%"
goto endInit
@REM Reaching here means variables are defined and arguments have been captured
diff --git a/solr/prometheus-exporter/build.gradle b/solr/prometheus-exporter/build.gradle
index 1f8faf0f53e..8d7e5d9418e 100644
--- a/solr/prometheus-exporter/build.gradle
+++ b/solr/prometheus-exporter/build.gradle
@@ -34,6 +34,7 @@ dependencies {
implementation ('net.sourceforge.argparse4j:argparse4j')
implementation ('com.github.ben-manes.caffeine:caffeine') { transitive = false }
implementation 'org.slf4j:slf4j-api'
+ implementation 'commons-codec:commons-codec'
runtimeOnly 'org.apache.logging.log4j:log4j-api'
runtimeOnly 'org.apache.logging.log4j:log4j-core'
diff --git a/solr/prometheus-exporter/conf/grafana-solr-dashboard.json b/solr/prometheus-exporter/conf/grafana-solr-dashboard.json
index 97c1a1fc426..730cd17c222 100644
--- a/solr/prometheus-exporter/conf/grafana-solr-dashboard.json
+++ b/solr/prometheus-exporter/conf/grafana-solr-dashboard.json
@@ -168,19 +168,19 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_collections_live_nodes",
+ "expr": "solr_collections_live_nodes{cluster_id=~\"$cluster_id\"}",
"format": "time_series",
"instant": false,
"interval": "",
- "legendFormat": "Live nodes",
+ "legendFormat": "Live nodes ({{cluster_id}})",
"refId": "A"
},
{
- "expr": "count(count by (node_name) (solr_collections_replica_state)) < solr_collections_live_nodes or solr_collections_live_nodes",
+ "expr": "count(count by (node_name) (solr_collections_replica_state{cluster_id=~\"$cluster_id\"})) < solr_collections_live_nodes{cluster_id=~\"$cluster_id\"} or solr_collections_live_nodes{cluster_id=~\"$cluster_id\"}",
"hide": false,
"interval": "",
"intervalFactor": 1,
- "legendFormat": "Total nodes",
+ "legendFormat": "Total nodes ({{cluster_id}})",
"refId": "B"
}
],
@@ -289,19 +289,19 @@
"steppedLine": false,
"targets": [
{
- "expr": "count(solr_zookeeper_nodestatus == 1)",
+ "expr": "count(solr_zookeeper_nodestatus{cluster_id=~\"$cluster_id\"} == 1)",
"format": "time_series",
"instant": false,
"interval": "",
- "legendFormat": "Healthy nodes",
+ "legendFormat": "Healthy nodes ({{cluster_id}})",
"refId": "A"
},
{
- "expr": "solr_zookeeper_ensemble_size",
+ "expr": "solr_zookeeper_ensemble_size{cluster_id=~\"$cluster_id\"}",
"hide": false,
"interval": "",
"intervalFactor": 1,
- "legendFormat": "Ensemble size",
+ "legendFormat": "Ensemble size ({{cluster_id}})",
"refId": "B"
}
],
@@ -399,9 +399,9 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_collections_shard_leader{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}",
+ "expr": "solr_collections_shard_leader{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}",
"interval": "",
- "legendFormat": "{{collection}} {{shard}} {{replica}} ({{node_name}})",
+ "legendFormat": "{{cluster_id}} {{collection}} {{shard}} {{replica}} ({{node_name}})",
"refId": "A"
}
],
@@ -495,7 +495,7 @@
"pluginVersion": "7.3.0",
"targets": [
{
- "expr": "count(count by (collection) (solr_collections_replica_state{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}))",
+ "expr": "count(count by (collection) (solr_collections_replica_state{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}))",
"interval": "",
"legendFormat": "",
"refId": "A"
@@ -551,31 +551,31 @@
"pluginVersion": "7.3.0",
"targets": [
{
- "expr": "count(solr_collections_replica_state{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"})",
+ "expr": "count(solr_collections_replica_state{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"})",
"interval": "",
"legendFormat": "Total replicas",
"refId": "B"
},
{
- "expr": "count(solr_collections_replica_state{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"} == 1) OR on() vector(0)",
+ "expr": "count(solr_collections_replica_state{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"} == 1) OR on() vector(0)",
"interval": "",
"legendFormat": "Active",
"refId": "D"
},
{
- "expr": "count(solr_collections_replica_state{state=\"down\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}) OR on() vector(0)",
+ "expr": "count(solr_collections_replica_state{state=\"down\",cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}) OR on() vector(0)",
"interval": "",
"legendFormat": "Down",
"refId": "A"
},
{
- "expr": "count(solr_collections_replica_state{state=\"recovering\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}) OR on() vector(0)",
+ "expr": "count(solr_collections_replica_state{state=\"recovering\",cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}) OR on() vector(0)",
"interval": "",
"legendFormat": "Recovering",
"refId": "C"
},
{
- "expr": "count(solr_collections_replica_state{state=\"recovery_failed\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}) OR on() vector(0)",
+ "expr": "count(solr_collections_replica_state{state=\"recovery_failed\",cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}) OR on() vector(0)",
"interval": "",
"legendFormat": "Recovery failed",
"refId": "E"
@@ -651,9 +651,9 @@
"steppedLine": false,
"targets": [
{
- "expr": "1 - solr_collections_replica_state{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}",
+ "expr": "1 - solr_collections_replica_state{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}",
"interval": "",
- "legendFormat": "{{collection}} {{shard}} {{replica}} ({{node_name}})={{state}}",
+ "legendFormat": "{{cluster_id}} {{collection}} {{shard}} {{replica}} ({{node_name}})={{state}}",
"refId": "A"
}
],
@@ -775,9 +775,9 @@
"steppedLine": false,
"targets": [
{
- "expr": "sum by (collection) (solr_metrics_core_query_1minRate{internal=\"false\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"})",
+ "expr": "sum by (cluster_id, collection) (solr_metrics_core_query_1minRate{internal=\"false\",cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"})",
"interval": "",
- "legendFormat": "{{collection}}",
+ "legendFormat": "{{collection}} ({{cluster_id}})",
"refId": "A"
}
],
@@ -886,7 +886,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "sum by (base_url) (solr_metrics_core_query_1minRate{internal=\"false\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"})",
+ "expr": "sum by (base_url) (solr_metrics_core_query_1minRate{internal=\"false\",cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"})",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{base_url}}",
@@ -994,7 +994,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_core_query_p95_ms{internal=\"false\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
+ "expr": "solr_metrics_core_query_p95_ms{internal=\"false\",cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -1095,7 +1095,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_core_query_5minRate{internal=\"false\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
+ "expr": "solr_metrics_core_query_5minRate{internal=\"false\",cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -1196,7 +1196,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_core_query_p99_ms{internal=\"false\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
+ "expr": "solr_metrics_core_query_p99_ms{internal=\"false\",cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -1297,7 +1297,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_core_query_1minRate{internal=\"false\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
+ "expr": "solr_metrics_core_query_1minRate{internal=\"false\",cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
"format": "time_series",
"instant": false,
"interval": "",
@@ -1399,7 +1399,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_core_query_p75_ms{internal=\"false\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
+ "expr": "solr_metrics_core_query_p75_ms{internal=\"false\",cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -1501,7 +1501,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_core_query_requests_total{internal!=\"false\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}[1m])",
+ "expr": "increase(solr_metrics_core_query_requests_total{internal!=\"false\",cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}[1m])",
"format": "time_series",
"instant": false,
"interval": "",
@@ -1603,7 +1603,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_core_query_1minRate{internal!=\"false\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
+ "expr": "solr_metrics_core_query_1minRate{internal!=\"false\",cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
"format": "time_series",
"instant": false,
"interval": "",
@@ -1705,7 +1705,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_core_query_p95_ms{internal!=\"false\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
+ "expr": "solr_metrics_core_query_p95_ms{internal!=\"false\",cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 3,
@@ -1805,7 +1805,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_core_query_client_errors_1minRate{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
+ "expr": "solr_metrics_core_query_client_errors_1minRate{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -1905,7 +1905,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_core_query_errors_1minRate{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
+ "expr": "solr_metrics_core_query_errors_1minRate{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\",searchHandler=~\"$searchHandler\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -2022,7 +2022,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_jetty_requests_total{base_url=~\"$base_url\"}[1m])",
+ "expr": "increase(solr_metrics_jetty_requests_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}[1m])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}} {{method}}",
@@ -2123,7 +2123,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_jetty_response_total{base_url=~\"$base_url\"}[1m])",
+ "expr": "increase(solr_metrics_jetty_response_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}[1m])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}} {{status}}",
@@ -2223,7 +2223,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_jetty_dispatches_total{base_url=~\"$base_url\"}[1m])",
+ "expr": "increase(solr_metrics_jetty_dispatches_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}[1m])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}}",
@@ -2337,7 +2337,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_jvm_buffers{base_url=~\"$base_url\"}",
+ "expr": "solr_metrics_jvm_buffers{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}} {{pool}}",
@@ -2435,7 +2435,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_jvm_buffers_bytes{base_url=~\"$base_url\"}",
+ "expr": "solr_metrics_jvm_buffers_bytes{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}} {{pool}} {{item}}",
@@ -2533,7 +2533,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_jvm_gc_total{base_url=~\"$base_url\"}[1m])",
+ "expr": "increase(solr_metrics_jvm_gc_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}[1m])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}} {{item}}",
@@ -2631,7 +2631,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_jvm_gc_seconds_total{base_url=~\"$base_url\"}[1m])",
+ "expr": "increase(solr_metrics_jvm_gc_seconds_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}[1m])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}} {{item}}",
@@ -2730,7 +2730,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_jvm_memory_heap_bytes{base_url=~\"$base_url\"}",
+ "expr": "solr_metrics_jvm_memory_heap_bytes{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}} {{item}}",
@@ -2828,7 +2828,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_jvm_memory_non_heap_bytes{base_url=~\"$base_url\"}",
+ "expr": "solr_metrics_jvm_memory_non_heap_bytes{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}} {{item}}",
@@ -2926,7 +2926,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_jvm_memory_pools_bytes{base_url=~\"$base_url\"}",
+ "expr": "solr_metrics_jvm_memory_pools_bytes{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}} {{space}} {{item}}",
@@ -3024,7 +3024,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_jvm_memory_bytes{base_url=~\"$base_url\"}",
+ "expr": "solr_metrics_jvm_memory_bytes{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}} {{item}}",
@@ -3125,7 +3125,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_jvm_threads{base_url=~\"$base_url\"}",
+ "expr": "solr_metrics_jvm_threads{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}} {{item}}",
@@ -3239,7 +3239,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_jvm_os_memory_bytes{base_url=~\"$base_url\"}",
+ "expr": "solr_metrics_jvm_os_memory_bytes{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}} {{item}}",
@@ -3337,7 +3337,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_jvm_os_file_descriptors{base_url=~\"$base_url\"}",
+ "expr": "solr_metrics_jvm_os_file_descriptors{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}} {{item}}",
@@ -3435,7 +3435,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_jvm_os_cpu_load{base_url=~\"$base_url\"}",
+ "expr": "solr_metrics_jvm_os_cpu_load{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}} {{item}}",
@@ -3533,7 +3533,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_jvm_os_cpu_time_seconds{base_url=~\"$base_url\"}[1m])",
+ "expr": "increase(solr_metrics_jvm_os_cpu_time_seconds{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}[1m])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -3635,7 +3635,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_jvm_os_load_average{base_url=~\"$base_url\"}",
+ "expr": "solr_metrics_jvm_os_load_average{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}} {{item}}",
@@ -3751,7 +3751,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_node_requests_total{base_url=~\"$base_url\"}",
+ "expr": "solr_metrics_node_requests_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -3852,7 +3852,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_node_time_seconds_total{base_url=~\"$base_url\"}[1m])",
+ "expr": "increase(solr_metrics_node_time_seconds_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}[1m])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -3953,7 +3953,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_node_client_errors_total{base_url=~\"$base_url\"}[1m])",
+ "expr": "increase(solr_metrics_node_client_errors_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}[1m])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -4054,7 +4054,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_node_server_errors_total{base_url=~\"$base_url\"}[1m])",
+ "expr": "increase(solr_metrics_node_server_errors_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}[1m])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}}{{handler}}",
@@ -4154,7 +4154,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_node_errors_total{base_url=~\"$base_url\"}[1m])",
+ "expr": "increase(solr_metrics_node_errors_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}[1m])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}}{{handler}}",
@@ -4254,7 +4254,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_node_timeouts_total{base_url=~\"$base_url\"}[1m])",
+ "expr": "increase(solr_metrics_node_timeouts_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}[1m])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}}{{handler}}",
@@ -4354,7 +4354,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_node_cores{base_url=~\"$base_url\"}",
+ "expr": "solr_metrics_node_cores{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}} {{item}}",
@@ -4452,7 +4452,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_node_core_root_fs_bytes{base_url=~\"$base_url\"}",
+ "expr": "solr_metrics_node_core_root_fs_bytes{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}} {{item}}",
@@ -4552,7 +4552,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_node_thread_pool_completed_total{base_url=~\"$base_url\"}",
+ "expr": "solr_metrics_node_thread_pool_completed_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -4653,7 +4653,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_node_thread_pool_submitted_total{base_url=~\"$base_url\"}",
+ "expr": "solr_metrics_node_thread_pool_submitted_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -4755,7 +4755,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_node_thread_pool_running{base_url=~\"$base_url\"}",
+ "expr": "solr_metrics_node_thread_pool_running{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -4857,7 +4857,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_node_connections{base_url=~\"$base_url\"}",
+ "expr": "solr_metrics_node_connections{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}} {{handler}} {{item}}",
@@ -4969,7 +4969,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "sum by (base_url) (solr_collections_shard_leader{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"})",
+ "expr": "sum by (base_url) (solr_collections_shard_leader{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"})",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{base_url}}",
@@ -5093,7 +5093,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_core_requests_total{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
+ "expr": "increase(solr_metrics_core_requests_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -5194,7 +5194,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_core_time_seconds_total{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
+ "expr": "increase(solr_metrics_core_time_seconds_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -5295,7 +5295,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_core_client_errors_total{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
+ "expr": "increase(solr_metrics_core_client_errors_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}}/{{core}}{{handler}}",
@@ -5395,7 +5395,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_core_server_errors_total{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
+ "expr": "increase(solr_metrics_core_server_errors_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}}/{{core}}{{handler}}",
@@ -5495,7 +5495,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_core_errors_total{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
+ "expr": "increase(solr_metrics_core_errors_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}}/{{core}}{{handler}}",
@@ -5595,7 +5595,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_core_timeouts_total{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
+ "expr": "increase(solr_metrics_core_timeouts_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}}/{{core}}{{handler}}",
@@ -5694,7 +5694,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_core_searcher_cache_ratio{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}",
+ "expr": "solr_metrics_core_searcher_cache_ratio{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -5795,7 +5795,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_core_searcher_cache{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}",
+ "expr": "solr_metrics_core_searcher_cache{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -5896,7 +5896,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_core_searcher_warmup_time_seconds{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}",
+ "expr": "solr_metrics_core_searcher_warmup_time_seconds{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}}/{{core}} {{type}}",
@@ -5996,7 +5996,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_core_searcher_cumulative_cache_total{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
+ "expr": "increase(solr_metrics_core_searcher_cumulative_cache_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}}/{{core}} {{type}} {{item}}",
@@ -6094,7 +6094,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_core_fs_bytes{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}",
+ "expr": "solr_metrics_core_fs_bytes{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}}/{{core}}",
@@ -6192,7 +6192,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_core_index_size_bytes{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}",
+ "expr": "solr_metrics_core_index_size_bytes{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}}/{{core}}",
@@ -6293,7 +6293,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_core_searcher_documents{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}",
+ "expr": "solr_metrics_core_searcher_documents{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}}/{{core}} {{item}}",
@@ -6392,7 +6392,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_core_update_handler_adds_total{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
+ "expr": "increase(solr_metrics_core_update_handler_adds_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -6491,7 +6491,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "solr_metrics_core_update_handler_pending_docs{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}",
+ "expr": "solr_metrics_core_update_handler_pending_docs{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}}/{{core}}",
@@ -6589,7 +6589,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_core_update_handler_deletes_by_id_total{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
+ "expr": "increase(solr_metrics_core_update_handler_deletes_by_id_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}}/{{core}}",
@@ -6687,7 +6687,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_core_update_handler_deletes_by_query_total{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
+ "expr": "increase(solr_metrics_core_update_handler_deletes_by_query_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}}/{{core}}",
@@ -6785,7 +6785,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_core_update_handler_expunge_deletes_total{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
+ "expr": "increase(solr_metrics_core_update_handler_expunge_deletes_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}}/{{core}}",
@@ -6883,7 +6883,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_core_update_handler_merges_total{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
+ "expr": "increase(solr_metrics_core_update_handler_merges_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}}/{{core}}",
@@ -6981,7 +6981,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_core_update_handler_splits_total{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
+ "expr": "increase(solr_metrics_core_update_handler_splits_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}}/{{core}}",
@@ -7079,7 +7079,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_core_update_handler_optimizes_total{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
+ "expr": "increase(solr_metrics_core_update_handler_optimizes_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}}/{{core}}",
@@ -7177,7 +7177,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_core_update_handler_rollbacks_total{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
+ "expr": "increase(solr_metrics_core_update_handler_rollbacks_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{base_url}}/{{core}}",
@@ -7277,7 +7277,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_core_update_handler_soft_auto_commits_total{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
+ "expr": "increase(solr_metrics_core_update_handler_soft_auto_commits_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -7378,7 +7378,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_core_update_handler_auto_commits_total{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
+ "expr": "increase(solr_metrics_core_update_handler_auto_commits_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -7479,7 +7479,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_core_update_handler_commits_total{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
+ "expr": "increase(solr_metrics_core_update_handler_commits_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -7581,7 +7581,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "increase(solr_metrics_core_update_handler_errors_total{base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
+ "expr": "increase(solr_metrics_core_update_handler_errors_total{cluster_id=~\"$cluster_id\",base_url=~\"$base_url\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\",core=~\"$core\"}[1m])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -7642,6 +7642,29 @@
"tags": [],
"templating": {
"list": [
+ {
+ "allValue": ".*",
+ "current": {},
+ "datasource": "${DS_PROMETHEUS}",
+ "definition": "",
+ "error": null,
+ "hide": 0,
+ "includeAll": true,
+ "label": null,
+ "multi": true,
+ "name": "cluster_id",
+ "options": [],
+ "query": "label_values(solr_ping{cluster_id=~\".+\"},cluster_id)",
+ "refresh": 1,
+ "regex": "/(.+)/",
+ "skipUrlSync": false,
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
{
"allValue": ".*",
"current": {},
@@ -7654,7 +7677,7 @@
"multi": true,
"name": "zk_host",
"options": [],
- "query": "label_values(solr_ping{zk_host=~\".+\"},zk_host)",
+ "query": "label_values(solr_ping{cluster_id=~\"$cluster_id\",zk_host=~\".+\"},zk_host)",
"refresh": 1,
"regex": "/(.+)/",
"skipUrlSync": false,
@@ -7677,7 +7700,7 @@
"multi": true,
"name": "base_url",
"options": [],
- "query": "label_values(solr_ping{base_url=~\".+\"},base_url)",
+ "query": "label_values(solr_ping{cluster_id=~\"$cluster_id\",base_url=~\".+\"},base_url)",
"refresh": 1,
"regex": "/(.+)/",
"skipUrlSync": false,
@@ -7700,7 +7723,7 @@
"multi": true,
"name": "collection",
"options": [],
- "query": "label_values(solr_collections_shard_state{zk_host=~\"$zk_host\",collection=~\".+\"},collection)",
+ "query": "label_values(solr_collections_shard_state{cluster_id=~\"$cluster_id\",zk_host=~\"$zk_host\",collection=~\".+\"},collection)",
"refresh": 1,
"regex": "/(.+)/",
"skipUrlSync": false,
@@ -7723,7 +7746,7 @@
"multi": true,
"name": "shard",
"options": [],
- "query": "label_values(solr_collections_shard_state{zk_host=~\"$zk_host\",collection=~\"$collection\",shard=~\".+\"},shard)",
+ "query": "label_values(solr_collections_shard_state{cluster_id=~\"$cluster_id\",zk_host=~\"$zk_host\",collection=~\"$collection\",shard=~\".+\"},shard)",
"refresh": 1,
"regex": "/(.+)/",
"skipUrlSync": false,
@@ -7746,7 +7769,7 @@
"multi": true,
"name": "replica",
"options": [],
- "query": "label_values(solr_collections_replica_state{zk_host=~\"$zk_host\",collection=~\"$collection\",shard=~\"$shard\",replica=~\".+\"},replica)",
+ "query": "label_values(solr_collections_replica_state{cluster_id=~\"$cluster_id\",zk_host=~\"$zk_host\",collection=~\"$collection\",shard=~\"$shard\",replica=~\".+\"},replica)",
"refresh": 1,
"regex": "/(.+)/",
"skipUrlSync": false,
@@ -7769,7 +7792,7 @@
"multi": true,
"name": "core",
"options": [],
- "query": "label_values(solr_collections_replica_state{zk_host=~\"$zk_host\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\"},core)",
+ "query": "label_values(solr_collections_replica_state{cluster_id=~\"$cluster_id\",zk_host=~\"$zk_host\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\"},core)",
"refresh": 1,
"regex": "/(.+)/",
"skipUrlSync": false,
@@ -7784,7 +7807,7 @@
"allValue": ".*",
"current": {},
"datasource": "${DS_PROMETHEUS}",
- "definition": "label_values(solr_metrics_core_query_errors_1minRate{zk_host=~\"$zk_host\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\"},searchHandler)",
+ "definition": "label_values(solr_metrics_core_query_errors_1minRate{cluster_id=~\"$cluster_id\",zk_host=~\"$zk_host\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\"},searchHandler)",
"error": null,
"hide": 0,
"includeAll": true,
@@ -7792,7 +7815,7 @@
"multi": true,
"name": "searchHandler",
"options": [],
- "query": "label_values(solr_metrics_core_query_errors_1minRate{zk_host=~\"$zk_host\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\"},searchHandler)",
+ "query": "label_values(solr_metrics_core_query_errors_1minRate{cluster_id=~\"$cluster_id\",zk_host=~\"$zk_host\",collection=~\"$collection\",shard=~\"$shard\",replica=~\"$replica\"},searchHandler)",
"refresh": 1,
"regex": "/(.+)/",
"skipUrlSync": false,
@@ -7838,4 +7861,4 @@
"title": "Solr Dashboard",
"uid": "yt5eXHbMz5",
"version": 2
-}
\ No newline at end of file
+}
diff --git a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrExporter.java b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrExporter.java
index eedddd1e6ed..139390f5f12 100644
--- a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrExporter.java
+++ b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrExporter.java
@@ -26,6 +26,8 @@ import net.sourceforge.argparse4j.ArgumentParsers;
import net.sourceforge.argparse4j.inf.ArgumentParser;
import net.sourceforge.argparse4j.inf.ArgumentParserException;
import net.sourceforge.argparse4j.inf.Namespace;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.solr.common.StringUtils;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.SolrNamedThreadFactory;
@@ -62,6 +64,13 @@ public class SolrExporter {
private static final String ARG_ZK_HOST_HELP =
"Specify the ZooKeeper connection string when connecting to Solr in SolrCloud mode. If omitted both the -b parameter and the -z parameter, connect to http://localhost:8983/solr. For example 'localhost:2181/solr'.";
+ private static final String[] ARG_CLUSTER_ID_FLAGS = {"-i", "--cluster-id"};
+ private static final String ARG_CLUSTER_ID_METAVAR = "CLUSTER_ID";
+ private static final String ARG_CLUSTER_ID_DEST = "clusterId";
+ private static final String ARG_CLUSTER_ID_DEFAULT = "";
+ private static final String ARG_CLUSTER_ID_HELP =
+ "Specify a unique identifier for the cluster, which can be used to select between multiple clusters in Grafana. By default this ID will be equal to a hash of the -b or -z argument";
+
private static final String[] ARG_CONFIG_FLAGS = {"-f", "--config-file"};
private static final String ARG_CONFIG_METAVAR = "CONFIG";
private static final String ARG_CONFIG_DEST = "configFile";
@@ -104,7 +113,8 @@ public class SolrExporter {
int numberThreads,
int scrapeInterval,
SolrScrapeConfiguration scrapeConfiguration,
- MetricsConfiguration metricsConfiguration) {
+ MetricsConfiguration metricsConfiguration,
+ String clusterId) {
this.port = port;
this.metricCollectorExecutor =
@@ -115,7 +125,8 @@ public class SolrExporter {
ExecutorUtil.newMDCAwareFixedThreadPool(
numberThreads, new SolrNamedThreadFactory("solr-exporter-requests"));
- this.solrScraper = createScraper(scrapeConfiguration, metricsConfiguration.getSettings());
+ this.solrScraper =
+ createScraper(scrapeConfiguration, metricsConfiguration.getSettings(), clusterId);
this.metricsCollector =
new MetricsCollectorFactory(
metricCollectorExecutor, scrapeInterval, solrScraper, metricsConfiguration)
@@ -147,18 +158,23 @@ public class SolrExporter {
}
private SolrScraper createScraper(
- SolrScrapeConfiguration configuration, PrometheusExporterSettings settings) {
+ SolrScrapeConfiguration configuration,
+ PrometheusExporterSettings settings,
+ String clusterId) {
SolrClientFactory factory = new SolrClientFactory(settings);
switch (configuration.getType()) {
case STANDALONE:
return new SolrStandaloneScraper(
- factory.createStandaloneSolrClient(configuration.getSolrHost().get()), requestExecutor);
+ factory.createStandaloneSolrClient(configuration.getSolrHost().get()),
+ requestExecutor,
+ clusterId);
case CLOUD:
return new SolrCloudScraper(
factory.createCloudSolrClient(configuration.getZookeeperConnectionString().get()),
requestExecutor,
- factory);
+ factory,
+ clusterId);
default:
throw new RuntimeException("Invalid type: " + configuration.getType());
}
@@ -218,14 +234,25 @@ public class SolrExporter {
.setDefault(ARG_NUM_THREADS_DEFAULT)
.help(ARG_NUM_THREADS_HELP);
+ parser
+ .addArgument(ARG_CLUSTER_ID_FLAGS)
+ .metavar(ARG_CLUSTER_ID_METAVAR)
+ .dest(ARG_CLUSTER_ID_DEST)
+ .type(String.class)
+ .setDefault(ARG_CLUSTER_ID_DEFAULT)
+ .help(ARG_CLUSTER_ID_HELP);
+
try {
Namespace res = parser.parseArgs(args);
SolrScrapeConfiguration scrapeConfiguration = null;
+ String defaultClusterId = "";
if (!res.getString(ARG_ZK_HOST_DEST).equals("")) {
+ defaultClusterId = makeShortHash(res.getString(ARG_ZK_HOST_DEST));
scrapeConfiguration = SolrScrapeConfiguration.solrCloud(res.getString(ARG_ZK_HOST_DEST));
} else if (!res.getString(ARG_BASE_URL_DEST).equals("")) {
+ defaultClusterId = makeShortHash(res.getString(ARG_BASE_URL_DEST));
scrapeConfiguration = SolrScrapeConfiguration.standalone(res.getString(ARG_BASE_URL_DEST));
}
@@ -234,18 +261,26 @@ public class SolrExporter {
}
int port = res.getInt(ARG_PORT_DEST);
+ String clusterId = res.getString(ARG_CLUSTER_ID_DEST);
+ if (StringUtils.isEmpty(clusterId)) {
+ clusterId = defaultClusterId;
+ }
+
SolrExporter solrExporter =
new SolrExporter(
port,
res.getInt(ARG_NUM_THREADS_DEST),
res.getInt(ARG_SCRAPE_INTERVAL_DEST),
scrapeConfiguration,
- loadMetricsConfiguration(res.getString(ARG_CONFIG_DEST)));
+ loadMetricsConfiguration(res.getString(ARG_CONFIG_DEST)),
+ clusterId);
log.info("Starting Solr Prometheus Exporting on port {}", port);
solrExporter.start();
log.info(
- "Solr Prometheus Exporter is running. Collecting metrics for {}", scrapeConfiguration);
+ "Solr Prometheus Exporter is running. Collecting metrics for cluster {}: {}",
+ clusterId,
+ scrapeConfiguration);
} catch (IOException e) {
log.error("Failed to start Solr Prometheus Exporter: ", e);
} catch (ArgumentParserException e) {
@@ -253,6 +288,16 @@ public class SolrExporter {
}
}
+ /**
+ * Creates a short 10-char hash of a longer string, based on first chars of the sha256 hash
+ *
+ * @param inputString original string
+ * @return 10 char hash
+ */
+ static String makeShortHash(String inputString) {
+ return DigestUtils.sha256Hex(inputString).substring(0, 10);
+ }
+
private static MetricsConfiguration loadMetricsConfiguration(String configPath) {
try {
return MetricsConfiguration.from(configPath);
diff --git a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java
index 2a6c6c3479a..3944dc042a9 100644
--- a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java
+++ b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java
@@ -43,8 +43,11 @@ public class SolrCloudScraper extends SolrScraper {
private Cache<String, HttpSolrClient> hostClientCache = Caffeine.newBuilder().build();
public SolrCloudScraper(
- CloudSolrClient solrClient, ExecutorService executor, SolrClientFactory solrClientFactory) {
- super(executor);
+ CloudSolrClient solrClient,
+ ExecutorService executor,
+ SolrClientFactory solrClientFactory,
+ String clusterId) {
+ super(executor, clusterId);
this.solrClient = solrClient;
this.solrClientFactory = solrClientFactory;
}
diff --git a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrScraper.java b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrScraper.java
index 2ebc561cc0b..270abf76222 100644
--- a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrScraper.java
+++ b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrScraper.java
@@ -48,14 +48,22 @@ import org.slf4j.LoggerFactory;
public abstract class SolrScraper implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ protected static final String ZK_HOST_LABEL = "zk_host";
+ protected static final String BASE_URL_LABEL = "base_url";
+ protected static final String CLUSTER_ID_LABEL = "cluster_id";
+
private static final Counter scrapeErrorTotal =
Counter.build()
.name("solr_exporter_scrape_error_total")
.help("Number of scrape error.")
+ .labelNames(ZK_HOST_LABEL, BASE_URL_LABEL, CLUSTER_ID_LABEL)
.register(SolrExporter.defaultRegistry);
protected static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ protected final String clusterId;
protected final ExecutorService executor;
@@ -71,8 +79,9 @@ public abstract class SolrScraper implements Closeable {
public abstract MetricSamples collections(MetricsQuery metricsQuery) throws IOException;
- public SolrScraper(ExecutorService executor) {
+ public SolrScraper(ExecutorService executor, String clusterId) {
this.executor = executor;
+ this.clusterId = clusterId;
}
protected Map<String, MetricSamples> sendRequestsInParallel(
@@ -113,6 +122,14 @@ public abstract class SolrScraper implements Closeable {
protected MetricSamples request(SolrClient client, MetricsQuery query) throws IOException {
MetricSamples samples = new MetricSamples();
+ String baseUrlLabelValue = "";
+ String zkHostLabelValue = "";
+ if (client instanceof HttpSolrClient) {
+ baseUrlLabelValue = ((HttpSolrClient) client).getBaseURL();
+ } else if (client instanceof CloudSolrClient) {
+ zkHostLabelValue = ((CloudSolrClient) client).getClusterStateProvider().getQuorumHosts();
+ }
+
QueryRequest queryRequest = new QueryRequest(query.getParameters());
queryRequest.setPath(query.getPath());
@@ -154,15 +171,19 @@ public abstract class SolrScraper implements Closeable {
/* Labels due to client */
if (client instanceof HttpSolrClient) {
- labelNames.add("base_url");
- labelValues.add(((HttpSolrClient) client).getBaseURL());
+ labelNames.add(BASE_URL_LABEL);
+ labelValues.add(baseUrlLabelValue);
}
if (client instanceof CloudSolrClient) {
- labelNames.add("zk_host");
- labelValues.add(((CloudSolrClient) client).getClusterStateProvider().getQuorumHosts());
+ labelNames.add(ZK_HOST_LABEL);
+ labelValues.add(zkHostLabelValue);
}
+ // Add the unique cluster ID, either as specified on cmdline -i or baseUrl/zkHost
+ labelNames.add(CLUSTER_ID_LABEL);
+ labelValues.add(clusterId);
+
// Deduce core if not there
if (labelNames.indexOf("core") < 0
&& labelNames.indexOf("collection") >= 0
@@ -187,7 +208,7 @@ public abstract class SolrScraper implements Closeable {
}
} catch (JsonQueryException e) {
log.error("Error apply JSON query={} to result", jsonQuery, e);
- scrapeErrorTotal.inc();
+ scrapeErrorTotal.labels(zkHostLabelValue, baseUrlLabelValue, clusterId).inc();
}
}
diff --git a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrStandaloneScraper.java b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrStandaloneScraper.java
index 7c87407a59a..90139e968f4 100644
--- a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrStandaloneScraper.java
+++ b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrStandaloneScraper.java
@@ -37,8 +37,9 @@ public class SolrStandaloneScraper extends SolrScraper {
private final HttpSolrClient solrClient;
- public SolrStandaloneScraper(HttpSolrClient solrClient, ExecutorService executor) {
- super(executor);
+ public SolrStandaloneScraper(
+ HttpSolrClient solrClient, ExecutorService executor, String clusterId) {
+ super(executor, clusterId);
this.solrClient = solrClient;
}
diff --git a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/exporter/SolrExporterTest.java b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/exporter/SolrExporterTest.java
new file mode 100644
index 00000000000..c01bc160304
--- /dev/null
+++ b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/exporter/SolrExporterTest.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.prometheus.exporter;
+
+import junit.framework.TestCase;
+
+public class SolrExporterTest extends TestCase {
+ public void testDefaultClusterIdHash() {
+ assertEquals(
+ "a2542cfcee",
+ SolrExporter.makeShortHash(
+ "my.zk.server:2181,other.zk.server:2181,third.zk.server:2181/solr"));
+ assertEquals("1999820420", SolrExporter.makeShortHash("my.solr.server:8983/solr"));
+ }
+}
diff --git a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/exporter/SolrExporterTestBase.java b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/exporter/SolrExporterTestBase.java
index 0e648fe37f6..7ef4e3dadb0 100644
--- a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/exporter/SolrExporterTestBase.java
+++ b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/exporter/SolrExporterTestBase.java
@@ -67,7 +67,8 @@ public class SolrExporterTestBase extends PrometheusExporterTestBase {
25,
10,
SolrScrapeConfiguration.solrCloud(cluster.getZkServer().getZkAddress()),
- Helpers.loadConfiguration(scrapeConfiguration));
+ Helpers.loadConfiguration(scrapeConfiguration),
+ "test");
solrExporter.start();
httpClient = HttpClients.createDefault();
diff --git a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrCloudScraperTest.java b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrCloudScraperTest.java
index 0106ef021be..dffb6bef17a 100644
--- a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrCloudScraperTest.java
+++ b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrCloudScraperTest.java
@@ -69,7 +69,7 @@ public class SolrCloudScraperTest extends PrometheusExporterTestBase {
SolrClientFactory factory = new SolrClientFactory(PrometheusExporterSettings.builder().build());
- return new SolrCloudScraper(solrClient, executor, factory);
+ return new SolrCloudScraper(solrClient, executor, factory, "test");
}
private ClusterState getClusterState() {
@@ -119,10 +119,9 @@ public class SolrCloudScraperTest extends PrometheusExporterTestBase {
assertEquals(1, collection1Metrics.samples.size());
assertEquals(1.0, collection1Metrics.samples.get(0).value, 0.001);
+ assertEquals(List.of("zk_host", "cluster_id"), collection1Metrics.samples.get(0).labelNames);
assertEquals(
- Collections.singletonList("zk_host"), collection1Metrics.samples.get(0).labelNames);
- assertEquals(
- Collections.singletonList(cluster.getZkServer().getZkAddress()),
+ List.of(cluster.getZkServer().getZkAddress(), "test"),
collection1Metrics.samples.get(0).labelValues);
}
@@ -183,7 +182,7 @@ public class SolrCloudScraperTest extends PrometheusExporterTestBase {
for (Collector.MetricFamilySamples.Sample sample : shardLeaderSamples.samples) {
assertEquals("solr_collections_shard_leader", sample.name);
assertEquals(
- Arrays.asList("collection", "shard", "replica", "core", "type", "zk_host"),
+ Arrays.asList("collection", "shard", "replica", "core", "type", "zk_host", "cluster_id"),
sample.labelNames);
assertEquals(
leaderCoreNames.contains(sample.labelValues.get(3)) ? 1.0 : 0.0, sample.value, 0.001);
@@ -203,6 +202,9 @@ public class SolrCloudScraperTest extends PrometheusExporterTestBase {
metricsByHost.get(replica.getBaseUrl()).asList();
assertEquals(1, replicaSamples.size());
assertEquals("solr_metrics_jvm_buffers", replicaSamples.get(0).name);
+
+ assertEquals("cluster_id", replicaSamples.get(0).samples.get(0).labelNames.get(2));
+ assertEquals("test", replicaSamples.get(0).samples.get(0).labelValues.get(2));
}
}
diff --git a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrStandaloneScraperTest.java b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrStandaloneScraperTest.java
index 7442c44d488..276e32fea77 100644
--- a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrStandaloneScraperTest.java
+++ b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrStandaloneScraperTest.java
@@ -20,7 +20,6 @@ package org.apache.solr.prometheus.scraper;
import io.prometheus.client.Collector;
import java.io.File;
import java.io.IOException;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
@@ -65,7 +64,7 @@ public class SolrStandaloneScraperTest extends RestTestBase {
Helpers.loadConfiguration("conf/prometheus-solr-exporter-scraper-test-config.xml");
solrClient = getHttpSolrClient(restTestHarness.getAdminURL());
- solrScraper = new SolrStandaloneScraper(solrClient, executor);
+ solrScraper = new SolrStandaloneScraper(solrClient, executor, "test");
NoOpResponseParser responseParser = new NoOpResponseParser();
responseParser.setWriterType("json");
@@ -113,10 +112,9 @@ public class SolrStandaloneScraperTest extends RestTestBase {
assertEquals("solr_ping", samples.name);
assertEquals(1, samples.samples.size());
assertEquals(1.0, samples.samples.get(0).value, 0.001);
- assertEquals(Collections.singletonList("base_url"), samples.samples.get(0).labelNames);
+ assertEquals(List.of("base_url", "cluster_id"), samples.samples.get(0).labelNames);
assertEquals(
- Collections.singletonList(restTestHarness.getAdminURL()),
- samples.samples.get(0).labelValues);
+ List.of(restTestHarness.getAdminURL(), "test"), samples.samples.get(0).labelValues);
}
@Test
@@ -139,8 +137,10 @@ public class SolrStandaloneScraperTest extends RestTestBase {
assertEquals(1, replicaSamples.size());
- assertEquals(1, replicaSamples.size());
assertEquals("solr_metrics_jvm_buffers", replicaSamples.get(0).name);
+
+ assertEquals("cluster_id", replicaSamples.get(0).samples.get(0).labelNames.get(2));
+ assertEquals("test", replicaSamples.get(0).samples.get(0).labelValues.get(2));
}
@Test
diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/monitoring-with-prometheus-and-grafana.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/monitoring-with-prometheus-and-grafana.adoc
index 1244584c38a..dbd4744f7bb 100644
--- a/solr/solr-ref-guide/modules/deployment-guide/pages/monitoring-with-prometheus-and-grafana.adoc
+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/monitoring-with-prometheus-and-grafana.adoc
@@ -161,6 +161,15 @@ The `solr-exporter` collects metrics from Solr every few seconds controlled by t
These metrics are cached and returned regardless of how frequently prometheus is configured to pull metrics from this tool.
The freshness of the metrics can be improved by reducing the scrape interval but do not set it to a very low value because metrics collection can be expensive and can execute arbitrary searches to ping Solr.
+`-i`, `--cluster-id`, `$CLUSTER_ID`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: _see description_
+|===
++
+A unique ID for the cluster to monitor. This ID will be added to all metrics as a label `cluster_id` and can be used as a filter in the Grafana dashboard if you operate multiple Solr clusters reporting to the same Prometheus instance. If this option is omitted, a hash of the `baseUrl` or `zkHost` will be used as ID by default.
+
The metrics exposed by `solr-exporter` can be seen at the metrics endpoint: `\http://localhost:8983/solr/admin/metrics`.
=== Environment Variable Options