You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2021/09/08 16:22:20 UTC
[lucene-solr] 02/02: SOLR-15572: Improve the default Prometheus
exporter config performance.
This is an automated email from the ASF dual-hosted git repository.
ab pushed a commit to branch branch_8_10
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 965b1ee759bb7cf03ccc6d58669ed4abf2bdce68
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Wed Sep 8 12:34:46 2021 +0200
SOLR-15572: Improve the default Prometheus exporter config performance.
---
solr/CHANGES.txt | 2 +
.../conf/solr-exporter-config.xml | 54 ++++++++++++++++++++--
.../solr/prometheus/exporter/MetricsQuery.java | 15 ++++--
.../solr/prometheus/scraper/SolrCloudScraper.java | 2 +-
...theus-solr-exporter-integration-test-config.xml | 17 +++++--
...rometheus-solr-exporter-scraper-test-config.xml | 15 +++++-
.../exporter/SolrExporterIntegrationTest.java | 2 +-
7 files changed, 91 insertions(+), 16 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 9827c0a..cb13863 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -62,6 +62,8 @@ Improvements
* SOLR-15607: Explicitly indent results in Admin Query UI by default (Olvia Falk via Eric Pugh)
+* SOLR-15572: Improve the default Prometheus exporter config performance. (ab)
+
Optimizations
---------------------
* SOLR-15433: Replace transient core cache LRU by Caffeine cache. (Bruno Roustant)
diff --git a/solr/contrib/prometheus-exporter/conf/solr-exporter-config.xml b/solr/contrib/prometheus-exporter/conf/solr-exporter-config.xml
index e2e6bfe..d111689 100644
--- a/solr/contrib/prometheus-exporter/conf/solr-exporter-config.xml
+++ b/solr/contrib/prometheus-exporter/conf/solr-exporter-config.xml
@@ -263,10 +263,56 @@
<lst name="query">
<str name="path">/admin/metrics</str>
<lst name="params">
- <str name="group">all</str>
- <str name="type">all</str>
- <str name="prefix"></str>
- <str name="property"></str>
+ <!--
+ trim some of these expressions as needed if you don't care about
+ a particular group of metrics.
+ -->
+ <str name="expr">solr\.jetty:.*DefaultHandler.*</str>
+ <str name="expr">solr\.jvm:.*</str>
+ <str name="expr">solr\.node:.*</str>
+ <str name="expr">solr\.overseer:.*</str>
+ <str name="expr">solr\.core\..*:QUERY\..*</str>
+ <str name="expr">solr\.core\..*:ADMIN\..*</str>
+ <str name="expr">solr\.core\..*:CACHE\..*</str>
+ <str name="expr">solr\.core\..*:UPDATE\.updateHandler\..*</str>
+ <str name="expr">solr\.core\..*:CORE\.fs\..*</str>
+ <str name="expr">solr\.core\..*:HIGHLIGHTER\..*</str>
+ <str name="expr">solr\.core\..*:INDEX\..*</str>
+ <str name="expr">solr\.core\..*:REPLICATION\.replication\..*</str>
+ <str name="expr">solr\.core\..*:SEARCHER\.searcher\..*</str>
+
+ <!-- Alternative expressions, which are much stricter but still provide
+ enough data to populate the default dashboard.
+ These expressions omit many unused properties of the complex metrics,
+ and also skip whole groups of rarely used metrics: core ADMIN, REPLICATION,
+ HIGHLIGHTER, and selects only the most common QUERY handlers.
+
+ In order to use these expressions remove the default list of expressions
+ above and the START / END lines below. -->
+
+ <!-- === START ===
+
+ <str name="expr">solr\.jetty:.*\.DefaultHandler\.(dispatches|.*-requests|.*xx-responses):count</str>
+
+ <str name="expr">solr\.jvm:(buffers|gc).*</str>
+ <str name="expr">solr\.jvm:memory\.(heap|non-heap|pools)\.*\.usage</str>
+ <str name="expr">solr\.jvm:memory\.total</str>
+ <str name="expr">solr\.jvm:os\..*(FileDescriptorCount|Load.*|Size|processCpuTime)</str>
+ <str name="expr">solr\.jvm:threads\..*count</str>
+
+ <str name="expr">solr\.node:CONTAINER\.(cores|fs).*</str>
+
+ <str name="expr">solr\.core\..*:CORE\.fs\..*Space</str>
+ <str name="expr">solr\.core\..*:INDEX\.sizeInBytes</str>
+ <str name="expr">solr\.core\..*:QUERY\./(select|get|export|stream|query|graph|sql)\..*requestTimes:(count|1minRate|5minRate|median_ms|meanRate|p75_ms|p95_ms|p99_ms)</str>
+ <str name="expr">solr\.core\..*:QUERY\./(select|get|export|stream|query|graph|sql)\.totalTime</str>
+ <str name="expr">solr\.core\..*:QUERY\./(select|get|export|stream|query|graph|sql)\..*rrors:(count!1minRate)</str>
+ <str name="expr">solr\.core\..*:SEARCHER\.searcher\..*Doc.*</str>
+ <str name="expr">solr\.core\..*:UPDATE\.updateHandler\..*</str>
+ <str name="expr">solr\core\..*:CACHE\..*</str>
+
+ === END === -->
+
</lst>
</lst>
<arr name="jsonQueries">
diff --git a/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/MetricsQuery.java b/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/MetricsQuery.java
index de9d9c5..65fce1e 100644
--- a/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/MetricsQuery.java
+++ b/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/MetricsQuery.java
@@ -18,10 +18,10 @@
package org.apache.solr.prometheus.exporter;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
import java.util.regex.Matcher;
import net.thisptr.jackson.jq.JsonQuery;
@@ -98,7 +98,7 @@ public class MetricsQuery {
for (NamedList request : requests) {
NamedList query = (NamedList) request.get("query");
- NamedList queryParameters = (NamedList) query.get("params");
+ NamedList<Object> queryParameters = (NamedList<Object>) query.get("params");
String path = (String) query.get("path");
String core = (String) query.get("core");
String collection = (String) query.get("collection");
@@ -106,8 +106,15 @@ public class MetricsQuery {
ModifiableSolrParams params = new ModifiableSolrParams();
if (queryParameters != null) {
- for (Map.Entry<String, String> entrySet : (Set<Map.Entry<String, String>>) queryParameters.asShallowMap().entrySet()) {
- params.add(entrySet.getKey(), entrySet.getValue());
+ for (Map.Entry<String, Object> entrySet : queryParameters.asShallowMap().entrySet()) {
+ if (entrySet.getValue() instanceof Collection) {
+ Collection<Object> values = (Collection<Object>) entrySet.getValue();
+ for (Object value : values) {
+ params.add(entrySet.getKey(), String.valueOf(value));
+ }
+ } else {
+ params.add(entrySet.getKey(), String.valueOf(entrySet.getValue()));
+ }
}
}
diff --git a/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java b/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java
index e4b98e7..8455a47 100644
--- a/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java
+++ b/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java
@@ -138,7 +138,7 @@ public class SolrCloudScraper extends SolrScraper {
}
private Set<String> getCollections() throws IOException {
- return solrClient.getClusterStateProvider().getClusterState().getCollectionsMap().keySet();
+ return solrClient.getClusterStateProvider().getClusterState().getCollectionStates().keySet();
}
@Override
diff --git a/solr/contrib/prometheus-exporter/src/test-files/conf/prometheus-solr-exporter-integration-test-config.xml b/solr/contrib/prometheus-exporter/src/test-files/conf/prometheus-solr-exporter-integration-test-config.xml
index 6b306c9..988a76d 100644
--- a/solr/contrib/prometheus-exporter/src/test-files/conf/prometheus-solr-exporter-integration-test-config.xml
+++ b/solr/contrib/prometheus-exporter/src/test-files/conf/prometheus-solr-exporter-integration-test-config.xml
@@ -92,10 +92,19 @@
<lst name="query">
<str name="path">/admin/metrics</str>
<lst name="params">
- <str name="group">all</str>
- <str name="type">all</str>
- <str name="prefix"></str>
- <str name="property"></str>
+ <str name="expr">solr\.jetty:.*DefaultHandler.*</str>
+ <str name="expr">solr\.jvm:.*</str>
+ <str name="expr">solr\.node:.*</str>
+ <str name="expr">solr\.overseer:.*</str>
+ <str name="expr">solr\.core\..*:QUERY\..*</str>
+ <str name="expr">solr\.core\..*:ADMIN\..*</str>
+ <str name="expr">solr\.core\..*:CACHE\..*</str>
+ <str name="expr">solr\.core\..*:UPDATE\.updateHandler\..*</str>
+ <str name="expr">solr\.core\..*:CORE\.fs\..*</str>
+ <str name="expr">solr\.core\..*:HIGHLIGHTER\..*</str>
+ <str name="expr">solr\.core\..*:INDEX\..*</str>
+ <str name="expr">solr\.core\..*:REPLICATION\.replication\..*</str>
+ <str name="expr">solr\.core\..*:SEARCHER\.searcher\..*</str>
</lst>
</lst>
<arr name="jsonQueries">
diff --git a/solr/contrib/prometheus-exporter/src/test-files/conf/prometheus-solr-exporter-scraper-test-config.xml b/solr/contrib/prometheus-exporter/src/test-files/conf/prometheus-solr-exporter-scraper-test-config.xml
index 7da8eff..de10dfe 100644
--- a/solr/contrib/prometheus-exporter/src/test-files/conf/prometheus-solr-exporter-scraper-test-config.xml
+++ b/solr/contrib/prometheus-exporter/src/test-files/conf/prometheus-solr-exporter-scraper-test-config.xml
@@ -94,8 +94,19 @@
<lst name="query">
<str name="path">/admin/metrics</str>
<lst name="params">
- <str name="group">all</str>
- <str name="type">all</str>
+ <str name="expr">solr\.jetty:.*DefaultHandler.*</str>
+ <str name="expr">solr\.jvm:.*</str>
+ <str name="expr">solr\.node:.*</str>
+ <str name="expr">solr\.overseer:.*</str>
+ <str name="expr">solr\.core\..*:QUERY\..*</str>
+ <str name="expr">solr\.core\..*:ADMIN\..*</str>
+ <str name="expr">solr\.core\..*:CACHE\..*</str>
+ <str name="expr">solr\.core\..*:UPDATE\.updateHandler\..*</str>
+ <str name="expr">solr\.core\..*:CORE\.fs\..*</str>
+ <str name="expr">solr\.core\..*:HIGHLIGHTER\..*</str>
+ <str name="expr">solr\.core\..*:INDEX\..*</str>
+ <str name="expr">solr\.core\..*:REPLICATION\.replication\..*</str>
+ <str name="expr">solr\.core\..*:SEARCHER\.searcher\..*</str>
</lst>
</lst>
<arr name="jsonQueries">
diff --git a/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/exporter/SolrExporterIntegrationTest.java b/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/exporter/SolrExporterIntegrationTest.java
index ceb4a4e..fa65ec0 100644
--- a/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/exporter/SolrExporterIntegrationTest.java
+++ b/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/exporter/SolrExporterIntegrationTest.java
@@ -23,7 +23,7 @@ import org.apache.lucene.util.LuceneTestCase.Slow;
import org.junit.Before;
import org.junit.Test;
-@org.apache.lucene.util.LuceneTestCase.AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13786")
+//@org.apache.lucene.util.LuceneTestCase.AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13786")
@Slow
public class SolrExporterIntegrationTest extends SolrExporterTestBase {