You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2021/09/08 16:22:18 UTC
[lucene-solr] branch branch_8_10 updated (ec297b4 -> 965b1ee)
This is an automated email from the ASF dual-hosted git repository.
ab pushed a change to branch branch_8_10
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from ec297b4 SOLR-15610: mention JsonTemplateLayout as a log4j option in ref-guide
new 55170e8 SOLR-14506: Fix NPE in COLSTATUS for collections using implicit routing.
new 965b1ee SOLR-15572: Improve the default Prometheus exporter config performance.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
solr/CHANGES.txt | 4 ++
.../conf/solr-exporter-config.xml | 54 ++++++++++++++++++++--
.../solr/prometheus/exporter/MetricsQuery.java | 15 ++++--
.../solr/prometheus/scraper/SolrCloudScraper.java | 2 +-
...theus-solr-exporter-integration-test-config.xml | 17 +++++--
...rometheus-solr-exporter-scraper-test-config.xml | 15 +++++-
.../exporter/SolrExporterIntegrationTest.java | 2 +-
.../org/apache/solr/handler/admin/ColStatus.java | 9 +++-
.../apache/solr/cloud/CollectionsAPISolrJTest.java | 14 ++++++
9 files changed, 114 insertions(+), 18 deletions(-)
[lucene-solr] 02/02: SOLR-15572: Improve the default Prometheus
exporter config performance.
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ab pushed a commit to branch branch_8_10
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 965b1ee759bb7cf03ccc6d58669ed4abf2bdce68
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Wed Sep 8 12:34:46 2021 +0200
SOLR-15572: Improve the default Prometheus exporter config performance.
---
solr/CHANGES.txt | 2 +
.../conf/solr-exporter-config.xml | 54 ++++++++++++++++++++--
.../solr/prometheus/exporter/MetricsQuery.java | 15 ++++--
.../solr/prometheus/scraper/SolrCloudScraper.java | 2 +-
...theus-solr-exporter-integration-test-config.xml | 17 +++++--
...rometheus-solr-exporter-scraper-test-config.xml | 15 +++++-
.../exporter/SolrExporterIntegrationTest.java | 2 +-
7 files changed, 91 insertions(+), 16 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 9827c0a..cb13863 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -62,6 +62,8 @@ Improvements
* SOLR-15607: Explicitly indent results in Admin Query UI by default (Olvia Falk via Eric Pugh)
+* SOLR-15572: Improve the default Prometheus exporter config performance. (ab)
+
Optimizations
---------------------
* SOLR-15433: Replace transient core cache LRU by Caffeine cache. (Bruno Roustant)
diff --git a/solr/contrib/prometheus-exporter/conf/solr-exporter-config.xml b/solr/contrib/prometheus-exporter/conf/solr-exporter-config.xml
index e2e6bfe..d111689 100644
--- a/solr/contrib/prometheus-exporter/conf/solr-exporter-config.xml
+++ b/solr/contrib/prometheus-exporter/conf/solr-exporter-config.xml
@@ -263,10 +263,56 @@
<lst name="query">
<str name="path">/admin/metrics</str>
<lst name="params">
- <str name="group">all</str>
- <str name="type">all</str>
- <str name="prefix"></str>
- <str name="property"></str>
+ <!--
+ trim some of these expressions as needed if you don't care about
+ a particular group of metrics.
+ -->
+ <str name="expr">solr\.jetty:.*DefaultHandler.*</str>
+ <str name="expr">solr\.jvm:.*</str>
+ <str name="expr">solr\.node:.*</str>
+ <str name="expr">solr\.overseer:.*</str>
+ <str name="expr">solr\.core\..*:QUERY\..*</str>
+ <str name="expr">solr\.core\..*:ADMIN\..*</str>
+ <str name="expr">solr\.core\..*:CACHE\..*</str>
+ <str name="expr">solr\.core\..*:UPDATE\.updateHandler\..*</str>
+ <str name="expr">solr\.core\..*:CORE\.fs\..*</str>
+ <str name="expr">solr\.core\..*:HIGHLIGHTER\..*</str>
+ <str name="expr">solr\.core\..*:INDEX\..*</str>
+ <str name="expr">solr\.core\..*:REPLICATION\.replication\..*</str>
+ <str name="expr">solr\.core\..*:SEARCHER\.searcher\..*</str>
+
+ <!-- Alternative expressions, which are much stricter but still provide
+ enough data to populate the default dashboard.
+ These expressions omit many unused properties of the complex metrics,
+ and also skip whole groups of rarely used metrics: core ADMIN, REPLICATION,
+ HIGHLIGHTER, and selects only the most common QUERY handlers.
+
+ In order to use these expressions remove the default list of expressions
+ above and the START / END lines below. -->
+
+ <!-- === START ===
+
+ <str name="expr">solr\.jetty:.*\.DefaultHandler\.(dispatches|.*-requests|.*xx-responses):count</str>
+
+ <str name="expr">solr\.jvm:(buffers|gc).*</str>
+ <str name="expr">solr\.jvm:memory\.(heap|non-heap|pools)\.*\.usage</str>
+ <str name="expr">solr\.jvm:memory\.total</str>
+ <str name="expr">solr\.jvm:os\..*(FileDescriptorCount|Load.*|Size|processCpuTime)</str>
+ <str name="expr">solr\.jvm:threads\..*count</str>
+
+ <str name="expr">solr\.node:CONTAINER\.(cores|fs).*</str>
+
+ <str name="expr">solr\.core\..*:CORE\.fs\..*Space</str>
+ <str name="expr">solr\.core\..*:INDEX\.sizeInBytes</str>
+ <str name="expr">solr\.core\..*:QUERY\./(select|get|export|stream|query|graph|sql)\..*requestTimes:(count|1minRate|5minRate|median_ms|meanRate|p75_ms|p95_ms|p99_ms)</str>
+ <str name="expr">solr\.core\..*:QUERY\./(select|get|export|stream|query|graph|sql)\.totalTime</str>
+ <str name="expr">solr\.core\..*:QUERY\./(select|get|export|stream|query|graph|sql)\..*rrors:(count!1minRate)</str>
+ <str name="expr">solr\.core\..*:SEARCHER\.searcher\..*Doc.*</str>
+ <str name="expr">solr\.core\..*:UPDATE\.updateHandler\..*</str>
+ <str name="expr">solr\core\..*:CACHE\..*</str>
+
+ === END === -->
+
</lst>
</lst>
<arr name="jsonQueries">
diff --git a/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/MetricsQuery.java b/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/MetricsQuery.java
index de9d9c5..65fce1e 100644
--- a/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/MetricsQuery.java
+++ b/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/MetricsQuery.java
@@ -18,10 +18,10 @@
package org.apache.solr.prometheus.exporter;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
import java.util.regex.Matcher;
import net.thisptr.jackson.jq.JsonQuery;
@@ -98,7 +98,7 @@ public class MetricsQuery {
for (NamedList request : requests) {
NamedList query = (NamedList) request.get("query");
- NamedList queryParameters = (NamedList) query.get("params");
+ NamedList<Object> queryParameters = (NamedList<Object>) query.get("params");
String path = (String) query.get("path");
String core = (String) query.get("core");
String collection = (String) query.get("collection");
@@ -106,8 +106,15 @@ public class MetricsQuery {
ModifiableSolrParams params = new ModifiableSolrParams();
if (queryParameters != null) {
- for (Map.Entry<String, String> entrySet : (Set<Map.Entry<String, String>>) queryParameters.asShallowMap().entrySet()) {
- params.add(entrySet.getKey(), entrySet.getValue());
+ for (Map.Entry<String, Object> entrySet : queryParameters.asShallowMap().entrySet()) {
+ if (entrySet.getValue() instanceof Collection) {
+ Collection<Object> values = (Collection<Object>) entrySet.getValue();
+ for (Object value : values) {
+ params.add(entrySet.getKey(), String.valueOf(value));
+ }
+ } else {
+ params.add(entrySet.getKey(), String.valueOf(entrySet.getValue()));
+ }
}
}
diff --git a/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java b/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java
index e4b98e7..8455a47 100644
--- a/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java
+++ b/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java
@@ -138,7 +138,7 @@ public class SolrCloudScraper extends SolrScraper {
}
private Set<String> getCollections() throws IOException {
- return solrClient.getClusterStateProvider().getClusterState().getCollectionsMap().keySet();
+ return solrClient.getClusterStateProvider().getClusterState().getCollectionStates().keySet();
}
@Override
diff --git a/solr/contrib/prometheus-exporter/src/test-files/conf/prometheus-solr-exporter-integration-test-config.xml b/solr/contrib/prometheus-exporter/src/test-files/conf/prometheus-solr-exporter-integration-test-config.xml
index 6b306c9..988a76d 100644
--- a/solr/contrib/prometheus-exporter/src/test-files/conf/prometheus-solr-exporter-integration-test-config.xml
+++ b/solr/contrib/prometheus-exporter/src/test-files/conf/prometheus-solr-exporter-integration-test-config.xml
@@ -92,10 +92,19 @@
<lst name="query">
<str name="path">/admin/metrics</str>
<lst name="params">
- <str name="group">all</str>
- <str name="type">all</str>
- <str name="prefix"></str>
- <str name="property"></str>
+ <str name="expr">solr\.jetty:.*DefaultHandler.*</str>
+ <str name="expr">solr\.jvm:.*</str>
+ <str name="expr">solr\.node:.*</str>
+ <str name="expr">solr\.overseer:.*</str>
+ <str name="expr">solr\.core\..*:QUERY\..*</str>
+ <str name="expr">solr\.core\..*:ADMIN\..*</str>
+ <str name="expr">solr\.core\..*:CACHE\..*</str>
+ <str name="expr">solr\.core\..*:UPDATE\.updateHandler\..*</str>
+ <str name="expr">solr\.core\..*:CORE\.fs\..*</str>
+ <str name="expr">solr\.core\..*:HIGHLIGHTER\..*</str>
+ <str name="expr">solr\.core\..*:INDEX\..*</str>
+ <str name="expr">solr\.core\..*:REPLICATION\.replication\..*</str>
+ <str name="expr">solr\.core\..*:SEARCHER\.searcher\..*</str>
</lst>
</lst>
<arr name="jsonQueries">
diff --git a/solr/contrib/prometheus-exporter/src/test-files/conf/prometheus-solr-exporter-scraper-test-config.xml b/solr/contrib/prometheus-exporter/src/test-files/conf/prometheus-solr-exporter-scraper-test-config.xml
index 7da8eff..de10dfe 100644
--- a/solr/contrib/prometheus-exporter/src/test-files/conf/prometheus-solr-exporter-scraper-test-config.xml
+++ b/solr/contrib/prometheus-exporter/src/test-files/conf/prometheus-solr-exporter-scraper-test-config.xml
@@ -94,8 +94,19 @@
<lst name="query">
<str name="path">/admin/metrics</str>
<lst name="params">
- <str name="group">all</str>
- <str name="type">all</str>
+ <str name="expr">solr\.jetty:.*DefaultHandler.*</str>
+ <str name="expr">solr\.jvm:.*</str>
+ <str name="expr">solr\.node:.*</str>
+ <str name="expr">solr\.overseer:.*</str>
+ <str name="expr">solr\.core\..*:QUERY\..*</str>
+ <str name="expr">solr\.core\..*:ADMIN\..*</str>
+ <str name="expr">solr\.core\..*:CACHE\..*</str>
+ <str name="expr">solr\.core\..*:UPDATE\.updateHandler\..*</str>
+ <str name="expr">solr\.core\..*:CORE\.fs\..*</str>
+ <str name="expr">solr\.core\..*:HIGHLIGHTER\..*</str>
+ <str name="expr">solr\.core\..*:INDEX\..*</str>
+ <str name="expr">solr\.core\..*:REPLICATION\.replication\..*</str>
+ <str name="expr">solr\.core\..*:SEARCHER\.searcher\..*</str>
</lst>
</lst>
<arr name="jsonQueries">
diff --git a/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/exporter/SolrExporterIntegrationTest.java b/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/exporter/SolrExporterIntegrationTest.java
index ceb4a4e..fa65ec0 100644
--- a/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/exporter/SolrExporterIntegrationTest.java
+++ b/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/exporter/SolrExporterIntegrationTest.java
@@ -23,7 +23,7 @@ import org.apache.lucene.util.LuceneTestCase.Slow;
import org.junit.Before;
import org.junit.Test;
-@org.apache.lucene.util.LuceneTestCase.AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13786")
+//@org.apache.lucene.util.LuceneTestCase.AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13786")
@Slow
public class SolrExporterIntegrationTest extends SolrExporterTestBase {
[lucene-solr] 01/02: SOLR-14506: Fix NPE in COLSTATUS for
collections using implicit routing.
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ab pushed a commit to branch branch_8_10
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 55170e8d8d4c53479b52ddecda20f6963a14c690
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Tue Aug 3 17:35:49 2021 +0200
SOLR-14506: Fix NPE in COLSTATUS for collections using implicit routing.
---
solr/CHANGES.txt | 2 ++
.../src/java/org/apache/solr/handler/admin/ColStatus.java | 9 +++++++--
.../org/apache/solr/cloud/CollectionsAPISolrJTest.java | 14 ++++++++++++++
3 files changed, 23 insertions(+), 2 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 53d2197..9827c0a 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -99,6 +99,8 @@ Bug Fixes
* SOLR-15595: Partial results from shard queries needlessly discarded for queries without sort field (Michael Kosten via Mike Drob, Eric Pugh)
+* SOLR-14506: Fix NPE in COLSTATUS for collections using implicit routing. (Austin Weidler, ab)
+
Other Changes
---------------------
* SOLR-15355: Upgrade Hadoop from 3.2.0 to 3.2.2. (Antoine Gruzelle via David Smiley)
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ColStatus.java b/solr/core/src/java/org/apache/solr/handler/admin/ColStatus.java
index 3f9cba3..271315f 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/ColStatus.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/ColStatus.java
@@ -76,7 +76,7 @@ public class ColStatus {
Collection<String> collections;
String col = props.getStr(ZkStateReader.COLLECTION_PROP);
if (col == null) {
- collections = new HashSet<>(clusterState.getCollectionsMap().keySet());
+ collections = new HashSet<>(clusterState.getCollectionStates().keySet());
} else {
collections = Collections.singleton(col);
}
@@ -149,7 +149,9 @@ public class ColStatus {
replicaMap.add("recovering", recoveringReplicas);
replicaMap.add("recovery_failed", recoveryFailedReplicas);
sliceMap.add("state", s.getState().toString());
- sliceMap.add("range", s.getRange().toString());
+ if (s.getRange() != null) {
+ sliceMap.add("range", s.getRange().toString());
+ }
Map<String, RoutingRule> rules = s.getRoutingRules();
if (rules != null && !rules.isEmpty()) {
sliceMap.add("routingRules", rules);
@@ -170,6 +172,9 @@ public class ColStatus {
continue;
}
String url = ZkCoreNodeProps.getCoreUrl(leader);
+ if (url == null) {
+ continue;
+ }
try (SolrClient client = solrClientCache.getHttpSolrClient(url)) {
ModifiableSolrParams params = new ModifiableSolrParams();
params.add(CommonParams.QT, "/admin/segments");
diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
index f2b6e59..8ead97a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
@@ -686,6 +686,20 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
assertEquals(0, rsp.getStatus());
Number down = (Number) rsp.getResponse().findRecursive(collectionName, "shards", "shard1", "replicas", "down");
assertTrue("should be some down replicas, but there were none in shard1:" + rsp, down.intValue() > 0);
+
+ // test for a collection with implicit router
+ String implicitColl = "implicitColl";
+ CollectionAdminRequest.createCollection(implicitColl, "conf2", 2, 1)
+ .setRouterName("implicit")
+ .setRouterField("routerField")
+ .setShards("shardA,shardB")
+ .process(cluster.getSolrClient());
+ cluster.waitForActiveCollection(implicitColl, 2, 2);
+ req = CollectionAdminRequest.collectionStatus(implicitColl);
+ rsp = req.process(cluster.getSolrClient());
+ assertNotNull(rsp.getResponse().get(implicitColl));
+ assertNotNull(rsp.toString(), rsp.getResponse().findRecursive(implicitColl, "shards", "shardA"));
+ assertNotNull(rsp.toString(), rsp.getResponse().findRecursive(implicitColl, "shards", "shardB"));
}
@Test