You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ho...@apache.org on 2022/09/23 16:50:35 UTC
[solr] branch main updated: SOLR-16404: Use Http2 clients in the Prometheus Exporter (#1004)
This is an automated email from the ASF dual-hosted git repository.
houston pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 9d6d88c8a37 SOLR-16404: Use Http2 clients in the Prometheus Exporter (#1004)
9d6d88c8a37 is described below
commit 9d6d88c8a37ac832f2ba2e30c644dd0a5d9ad1f3
Author: Houston Putman <ho...@apache.org>
AuthorDate: Fri Sep 23 12:50:28 2022 -0400
SOLR-16404: Use Http2 clients in the Prometheus Exporter (#1004)
---
solr/CHANGES.txt | 2 +
solr/prometheus-exporter/build.gradle | 1 +
.../prometheus/exporter/SolrClientFactory.java | 55 +++++++++-------------
.../solr/prometheus/scraper/SolrCloudScraper.java | 14 +++---
.../solr/prometheus/scraper/SolrScraper.java | 12 ++---
.../prometheus/scraper/SolrStandaloneScraper.java | 6 +--
.../prometheus/scraper/SolrCloudScraperTest.java | 4 +-
.../scraper/SolrStandaloneScraperTest.java | 6 +--
8 files changed, 46 insertions(+), 54 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index f76e667420d..2a4541227f1 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -89,6 +89,8 @@ Improvements
* SOLR-16361: mod() is now accurate for all integers, floats, doubles and longs upto 2^52 (Dan Rosher via Eric Pugh)
+* SOLR-16404: Prometheus Exporter: Use HTTP2 Solr clients. (Houston Putman)
+
Optimizations
---------------------
* SOLR-16120: Optimise hl.fl expansion. (Christine Poerschke, David Smiley, Mike Drob)
diff --git a/solr/prometheus-exporter/build.gradle b/solr/prometheus-exporter/build.gradle
index 1d6e69c3529..613318fa579 100644
--- a/solr/prometheus-exporter/build.gradle
+++ b/solr/prometheus-exporter/build.gradle
@@ -22,6 +22,7 @@ description = 'Prometheus exporter for exposing metrics from Solr using Metrics
dependencies {
implementation project(':solr:solrj')
+ runtimeOnly project(':solr:solrj-zookeeper')
// ideally remove ZK dep
implementation('org.apache.zookeeper:zookeeper', {
exclude group: "org.apache.yetus", module: "audience-annotations"
diff --git a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrClientFactory.java b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrClientFactory.java
index 877058b2be6..f879dc3b0e7 100644
--- a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrClientFactory.java
+++ b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrClientFactory.java
@@ -19,9 +19,9 @@ package org.apache.solr.prometheus.exporter;
import java.util.Optional;
import java.util.stream.Collectors;
-import org.apache.solr.client.solrj.impl.CloudLegacySolrClient;
+import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.impl.NoOpResponseParser;
import org.apache.zookeeper.client.ConnectStringParser;
@@ -33,43 +33,34 @@ public class SolrClientFactory {
this.settings = settings;
}
- public HttpSolrClient createStandaloneSolrClient(String solrHost) {
- NoOpResponseParser responseParser = new NoOpResponseParser();
- responseParser.setWriterType("json");
+ public Http2SolrClient createStandaloneSolrClient(String solrHost) {
+ Http2SolrClient http2SolrClient =
+ new Http2SolrClient.Builder(solrHost)
+ .idleTimeout(settings.getHttpReadTimeout())
+ .connectionTimeout(settings.getHttpConnectionTimeout())
+ .build();
- HttpSolrClient.Builder standaloneBuilder = new HttpSolrClient.Builder();
+ http2SolrClient.setParser(new NoOpResponseParser("json"));
- standaloneBuilder.withBaseSolrUrl(solrHost);
-
- standaloneBuilder
- .withConnectionTimeout(settings.getHttpConnectionTimeout())
- .withSocketTimeout(settings.getHttpReadTimeout());
-
- HttpSolrClient httpSolrClient = standaloneBuilder.build();
- httpSolrClient.setParser(responseParser);
-
- return httpSolrClient;
+ return http2SolrClient;
}
public CloudSolrClient createCloudSolrClient(String zookeeperConnectionString) {
- NoOpResponseParser responseParser = new NoOpResponseParser();
- responseParser.setWriterType("json");
-
ConnectStringParser parser = new ConnectStringParser(zookeeperConnectionString);
- var cloudBuilder =
- new CloudLegacySolrClient.Builder(
- parser.getServerAddresses().stream()
- .map(address -> address.getHostString() + ":" + address.getPort())
- .collect(Collectors.toList()),
- Optional.ofNullable(parser.getChrootPath()));
-
- cloudBuilder
- .withConnectionTimeout(settings.getHttpConnectionTimeout())
- .withSocketTimeout(settings.getHttpReadTimeout());
-
- CloudSolrClient client = cloudBuilder.build();
- client.setParser(responseParser);
+ CloudSolrClient client =
+ new CloudHttp2SolrClient.Builder(
+ parser.getServerAddresses().stream()
+ .map(address -> address.getHostString() + ":" + address.getPort())
+ .collect(Collectors.toList()),
+ Optional.ofNullable(parser.getChrootPath()))
+ .withInternalClientBuilder(
+ new Http2SolrClient.Builder()
+ .idleTimeout(settings.getHttpReadTimeout())
+ .connectionTimeout(settings.getHttpConnectionTimeout()))
+ .build();
+
+ client.setParser(new NoOpResponseParser("json"));
client.connect();
diff --git a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java
index 3944dc042a9..55ded335c1d 100644
--- a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java
+++ b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java
@@ -26,7 +26,7 @@ import java.util.concurrent.ExecutorService;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.util.IOUtils;
@@ -40,7 +40,7 @@ public class SolrCloudScraper extends SolrScraper {
private final CloudSolrClient solrClient;
private final SolrClientFactory solrClientFactory;
- private Cache<String, HttpSolrClient> hostClientCache = Caffeine.newBuilder().build();
+ private Cache<String, Http2SolrClient> hostClientCache = Caffeine.newBuilder().build();
public SolrCloudScraper(
CloudSolrClient solrClient,
@@ -54,7 +54,7 @@ public class SolrCloudScraper extends SolrScraper {
@Override
public Map<String, MetricSamples> pingAllCores(MetricsQuery query) throws IOException {
- Map<String, HttpSolrClient> httpSolrClients = createHttpSolrClients();
+ Map<String, Http2SolrClient> httpSolrClients = createHttpSolrClients();
Map<String, DocCollection> collectionState = solrClient.getClusterState().getCollectionsMap();
@@ -67,7 +67,7 @@ public class SolrCloudScraper extends SolrScraper {
List<String> coreNames =
replicas.stream().map(Replica::getCoreName).collect(Collectors.toList());
- Map<String, HttpSolrClient> coreToClient =
+ Map<String, Http2SolrClient> coreToClient =
replicas.stream()
.map(
replica ->
@@ -85,10 +85,10 @@ public class SolrCloudScraper extends SolrScraper {
});
}
- private Map<String, HttpSolrClient> createHttpSolrClients() throws IOException {
+ private Map<String, Http2SolrClient> createHttpSolrClients() throws IOException {
return getBaseUrls().stream()
.map(url -> hostClientCache.get(url, solrClientFactory::createStandaloneSolrClient))
- .collect(Collectors.toMap(HttpSolrClient::getBaseURL, Function.identity()));
+ .collect(Collectors.toMap(Http2SolrClient::getBaseURL, Function.identity()));
}
@Override
@@ -106,7 +106,7 @@ public class SolrCloudScraper extends SolrScraper {
@Override
public Map<String, MetricSamples> metricsForAllHosts(MetricsQuery query) throws IOException {
- Map<String, HttpSolrClient> httpSolrClients = createHttpSolrClients();
+ Map<String, Http2SolrClient> httpSolrClients = createHttpSolrClients();
return sendRequestsInParallel(
httpSolrClients.keySet(),
diff --git a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrScraper.java b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrScraper.java
index 270abf76222..a65de0de431 100644
--- a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrScraper.java
+++ b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrScraper.java
@@ -37,7 +37,7 @@ import net.thisptr.jackson.jq.exception.JsonQueryException;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.prometheus.collector.MetricSamples;
@@ -124,8 +124,8 @@ public abstract class SolrScraper implements Closeable {
String baseUrlLabelValue = "";
String zkHostLabelValue = "";
- if (client instanceof HttpSolrClient) {
- baseUrlLabelValue = ((HttpSolrClient) client).getBaseURL();
+ if (client instanceof Http2SolrClient) {
+ baseUrlLabelValue = ((Http2SolrClient) client).getBaseURL();
} else if (client instanceof CloudSolrClient) {
zkHostLabelValue = ((CloudSolrClient) client).getClusterStateProvider().getQuorumHosts();
}
@@ -170,12 +170,10 @@ public abstract class SolrScraper implements Closeable {
}
/* Labels due to client */
- if (client instanceof HttpSolrClient) {
+ if (!baseUrlLabelValue.isEmpty()) {
labelNames.add(BASE_URL_LABEL);
labelValues.add(baseUrlLabelValue);
- }
-
- if (client instanceof CloudSolrClient) {
+ } else if (!zkHostLabelValue.isEmpty()) {
labelNames.add(ZK_HOST_LABEL);
labelValues.add(zkHostLabelValue);
}
diff --git a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrStandaloneScraper.java b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrStandaloneScraper.java
index 90139e968f4..734279653f8 100644
--- a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrStandaloneScraper.java
+++ b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrStandaloneScraper.java
@@ -25,7 +25,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.util.IOUtils;
@@ -35,10 +35,10 @@ import org.apache.solr.prometheus.exporter.MetricsQuery;
public class SolrStandaloneScraper extends SolrScraper {
- private final HttpSolrClient solrClient;
+ private final Http2SolrClient solrClient;
public SolrStandaloneScraper(
- HttpSolrClient solrClient, ExecutorService executor, String clusterId) {
+ Http2SolrClient solrClient, ExecutorService executor, String clusterId) {
super(executor, clusterId);
this.solrClient = solrClient;
}
diff --git a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrCloudScraperTest.java b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrCloudScraperTest.java
index dffb6bef17a..a5454296caf 100644
--- a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrCloudScraperTest.java
+++ b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrCloudScraperTest.java
@@ -28,7 +28,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.stream.Collectors;
-import org.apache.solr.client.solrj.impl.CloudLegacySolrClient;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.NoOpResponseParser;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
@@ -56,7 +56,7 @@ public class SolrCloudScraperTest extends PrometheusExporterTestBase {
private SolrCloudScraper createSolrCloudScraper() {
var solrClient =
- new CloudLegacySolrClient.Builder(
+ new CloudSolrClient.Builder(
Collections.singletonList(cluster.getZkServer().getZkAddress()), Optional.empty())
.build();
diff --git a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrStandaloneScraperTest.java b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrStandaloneScraperTest.java
index 276e32fea77..c2d052c2a29 100644
--- a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrStandaloneScraperTest.java
+++ b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrStandaloneScraperTest.java
@@ -24,7 +24,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import org.apache.commons.io.FileUtils;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.impl.NoOpResponseParser;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.IOUtils;
@@ -43,7 +43,7 @@ public class SolrStandaloneScraperTest extends RestTestBase {
private static MetricsConfiguration configuration;
private static SolrStandaloneScraper solrScraper;
private static ExecutorService executor;
- private static HttpSolrClient solrClient;
+ private static Http2SolrClient solrClient;
@BeforeClass
public static void setupBeforeClass() throws Exception {
@@ -63,7 +63,7 @@ public class SolrStandaloneScraperTest extends RestTestBase {
configuration =
Helpers.loadConfiguration("conf/prometheus-solr-exporter-scraper-test-config.xml");
- solrClient = getHttpSolrClient(restTestHarness.getAdminURL());
+ solrClient = new Http2SolrClient.Builder(restTestHarness.getAdminURL()).build();
solrScraper = new SolrStandaloneScraper(solrClient, executor, "test");
NoOpResponseParser responseParser = new NoOpResponseParser();