You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/09/21 09:27:59 UTC
lucene-solr:jira/http2: Introduce maxConnectionPerHost parameter for
Http2SolrClient
Repository: lucene-solr
Updated Branches:
refs/heads/jira/http2 d0972b73c -> a833e356c
Introduce maxConnectionPerHost parameter for Http2SolrClient
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a833e356
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a833e356
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a833e356
Branch: refs/heads/jira/http2
Commit: a833e356c39f42f847d7506a67709aa77e0faa36
Parents: d0972b7
Author: Cao Manh Dat <da...@apache.org>
Authored: Fri Sep 21 16:27:47 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Fri Sep 21 16:27:47 2018 +0700
----------------------------------------------------------------------
.../component/HttpShardHandlerFactory.java | 29 ++++++++------------
.../apache/solr/update/UpdateShardHandler.java | 28 +++++++++----------
.../solr/client/solrj/impl/Http2SolrClient.java | 11 ++++++--
3 files changed, 34 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a833e356/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
index 1bc2069..a093517 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
@@ -90,11 +90,7 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
protected Http2SolrClient defaultClient;
protected InstrumentedHttpListenerFactory httpListenerFactory;
private LBHttp2SolrClient loadbalancer;
- //default values:
- int soTimeout = HttpClientUtil.DEFAULT_SO_TIMEOUT;
- int connectionTimeout = HttpClientUtil.DEFAULT_CONNECT_TIMEOUT;
- int maxConnectionsPerHost = HttpClientUtil.DEFAULT_MAXCONNECTIONSPERHOST;
- int maxConnections = HttpClientUtil.DEFAULT_MAXCONNECTIONS;
+
int corePoolSize = 0;
int maximumPoolSize = Integer.MAX_VALUE;
int keepAliveTime = 5;
@@ -156,7 +152,6 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
public void init(PluginInfo info) {
StringBuilder sb = new StringBuilder();
NamedList args = info.initArgs;
- this.soTimeout = getParameter(args, HttpClientUtil.PROP_SO_TIMEOUT, soTimeout,sb);
this.scheme = getParameter(args, INIT_URL_SCHEME, null,sb);
if(StringUtils.endsWith(this.scheme, "://")) {
this.scheme = StringUtils.removeEnd(this.scheme, "://");
@@ -169,9 +164,6 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
"Unknown metricNameStrategy: " + strategy + " found. Must be one of: " + KNOWN_METRIC_NAME_STRATEGIES.keySet());
}
- this.connectionTimeout = getParameter(args, HttpClientUtil.PROP_CONNECTION_TIMEOUT, connectionTimeout, sb);
- this.maxConnectionsPerHost = getParameter(args, HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, maxConnectionsPerHost,sb);
- this.maxConnections = getParameter(args, HttpClientUtil.PROP_MAX_CONNECTIONS, maxConnections,sb);
this.corePoolSize = getParameter(args, INIT_CORE_POOL_SIZE, corePoolSize,sb);
this.maximumPoolSize = getParameter(args, INIT_MAX_POOL_SIZE, maximumPoolSize,sb);
this.keepAliveTime = getParameter(args, MAX_THREAD_IDLE_TIME, keepAliveTime,sb);
@@ -208,7 +200,17 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
);
this.httpListenerFactory = new InstrumentedHttpListenerFactory(this.metricNameStrategy);
- this.defaultClient = new Http2SolrClient.Builder().connectionTimeout(connectionTimeout).idleTimeout(soTimeout).build();
+ int connectionTimeout = getParameter(args, HttpClientUtil.PROP_CONNECTION_TIMEOUT,
+ HttpClientUtil.DEFAULT_CONNECT_TIMEOUT, sb);
+ int maxConnectionsPerHost = getParameter(args, HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST,
+ HttpClientUtil.DEFAULT_MAXCONNECTIONSPERHOST, sb);
+ int soTimeout = getParameter(args, HttpClientUtil.PROP_SO_TIMEOUT,
+ HttpClientUtil.DEFAULT_SO_TIMEOUT, sb);
+
+ this.defaultClient = new Http2SolrClient.Builder()
+ .connectionTimeout(connectionTimeout)
+ .idleTimeout(soTimeout)
+ .maxConnectionsPerHost(maxConnectionsPerHost).build();
this.defaultClient.setListenerFactory(this.httpListenerFactory);
this.loadbalancer = new LBHttp2SolrClient(defaultClient);
}
@@ -220,13 +222,6 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
}
}
- protected ModifiableSolrParams getClientParams() {
- ModifiableSolrParams clientParams = new ModifiableSolrParams();
- clientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, maxConnectionsPerHost);
- clientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS, maxConnections);
- return clientParams;
- }
-
protected <T> T getParameter(NamedList initArgs, String configKey, T defaultValue, StringBuilder sb) {
T toReturn = defaultValue;
if (initArgs != null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a833e356/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java b/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
index 4c9e703..583c26e 100644
--- a/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
+++ b/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
@@ -25,6 +25,7 @@ import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import com.codahale.metrics.MetricRegistry;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.http.client.HttpClient;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
@@ -87,41 +88,36 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
public UpdateShardHandler(UpdateShardHandlerConfig cfg) {
defaultConnectionManager = new InstrumentedPoolingHttpClientConnectionManager(HttpClientUtil.getSchemaRegisteryProvider().getSchemaRegistry());
+ ModifiableSolrParams clientParams = new ModifiableSolrParams();
if (cfg != null ) {
defaultConnectionManager.setMaxTotal(cfg.getMaxUpdateConnections());
defaultConnectionManager.setDefaultMaxPerRoute(cfg.getMaxUpdateConnectionsPerHost());
- }
-
- ModifiableSolrParams clientParams = new ModifiableSolrParams();
- if (cfg != null) {
clientParams.set(HttpClientUtil.PROP_SO_TIMEOUT, cfg.getDistributedSocketTimeout());
clientParams.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, cfg.getDistributedConnectionTimeout());
+ // following is done only for logging complete configuration.
+ // The maxConnections and maxConnectionsPerHost have already been specified on the connection manager
+ clientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS, cfg.getMaxUpdateConnections());
+ clientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, cfg.getMaxUpdateConnectionsPerHost());
socketTimeout = cfg.getDistributedSocketTimeout();
connectionTimeout = cfg.getDistributedConnectionTimeout();
}
+ log.debug("Created default UpdateShardHandler HTTP client with params: {}", clientParams);
httpRequestExecutor = new InstrumentedHttpRequestExecutor(getMetricNameStrategy(cfg));
updateHttpListenerFactory = new InstrumentedHttpListenerFactory(getNameStrategy(cfg));
Http2SolrClient.Builder updateOnlyClientBuilder = new Http2SolrClient.Builder();
if (cfg != null) {
- updateOnlyClientBuilder.connectionTimeout(cfg.getDistributedConnectionTimeout())
- .idleTimeout(cfg.getDistributedSocketTimeout());
+ updateOnlyClientBuilder
+ .connectionTimeout(cfg.getDistributedConnectionTimeout())
+ .idleTimeout(cfg.getDistributedSocketTimeout())
+ .maxConnectionsPerHost(cfg.getMaxUpdateConnectionsPerHost());
}
updateOnlyClient = updateOnlyClientBuilder.build();
updateOnlyClient.setListenerFactory(updateHttpListenerFactory);
defaultClient = HttpClientUtil.createClient(clientParams, defaultConnectionManager, false, httpRequestExecutor);
- // following is done only for logging complete configuration.
- // The maxConnections and maxConnectionsPerHost have already been specified on the connection manager
- if (cfg != null) {
- clientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS, cfg.getMaxUpdateConnections());
- clientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, cfg.getMaxUpdateConnectionsPerHost());
- }
- log.debug("Created default UpdateShardHandler HTTP client with params: {}", clientParams);
- log.debug("Created update only UpdateShardHandler HTTP client with params: {}", clientParams);
-
ThreadFactory recoveryThreadFactory = new SolrjNamedThreadFactory("recoveryExecutor");
if (cfg != null && cfg.getMaxRecoveryThreads() > 0) {
log.debug("Creating recoveryExecutor with pool size {}", cfg.getMaxRecoveryThreads());
@@ -241,10 +237,12 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
}
}
+ @VisibleForTesting
public int getSocketTimeout() {
return socketTimeout;
}
+ @VisibleForTesting
public int getConnectionTimeout() {
return connectionTimeout;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a833e356/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
index 320ea27..85d0f13 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
@@ -215,10 +215,11 @@ public class Http2SolrClient extends SolrClient {
httpClient.setStrictEventOrdering(false);
httpClient.setConnectBlocking(true);
httpClient.setFollowRedirects(false);
- httpClient.setMaxConnectionsPerDestination(4);
- httpClient.setMaxRequestsQueuedPerDestination(MAX_OUTSTANDING_REQUESTS * 4); // comfortably above max outstanding // requests
+ // comfortably above max outstanding requests
+ httpClient.setMaxRequestsQueuedPerDestination(MAX_OUTSTANDING_REQUESTS * 4);
httpClient.setUserAgentField(new HttpField(HttpHeader.USER_AGENT, AGENT));
+ if (builder.maxConnectionsPerHost != null) httpClient.setMaxConnectionsPerDestination(builder.maxConnectionsPerHost);
if (builder.idleTimeout != null) httpClient.setIdleTimeout(builder.idleTimeout);
if (builder.connectionTimeout != null) httpClient.setConnectTimeout(builder.connectionTimeout);
return httpClient;
@@ -638,6 +639,7 @@ public class Http2SolrClient extends SolrClient {
private SSLConfig sslConfig = defaultSSLConfig;
private Integer idleTimeout;
private Integer connectionTimeout;
+ private Integer maxConnectionsPerHost;
private boolean useHttp1_1 = Boolean.getBoolean("solr.http1");
protected String baseSolrUrl;
private Request.BeginListener beginListener = request -> {};
@@ -664,6 +666,11 @@ public class Http2SolrClient extends SolrClient {
return this;
}
+ public Builder maxConnectionsPerHost(int max) {
+ this.maxConnectionsPerHost = max;
+ return this;
+ }
+
public Builder idleTimeout(int idleConnectionTimeout) {
this.idleTimeout = idleConnectionTimeout;
return this;