You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mk...@apache.org on 2016/10/09 07:00:23 UTC
lucene-solr:master: SOLR-9604,
SOLR-9608: Ensure SSL connections are re-used
Repository: lucene-solr
Updated Branches:
refs/heads/master 041457034 -> f22b1da26
SOLR-9604,SOLR-9608: Ensure SSL connections are re-used
Fix ConnectionReuseTest. Add coverage for all SolrClients.
Remove explicit cacheKey for HttpRequestContext, make it singleton.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f22b1da2
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f22b1da2
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f22b1da2
Branch: refs/heads/master
Commit: f22b1da261b93f60687431b161828e2fb27fdc8f
Parents: 0414570
Author: Alan Woodward <ro...@apache.org>
Authored: Wed Oct 5 10:22:22 2016 +0100
Committer: Mikhail Khludnev <mk...@apache.org>
Committed: Sun Oct 9 08:06:09 2016 +0300
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../solr/security/PKIAuthenticationPlugin.java | 18 +-
.../org/apache/solr/servlet/HttpSolrCall.java | 3 +-
.../solr/client/solrj/ConnectionReuseTest.java | 198 -------------------
.../client/solrj/impl/ConnectionReuseTest.java | 196 ++++++++++++++++++
.../solrj/impl/ConcurrentUpdateSolrClient.java | 3 +-
.../solr/client/solrj/impl/HttpClientUtil.java | 16 +-
.../solr/client/solrj/impl/HttpSolrClient.java | 6 +-
.../impl/SolrHttpClientContextBuilder.java | 12 +-
.../client/solrj/SolrSchemalessExampleTest.java | 16 +-
.../solrj/embedded/SolrExampleJettyTest.java | 6 +-
.../solrj/impl/HttpSolrClientConPoolTest.java | 182 +++++++++++++++++
.../impl/HttpSolrClientSSLAuthConPoolTest.java | 40 ++++
.../org/apache/solr/util/RestTestHarness.java | 14 +-
14 files changed, 481 insertions(+), 232 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f22b1da2/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index f27216e..5d5ffaf 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -160,6 +160,9 @@ Bug Fixes
* SOLR-9278: Index replication interactions with IndexWriter can cause deadlock. (Xunlong via Mark Miller)
+* SOLR-9604: Pooled SSL connections were not being re-used (Alan Woodward,
+ Mikhail Khludnev, hossman)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f22b1da2/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java b/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java
index e5d2653..9de9e42 100644
--- a/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java
+++ b/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java
@@ -16,8 +16,11 @@
*/
package org.apache.solr.security;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
+import javax.servlet.FilterChain;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletRequestWrapper;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.nio.ByteBuffer;
@@ -27,12 +30,6 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
-import javax.servlet.FilterChain;
-import javax.servlet.ServletRequest;
-import javax.servlet.ServletResponse;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletRequestWrapper;
-
import org.apache.http.HttpException;
import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor;
@@ -58,6 +55,8 @@ import org.apache.solr.util.CryptoKeys;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
public class PKIAuthenticationPlugin extends AuthenticationPlugin implements HttpClientBuilderPlugin {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -198,7 +197,8 @@ public class PKIAuthenticationPlugin extends AuthenticationPlugin implements Htt
try {
String uri = url + PATH + "?wt=json&omitHeader=true";
log.debug("Fetching fresh public key from : {}",uri);
- HttpResponse rsp = cores.getUpdateShardHandler().getHttpClient().execute(new HttpGet(uri), HttpClientUtil.createNewHttpClientRequestContext());
+ HttpResponse rsp = cores.getUpdateShardHandler().getHttpClient()
+ .execute(new HttpGet(uri), HttpClientUtil.createNewHttpClientRequestContext());
byte[] bytes = EntityUtils.toByteArray(rsp.getEntity());
Map m = (Map) Utils.fromJSON(bytes);
String key = (String) m.get("key");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f22b1da2/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
index bc5edd5..c41595e 100644
--- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
+++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
@@ -558,7 +558,8 @@ public class HttpSolrCall {
method.removeHeaders(CONTENT_LENGTH_HEADER);
}
- final HttpResponse response = solrDispatchFilter.httpClient.execute(method, HttpClientUtil.createNewHttpClientRequestContext());
+ final HttpResponse response
+ = solrDispatchFilter.httpClient.execute(method, HttpClientUtil.createNewHttpClientRequestContext());
int httpStatus = response.getStatusLine().getStatusCode();
httpEntity = response.getEntity();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f22b1da2/solr/core/src/test/org/apache/solr/client/solrj/ConnectionReuseTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/client/solrj/ConnectionReuseTest.java b/solr/core/src/test/org/apache/solr/client/solrj/ConnectionReuseTest.java
deleted file mode 100644
index a397f55..0000000
--- a/solr/core/src/test/org/apache/solr/client/solrj/ConnectionReuseTest.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.client.solrj;
-
-import java.io.IOException;
-import java.net.URL;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.http.HttpClientConnection;
-import org.apache.http.HttpConnectionMetrics;
-import org.apache.http.HttpException;
-import org.apache.http.HttpHost;
-import org.apache.http.HttpRequest;
-import org.apache.http.HttpVersion;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.protocol.HttpClientContext;
-import org.apache.http.conn.ConnectionPoolTimeoutException;
-import org.apache.http.conn.ConnectionRequest;
-import org.apache.http.conn.routing.HttpRoute;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
-import org.apache.http.message.BasicHttpRequest;
-import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient;
-import org.apache.solr.client.solrj.impl.HttpClientUtil;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.update.AddUpdateCommand;
-import org.apache.solr.util.TestInjection;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-@SuppressSSL
-public class ConnectionReuseTest extends SolrCloudTestCase {
-
- private AtomicInteger id = new AtomicInteger();
- private HttpClientContext context = HttpClientContext.create();
-
- private static final String COLLECTION = "collection1";
-
- @BeforeClass
- public static void setupCluster() throws Exception {
- TestInjection.failUpdateRequests = "true:100";
- configureCluster(1)
- .addConfig("config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
- .configure();
-
- CollectionAdminRequest.createCollection(COLLECTION, "config", 1, 1)
- .processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);
-
- cluster.getSolrClient().waitForState(COLLECTION, DEFAULT_TIMEOUT, TimeUnit.SECONDS,
- (n, c) -> DocCollection.isFullyActive(n, c, 1, 1));
- }
-
- private SolrClient buildClient(CloseableHttpClient httpClient, URL url) {
- switch (random().nextInt(3)) {
- case 0:
- // currently only testing with 1 thread
- return getConcurrentUpdateSolrClient(url.toString() + "/" + COLLECTION, httpClient, 6, 1);
- case 1:
- return getHttpSolrClient(url.toString() + "/" + COLLECTION, httpClient);
- case 2:
- CloudSolrClient client = getCloudSolrClient(cluster.getZkServer().getZkAddress(), random().nextBoolean(), httpClient);
- client.setParallelUpdates(random().nextBoolean());
- client.setDefaultCollection(COLLECTION);
- client.getLbClient().setConnectionTimeout(30000);
- client.getLbClient().setSoTimeout(60000);
- return client;
- }
- throw new RuntimeException("impossible");
- }
-
- @Test
- public void testConnectionReuse() throws Exception {
-
- URL url = cluster.getJettySolrRunners().get(0).getBaseUrl();
- PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
-
- CloseableHttpClient httpClient = HttpClientUtil.createClient(null, cm);
- try (SolrClient client = buildClient(httpClient, url)) {
-
- HttpHost target = new HttpHost(url.getHost(), url.getPort(), isSSLMode() ? "https" : "http");
- HttpRoute route = new HttpRoute(target);
-
- ConnectionRequest mConn = getClientConnectionRequest(httpClient, route, cm);
-
- HttpClientConnection conn1 = getConn(mConn);
- headerRequest(target, route, conn1, cm);
-
- cm.releaseConnection(conn1, null, -1, TimeUnit.MILLISECONDS);
-
- int queueBreaks = 0;
- int cnt1 = atLeast(3);
- int cnt2 = atLeast(30);
- for (int j = 0; j < cnt1; j++) {
- boolean done = false;
- for (int i = 0; i < cnt2; i++) {
- AddUpdateCommand c = new AddUpdateCommand(null);
- c.solrDoc = sdoc("id", id.incrementAndGet());
- try {
- client.add(c.solrDoc);
- } catch (Exception e) {
- e.printStackTrace();
- }
- if (!done && i > 0 && i < cnt2 - 1 && client instanceof ConcurrentUpdateSolrClient
- && random().nextInt(10) > 8) {
- queueBreaks++;
- done = true;
- Thread.sleep(350); // wait past streaming client poll time of 250ms
- }
- }
- if (client instanceof ConcurrentUpdateSolrClient) {
- ((ConcurrentUpdateSolrClient) client).blockUntilFinished();
- }
- }
-
- route = new HttpRoute(new HttpHost(url.getHost(), url.getPort(), isSSLMode() ? "https" : "http"));
-
- mConn = cm.requestConnection(route, null);
-
- HttpClientConnection conn2 = getConn(mConn);
-
- HttpConnectionMetrics metrics = conn2.getMetrics();
- headerRequest(target, route, conn2, cm);
-
- cm.releaseConnection(conn2, null, -1, TimeUnit.MILLISECONDS);
-
- assertNotNull("No connection metrics found - is the connection getting aborted? server closing the connection? "
- + client.getClass().getSimpleName(), metrics);
-
- // we try and make sure the connection we get has handled all of the requests in this test
- if (client instanceof ConcurrentUpdateSolrClient) {
- // we can't fully control queue polling breaking up requests - allow a bit of leeway
- int exp = cnt1 + queueBreaks + 2;
- assertTrue(
- "We expected all communication via streaming client to use one connection! expected=" + exp + " got="
- + metrics.getRequestCount(),
- Math.max(exp, metrics.getRequestCount()) - Math.min(exp, metrics.getRequestCount()) < 3);
- } else {
- assertTrue("We expected all communication to use one connection! " + client.getClass().getSimpleName() + " "
- + metrics.getRequestCount(),
- cnt1 * cnt2 + 2 <= metrics.getRequestCount());
- }
-
- }
- finally {
- HttpClientUtil.close(httpClient);
- }
- }
-
- public HttpClientConnection getConn(ConnectionRequest mConn)
- throws InterruptedException, ConnectionPoolTimeoutException, ExecutionException {
- HttpClientConnection conn = mConn.get(30, TimeUnit.SECONDS);
-
- return conn;
- }
-
- public void headerRequest(HttpHost target, HttpRoute route, HttpClientConnection conn, PoolingHttpClientConnectionManager cm)
- throws IOException, HttpException {
- HttpRequest req = new BasicHttpRequest("OPTIONS", "*", HttpVersion.HTTP_1_1);
-
- req.addHeader("Host", target.getHostName());
- if (!conn.isOpen()) {
- // establish connection based on its route info
- cm.connect(conn, route, 1000, context);
- // and mark it as route complete
- cm.routeComplete(conn, route, context);
- }
- conn.sendRequestHeader(req);
- conn.flush();
- conn.receiveResponseHeader();
- }
-
- public ConnectionRequest getClientConnectionRequest(HttpClient httpClient, HttpRoute route, PoolingHttpClientConnectionManager cm) {
- ConnectionRequest mConn = cm.requestConnection(route, null);
- return mConn;
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f22b1da2/solr/core/src/test/org/apache/solr/client/solrj/impl/ConnectionReuseTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/client/solrj/impl/ConnectionReuseTest.java b/solr/core/src/test/org/apache/solr/client/solrj/impl/ConnectionReuseTest.java
new file mode 100644
index 0000000..56ec3f2
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/client/solrj/impl/ConnectionReuseTest.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.impl;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.http.HttpClientConnection;
+import org.apache.http.HttpConnectionMetrics;
+import org.apache.http.HttpException;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpRequest;
+import org.apache.http.HttpVersion;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.conn.ConnectionPoolTimeoutException;
+import org.apache.http.conn.ConnectionRequest;
+import org.apache.http.conn.routing.HttpRoute;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.message.BasicHttpRequest;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.update.AddUpdateCommand;
+import org.apache.solr.util.TestInjection;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+@SuppressSSL
+public class ConnectionReuseTest extends SolrCloudTestCase {
+
+ private AtomicInteger id = new AtomicInteger();
+ private HttpClientContext context = HttpClientContext.create();
+
+ private static final String COLLECTION = "collection1";
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ TestInjection.failUpdateRequests = "true:100";
+ configureCluster(1)
+ .addConfig("config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
+ .configure();
+
+ CollectionAdminRequest.createCollection(COLLECTION, "config", 1, 1)
+ .processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);
+
+ cluster.getSolrClient().waitForState(COLLECTION, DEFAULT_TIMEOUT, TimeUnit.SECONDS,
+ (n, c) -> DocCollection.isFullyActive(n, c, 1, 1));
+ }
+
+ private SolrClient buildClient(CloseableHttpClient httpClient, URL url) {
+ switch (random().nextInt(3)) {
+ case 0:
+ // currently only testing with 1 thread
+ return getConcurrentUpdateSolrClient(url.toString() + "/" + COLLECTION, httpClient, 6, 1);
+ case 1:
+ return getHttpSolrClient(url.toString() + "/" + COLLECTION, httpClient);
+ case 2:
+ CloudSolrClient client = getCloudSolrClient(cluster.getZkServer().getZkAddress(), random().nextBoolean(), httpClient);
+ client.setParallelUpdates(random().nextBoolean());
+ client.setDefaultCollection(COLLECTION);
+ client.getLbClient().setConnectionTimeout(30000);
+ client.getLbClient().setSoTimeout(60000);
+ return client;
+ }
+ throw new RuntimeException("impossible");
+ }
+
+ @Test
+ public void testConnectionReuse() throws Exception {
+
+ URL url = cluster.getJettySolrRunners().get(0).getBaseUrl();
+ PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
+
+ CloseableHttpClient httpClient = HttpClientUtil.createClient(null, cm);
+ try (SolrClient client = buildClient(httpClient, url)) {
+
+ HttpHost target = new HttpHost(url.getHost(), url.getPort(), isSSLMode() ? "https" : "http");
+ HttpRoute route = new HttpRoute(target);
+
+ ConnectionRequest mConn = getClientConnectionRequest(httpClient, route, cm);
+
+ HttpClientConnection conn1 = getConn(mConn);
+ headerRequest(target, route, conn1, cm);
+
+ cm.releaseConnection(conn1, null, -1, TimeUnit.MILLISECONDS);
+
+ int queueBreaks = 0;
+ int cnt1 = atLeast(3);
+ int cnt2 = atLeast(30);
+ for (int j = 0; j < cnt1; j++) {
+ boolean done = false;
+ for (int i = 0; i < cnt2; i++) {
+ AddUpdateCommand c = new AddUpdateCommand(null);
+ c.solrDoc = sdoc("id", id.incrementAndGet());
+ try {
+ client.add(c.solrDoc);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ if (!done && i > 0 && i < cnt2 - 1 && client instanceof ConcurrentUpdateSolrClient
+ && random().nextInt(10) > 8) {
+ queueBreaks++;
+ done = true;
+ Thread.sleep(350); // wait past streaming client poll time of 250ms
+ }
+ }
+ if (client instanceof ConcurrentUpdateSolrClient) {
+ ((ConcurrentUpdateSolrClient) client).blockUntilFinished();
+ }
+ }
+
+ route = new HttpRoute(new HttpHost(url.getHost(), url.getPort(), isSSLMode() ? "https" : "http"));
+
+ mConn = cm.requestConnection(route, HttpSolrClient.cacheKey);
+
+ HttpClientConnection conn2 = getConn(mConn);
+
+ HttpConnectionMetrics metrics = conn2.getMetrics();
+ headerRequest(target, route, conn2, cm);
+
+ cm.releaseConnection(conn2, null, -1, TimeUnit.MILLISECONDS);
+
+ assertNotNull("No connection metrics found - is the connection getting aborted? server closing the connection? "
+ + client.getClass().getSimpleName(), metrics);
+
+ // we try and make sure the connection we get has handled all of the requests in this test
+ if (client instanceof ConcurrentUpdateSolrClient) {
+ // we can't fully control queue polling breaking up requests - allow a bit of leeway
+ int exp = cnt1 + queueBreaks + 2;
+ assertTrue(
+ "We expected all communication via streaming client to use one connection! expected=" + exp + " got="
+ + metrics.getRequestCount(),
+ Math.max(exp, metrics.getRequestCount()) - Math.min(exp, metrics.getRequestCount()) < 3);
+ } else {
+ assertTrue("We expected all communication to use one connection! " + client.getClass().getSimpleName() + " "
+ + metrics.getRequestCount(),
+ cnt1 * cnt2 + 2 <= metrics.getRequestCount());
+ }
+
+ }
+ finally {
+ HttpClientUtil.close(httpClient);
+ }
+ }
+
+ public HttpClientConnection getConn(ConnectionRequest mConn)
+ throws InterruptedException, ConnectionPoolTimeoutException, ExecutionException {
+ HttpClientConnection conn = mConn.get(30, TimeUnit.SECONDS);
+
+ return conn;
+ }
+
+ public void headerRequest(HttpHost target, HttpRoute route, HttpClientConnection conn, PoolingHttpClientConnectionManager cm)
+ throws IOException, HttpException {
+ HttpRequest req = new BasicHttpRequest("OPTIONS", "*", HttpVersion.HTTP_1_1);
+
+ req.addHeader("Host", target.getHostName());
+ if (!conn.isOpen()) {
+ // establish connection based on its route info
+ cm.connect(conn, route, 1000, context);
+ // and mark it as route complete
+ cm.routeComplete(conn, route, context);
+ }
+ conn.sendRequestHeader(req);
+ conn.flush();
+ conn.receiveResponseHeader();
+ }
+
+ public ConnectionRequest getClientConnectionRequest(HttpClient httpClient, HttpRoute route, PoolingHttpClientConnectionManager cm) {
+ ConnectionRequest mConn = cm.requestConnection(route, HttpSolrClient.cacheKey);
+ return mConn;
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f22b1da2/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java
index f209672..b96cc23 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java
@@ -320,7 +320,8 @@ public class ConcurrentUpdateSolrClient extends SolrClient {
method.addHeader("User-Agent", HttpSolrClient.AGENT);
method.addHeader("Content-Type", contentType);
- response = client.getHttpClient().execute(method, HttpClientUtil.createNewHttpClientRequestContext());
+ response = client.getHttpClient()
+ .execute(method, HttpClientUtil.createNewHttpClientRequestContext());
rspBody = response.getEntity().getContent();
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f22b1da2/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java
index b9580b8..ed285d1 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java
@@ -198,7 +198,12 @@ public class HttpClientUtil {
* configuration (no additional configuration) is created.
*/
public static CloseableHttpClient createClient(SolrParams params) {
- return createClient(params, new PoolingHttpClientConnectionManager(schemaRegistryProvider.getSchemaRegistry()));
+ return createClient(params, createPoolingConnectionManager());
+ }
+
+ /** test usage subject to change @lucene.experimental */
+ static PoolingHttpClientConnectionManager createPoolingConnectionManager() {
+ return new PoolingHttpClientConnectionManager(schemaRegistryProvider.getSchemaRegistry());
}
public static CloseableHttpClient createClient(SolrParams params, PoolingHttpClientConnectionManager cm) {
@@ -396,10 +401,15 @@ public class HttpClientUtil {
}
/**
- *
+ * Create a HttpClientContext object and {@link HttpClientContext#setUserToken(Object)}
+ * to an internal singleton. It allows to reuse underneath {@link HttpClient}
+ * in connection pools if client authentication is enabled.
*/
public static HttpClientContext createNewHttpClientRequestContext() {
- return httpClientRequestContextBuilder.createContext();
+ HttpClientContext context = new HttpClientContext();
+
+ context.setUserToken(HttpSolrClient.cacheKey);
+ return context;
}
public static Builder createDefaultRequestConfigBuilder() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f22b1da2/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java
index e43fd33..3e2355c 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java
@@ -47,6 +47,7 @@ import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.conn.HttpClientConnectionManager;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.InputStreamEntity;
@@ -111,6 +112,8 @@ public class HttpSolrClient extends SolrClient {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ static final Class<HttpSolrClient> cacheKey = HttpSolrClient.class;
+
/**
* The URL of the Solr server.
*/
@@ -508,7 +511,8 @@ public class HttpSolrClient extends SolrClient {
boolean shouldClose = true;
try {
// Execute the method.
- final HttpResponse response = httpClient.execute(method, HttpClientUtil.createNewHttpClientRequestContext());
+ HttpClientContext httpClientRequestContext = HttpClientUtil.createNewHttpClientRequestContext();
+ final HttpResponse response = httpClient.execute(method, httpClientRequestContext);
int httpStatus = response.getStatusLine().getStatusCode();
// Read the contents
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f22b1da2/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrHttpClientContextBuilder.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrHttpClientContextBuilder.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrHttpClientContextBuilder.java
index b678df7..f57848d 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrHttpClientContextBuilder.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrHttpClientContextBuilder.java
@@ -76,8 +76,16 @@ public class SolrHttpClientContextBuilder {
public CredentialsProviderProvider getCredentialsProviderProvider() {
return credentialsProviderProvider;
}
-
+
+ /**
+ * @deprecated use {@link #createContext(Object)}
+ */
+ @Deprecated
public HttpClientContext createContext() {
+ return createContext(null);
+ }
+
+ public HttpClientContext createContext(Object userToken) {
HttpClientContext context = new HttpClientContext();
if (getCredentialsProviderProvider() != null) {
context.setCredentialsProvider(getCredentialsProviderProvider().getCredentialsProvider());
@@ -89,6 +97,8 @@ public class SolrHttpClientContextBuilder {
if (getCookieSpecRegistryProvider() != null) {
context.setCookieSpecRegistry(getCookieSpecRegistryProvider().getCookieSpecRegistry());
}
+
+ context.setUserToken(userToken);
return context;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f22b1da2/solr/solrj/src/test/org/apache/solr/client/solrj/SolrSchemalessExampleTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrSchemalessExampleTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrSchemalessExampleTest.java
index d81667b..3d553d8 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrSchemalessExampleTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrSchemalessExampleTest.java
@@ -16,6 +16,14 @@
*/
package org.apache.solr.client.solrj;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.OutputStreamWriter;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Properties;
+
import org.apache.commons.io.FileUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
@@ -32,14 +40,6 @@ import org.apache.solr.util.ExternalPaths;
import org.junit.BeforeClass;
import org.junit.Test;
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.OutputStreamWriter;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Properties;
-
public class SolrSchemalessExampleTest extends SolrExampleTestsBase {
@BeforeClass
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f22b1da2/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java
index a8832e7..2cf8e04 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java
@@ -16,6 +16,9 @@
*/
package org.apache.solr.client.solrj.embedded;
+import java.io.ByteArrayInputStream;
+import java.util.Map;
+
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPost;
@@ -32,9 +35,6 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.noggit.ObjectBuilder;
-import java.io.ByteArrayInputStream;
-import java.util.Map;
-
/**
* TODO? perhaps use:
* http://docs.codehaus.org/display/JETTY/ServletTester
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f22b1da2/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientConPoolTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientConPoolTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientConPoolTest.java
new file mode 100644
index 0000000..1e9b0f3
--- /dev/null
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientConPoolTest.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.impl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.pool.PoolStats;
+import org.apache.solr.SolrJettyTestBase;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.SolrjNamedThreadFactory;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+public class HttpSolrClientConPoolTest extends SolrJettyTestBase {
+
+ protected static JettySolrRunner yetty;
+ private static String fooUrl;
+ private static String barUrl;
+
+ @BeforeClass
+ public static void beforeTest() throws Exception {
+ createJetty(legacyExampleCollection1SolrHome());
+ // stealing the first made jetty
+ yetty = jetty;
+ barUrl = yetty.getBaseUrl().toString() + "/" + "collection1";
+
+ createJetty(legacyExampleCollection1SolrHome());
+
+ fooUrl = jetty.getBaseUrl().toString() + "/" + "collection1";
+ }
+
+ @AfterClass
+ public static void stopYetty() throws Exception {
+ yetty.stop();
+ yetty = null;
+ }
+
+ public void testPoolSize() throws SolrServerException, IOException {
+ PoolingHttpClientConnectionManager pool = HttpClientUtil.createPoolingConnectionManager();
+ final HttpSolrClient client1 ;
+ final String fooUrl;
+ {
+ fooUrl = jetty.getBaseUrl().toString() + "/" + "collection1";
+ CloseableHttpClient httpClient = HttpClientUtil.createClient(new ModifiableSolrParams(), pool,
+ false /* let client shutdown it*/);
+ client1 = getHttpSolrClient(fooUrl, httpClient);
+ client1.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT);
+ }
+ final String barUrl = yetty.getBaseUrl().toString() + "/" + "collection1";
+
+ List<String> urls = new ArrayList<>();
+ for(int i=0; i<17; i++) {
+ urls.add(fooUrl);
+ }
+ for(int i=0; i<31; i++) {
+ urls.add(barUrl);
+ }
+
+ Collections.shuffle(urls, random());
+
+ try {
+ int i=0;
+ for (String url : urls) {
+ if (!client1.getBaseURL().equals(url)) {
+ client1.setBaseURL(url);
+ }
+ client1.add(new SolrInputDocument("id", ""+(i++)));
+ }
+ client1.setBaseURL(fooUrl);
+ client1.commit();
+ assertEquals(17, client1.query(new SolrQuery("*:*")).getResults().getNumFound());
+
+ client1.setBaseURL(barUrl);
+ client1.commit();
+ assertEquals(31, client1.query(new SolrQuery("*:*")).getResults().getNumFound());
+
+ PoolStats stats = pool.getTotalStats();
+ assertEquals("oh "+stats, 2, stats.getAvailable());
+ } finally {
+ for (HttpSolrClient c : new HttpSolrClient []{ client1}) {
+ HttpClientUtil.close(c.getHttpClient());
+ c.close();
+ }
+ }
+ }
+
+
+ public void testLBClient() throws IOException, SolrServerException {
+
+ PoolingHttpClientConnectionManager pool = HttpClientUtil.createPoolingConnectionManager();
+ final HttpSolrClient client1 ;
+ int threadCount = atLeast(2);
+ final ExecutorService threads = ExecutorUtil.newMDCAwareFixedThreadPool(threadCount,
+ new SolrjNamedThreadFactory(getClass().getSimpleName()+"TestScheduler"));
+ CloseableHttpClient httpClient = HttpClientUtil.createClient(new ModifiableSolrParams(), pool);
+ try{
+ final LBHttpSolrClient roundRobin = new LBHttpSolrClient.Builder().
+ withBaseSolrUrl(fooUrl).
+ withBaseSolrUrl(barUrl).
+ withHttpClient(httpClient)
+ .build();
+
+ List<ConcurrentUpdateSolrClient> concurrentClients = Arrays.asList(
+ new ConcurrentUpdateSolrClient.Builder(fooUrl)
+ .withHttpClient(httpClient).withThreadCount(threadCount)
+ .withQueueSize(10)
+ .withExecutorService(threads).build(),
+ new ConcurrentUpdateSolrClient.Builder(barUrl)
+ .withHttpClient(httpClient).withThreadCount(threadCount)
+ .withQueueSize(10)
+ .withExecutorService(threads).build());
+
+ for (int i=0; i<2; i++) {
+ roundRobin.deleteByQuery("*:*");
+ }
+
+ for (int i=0; i<57; i++) {
+ final SolrInputDocument doc = new SolrInputDocument("id", ""+i);
+ if (random().nextBoolean()) {
+ final ConcurrentUpdateSolrClient concurrentClient = concurrentClients.get(random().nextInt(concurrentClients.size()));
+ concurrentClient.add(doc); // here we are testing that CUSC and plain clients reuse pool
+ concurrentClient.blockUntilFinished();
+ } else {
+ if (random().nextBoolean()) {
+ roundRobin.add(doc);
+ } else {
+ final UpdateRequest updateRequest = new UpdateRequest();
+ updateRequest.add(doc); // here we mimic CloudSolrClient impl
+ final List<String> urls = Arrays.asList(fooUrl, barUrl);
+ Collections.shuffle(urls, random());
+ LBHttpSolrClient.Req req = new LBHttpSolrClient.Req(updateRequest,
+ urls);
+ roundRobin.request(req);
+ }
+ }
+ }
+
+ for (int i=0; i<2; i++) {
+ roundRobin.commit();
+ }
+ int total=0;
+ for (int i=0; i<2; i++) {
+ total += roundRobin.query(new SolrQuery("*:*")).getResults().getNumFound();
+ }
+ assertEquals(57, total);
+ PoolStats stats = pool.getTotalStats();
+ //System.out.println("\n"+stats);
+ assertEquals("expected number of connections shouldn't exceed number of endpoints" + stats,
+ 2, stats.getAvailable());
+ }finally {
+ threads.shutdown();
+ HttpClientUtil.close(httpClient);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f22b1da2/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientSSLAuthConPoolTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientSSLAuthConPoolTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientSSLAuthConPoolTest.java
new file mode 100644
index 0000000..3a39c67
--- /dev/null
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientSSLAuthConPoolTest.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.impl;
+
+
+import java.net.URL;
+import java.util.Arrays;
+
+import org.apache.solr.util.RandomizeSSL;
+import org.junit.BeforeClass;
+
+@RandomizeSSL(1.0)
+public class HttpSolrClientSSLAuthConPoolTest extends HttpSolrClientConPoolTest {
+
+ @BeforeClass
+ public static void checkUrls() throws Exception {
+ URL[] urls = new URL[] {
+ jetty.getBaseUrl(), yetty.getBaseUrl()
+ };
+ for (URL u : urls) {
+ assertEquals("expect https urls ","https", u.getProtocol());
+ }
+ assertFalse("expect different urls "+Arrays.toString(urls),
+ urls[0].equals(urls[1]));
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f22b1da2/solr/test-framework/src/java/org/apache/solr/util/RestTestHarness.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/util/RestTestHarness.java b/solr/test-framework/src/java/org/apache/solr/util/RestTestHarness.java
index 3f2a699..0f1c8c9 100644
--- a/solr/test-framework/src/java/org/apache/solr/util/RestTestHarness.java
+++ b/solr/test-framework/src/java/org/apache/solr/util/RestTestHarness.java
@@ -15,6 +15,13 @@
* limitations under the License.
*/
package org.apache.solr.util;
+import javax.xml.xpath.XPathConstants;
+import javax.xml.xpath.XPathExpressionException;
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpGet;
@@ -28,13 +35,6 @@ import org.apache.http.util.EntityUtils;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.common.params.ModifiableSolrParams;
-import javax.xml.xpath.XPathConstants;
-import javax.xml.xpath.XPathExpressionException;
-import java.io.Closeable;
-import java.io.IOException;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
-
/**
* Facilitates testing Solr's REST API via a provided embedded Jetty
*/