You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2015/01/14 20:05:09 UTC
svn commit: r1651775 - in /lucene/dev/branches/lucene_solr_5_0: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/handler/component/
solr/core/src/java/org/apache/solr/update/ solr/solrj/
solr/solrj/src/java/org/apache/solr/client/solrj/impl/
Author: markrmiller
Date: Wed Jan 14 19:05:08 2015
New Revision: 1651775
URL: http://svn.apache.org/r1651775
Log:
SOLR-6931: We should do a limited retry when using HttpClient.
Modified:
lucene/dev/branches/lucene_solr_5_0/ (props changed)
lucene/dev/branches/lucene_solr_5_0/.gitignore
lucene/dev/branches/lucene_solr_5_0/solr/ (props changed)
lucene/dev/branches/lucene_solr_5_0/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene_solr_5_0/solr/core/ (props changed)
lucene/dev/branches/lucene_solr_5_0/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
lucene/dev/branches/lucene_solr_5_0/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
lucene/dev/branches/lucene_solr_5_0/solr/solrj/ (props changed)
lucene/dev/branches/lucene_solr_5_0/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientConfigurer.java
lucene/dev/branches/lucene_solr_5_0/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java
lucene/dev/branches/lucene_solr_5_0/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java
Modified: lucene/dev/branches/lucene_solr_5_0/.gitignore
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_5_0/.gitignore?rev=1651775&r1=1651774&r2=1651775&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_5_0/.gitignore (original)
+++ lucene/dev/branches/lucene_solr_5_0/.gitignore Wed Jan 14 19:05:08 2015
@@ -1,49 +1 @@
-# .
-/eclipse-build
-/classes
-**/build
-/idea-build
-**/dist
-**/lib
-**/test-lib
-/*~
-/velocity.log
-/build.properties
-/.idea
-lucene/**/*.iml
-solr/**/*.iml
-parent.iml
-**/*.ipr
-**/*.iws
-/.project
-/.classpath
-/.settings
-/.caches
-/prj.el
-/bin
-/bin.*
-**/pom.xml
-/nbproject
-/nb-build
-
-/solr/package
-
-# can this be minimized?
-/solr/example/start.jar
-/solr/example/webapps/*
-/solr/example/logs/*.log
-/solr/example/**/data
-/solr/example/solr/lib
-/solr/example/solr/logs
-/solr/example/solr/zoo_data
-/solr/example/work/*
-/solr/example/exampledocs/post.jar
-
-/solr/example/example-DIH/**/data
-/solr/example/example-DIH/**/dataimport.properties
-/solr/example/example-DIH/solr/mail/lib/*.jar
-
-solr/contrib/dataimporthandler/test-lib/
-solr/contrib/morphlines-core/test-lib/
-
-solr/core/test-lib/
+/bin/
Modified: lucene/dev/branches/lucene_solr_5_0/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_5_0/solr/CHANGES.txt?rev=1651775&r1=1651774&r2=1651775&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_5_0/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene_solr_5_0/solr/CHANGES.txt Wed Jan 14 19:05:08 2015
@@ -400,6 +400,9 @@ Bug Fixes
* SOLR-6941: DistributedQueue#containsTaskWithRequestId can fail with NPE. (Mark Miller)
+* SOLR-6931: We should do a limited retry when using HttpClient.
+ (Mark Miller, Hrishikesh Gadre, Gregory Chanan)
+
Optimizations
----------------------
Modified: lucene/dev/branches/lucene_solr_5_0/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_5_0/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java?rev=1651775&r1=1651774&r2=1651775&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_5_0/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java (original)
+++ lucene/dev/branches/lucene_solr_5_0/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java Wed Jan 14 19:05:08 2015
@@ -16,8 +16,22 @@ package org.apache.solr.handler.componen
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
import org.apache.commons.lang.StringUtils;
import org.apache.http.client.HttpClient;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
@@ -33,18 +47,6 @@ import org.apache.solr.util.DefaultSolrT
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.apache.solr.util.plugin.PluginInfoInitialized {
protected static Logger log = LoggerFactory.getLogger(HttpShardHandlerFactory.class);
@@ -76,6 +78,7 @@ public class HttpShardHandlerFactory ext
int keepAliveTime = 5;
int queueSize = -1;
boolean accessPolicy = false;
+ boolean useRetries = false;
private String scheme = null;
@@ -98,6 +101,10 @@ public class HttpShardHandlerFactory ext
// Configure if the threadpool favours fairness over throughput
static final String INIT_FAIRNESS_POLICY = "fairnessPolicy";
+
+ // Turn on retries for certain IOExceptions, many of which can happen
+ // due to connection pooling limitations / races
+ static final String USE_RETRIES = "useRetries";
/**
* Get {@link ShardHandler} that uses the default http client.
@@ -130,6 +137,7 @@ public class HttpShardHandlerFactory ext
this.keepAliveTime = getParameter(args, MAX_THREAD_IDLE_TIME, keepAliveTime);
this.queueSize = getParameter(args, INIT_SIZE_OF_QUEUE, queueSize);
this.accessPolicy = getParameter(args, INIT_FAIRNESS_POLICY, accessPolicy);
+ this.useRetries = getParameter(args, USE_RETRIES, useRetries);
// magic sysprop to make tests reproducible: set by SolrTestCaseJ4.
String v = System.getProperty("tests.shardhandler.randomSeed");
@@ -154,8 +162,18 @@ public class HttpShardHandlerFactory ext
clientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS, maxConnections);
clientParams.set(HttpClientUtil.PROP_SO_TIMEOUT, soTimeout);
clientParams.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, connectionTimeout);
- clientParams.set(HttpClientUtil.PROP_USE_RETRY, false);
+ if (!useRetries) {
+ clientParams.set(HttpClientUtil.PROP_USE_RETRY, false);
+ }
this.defaultClient = HttpClientUtil.createClient(clientParams);
+
+ // must come after createClient
+ if (useRetries) {
+ // our default retry handler will never retry on IOException if the request has been sent already,
+ // but for these read only requests we can use the standard DefaultHttpRequestRetryHandler rules
+ ((DefaultHttpClient) this.defaultClient).setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler());
+ }
+
this.loadbalancer = createLoadbalancer(defaultClient);
}
Modified: lucene/dev/branches/lucene_solr_5_0/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_5_0/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java?rev=1651775&r1=1651774&r2=1651775&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_5_0/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java (original)
+++ lucene/dev/branches/lucene_solr_5_0/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java Wed Jan 14 19:05:08 2015
@@ -61,7 +61,10 @@ public class UpdateShardHandler {
params.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT,
cfg.getDistributedConnectionTimeout());
}
- params.set(HttpClientUtil.PROP_USE_RETRY, false);
+ // in the update case, we want to do retries, and to use
+ // the default Solr retry handler that createClient will
+ // give us
+ params.set(HttpClientUtil.PROP_USE_RETRY, true);
log.info("Creating UpdateShardHandler HTTP client with params: {}", params);
client = HttpClientUtil.createClient(params, clientConnectionManager);
}
Modified: lucene/dev/branches/lucene_solr_5_0/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientConfigurer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_5_0/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientConfigurer.java?rev=1651775&r1=1651774&r2=1651775&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_5_0/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientConfigurer.java (original)
+++ lucene/dev/branches/lucene_solr_5_0/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientConfigurer.java Wed Jan 14 19:05:08 2015
@@ -51,16 +51,15 @@ public class HttpClientConfigurer {
config.getInt(HttpClientUtil.PROP_SO_TIMEOUT));
}
- if (config.get(HttpClientUtil.PROP_USE_RETRY) != null) {
- HttpClientUtil.setUseRetry(httpClient,
- config.getBool(HttpClientUtil.PROP_USE_RETRY));
- }
-
if (config.get(HttpClientUtil.PROP_FOLLOW_REDIRECTS) != null) {
HttpClientUtil.setFollowRedirects(httpClient,
config.getBool(HttpClientUtil.PROP_FOLLOW_REDIRECTS));
}
+ // always call setUseRetry, whether it is in config or not
+ HttpClientUtil.setUseRetry(httpClient,
+ config.getBool(HttpClientUtil.PROP_USE_RETRY, true));
+
final String basicAuthUser = config
.get(HttpClientUtil.PROP_BASIC_AUTH_USER);
final String basicAuthPass = config
Modified: lucene/dev/branches/lucene_solr_5_0/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_5_0/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java?rev=1651775&r1=1651774&r2=1651775&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_5_0/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java (original)
+++ lucene/dev/branches/lucene_solr_5_0/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java Wed Jan 14 19:05:08 2015
@@ -241,7 +241,14 @@ public class HttpClientUtil {
if (!useRetry) {
httpClient.setHttpRequestRetryHandler(NO_RETRY);
} else {
- httpClient.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler());
+ // if the request is not fully sent, we retry
+ // streaming updates are not a problem, because they are not retryable
+ httpClient.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(){
+ @Override
+ protected boolean handleAsIdempotent(final HttpRequest request) {
+ return false; // we can't tell if a Solr request is idempotent
+ }
+ });
}
}
Modified: lucene/dev/branches/lucene_solr_5_0/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_5_0/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java?rev=1651775&r1=1651774&r2=1651775&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_5_0/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java (original)
+++ lucene/dev/branches/lucene_solr_5_0/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java Wed Jan 14 19:05:08 2015
@@ -223,7 +223,12 @@ public class LBHttpSolrClient extends So
this.parser = parser;
if (httpClient == null) {
ModifiableSolrParams params = new ModifiableSolrParams();
- params.set(HttpClientUtil.PROP_USE_RETRY, false);
+ if (solrServerUrl.length > 1) {
+ // we prefer retrying another server
+ params.set(HttpClientUtil.PROP_USE_RETRY, false);
+ } else {
+ params.set(HttpClientUtil.PROP_USE_RETRY, true);
+ }
this.httpClient = HttpClientUtil.createClient(params);
} else {
this.httpClient = httpClient;