You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2016/02/10 16:41:52 UTC
[1/4] lucene-solr git commit: SOLR-8450: Our HttpClient retry policy
is too permissive.
Repository: lucene-solr
Updated Branches:
refs/heads/branch_5x 0417cc41d -> df004731d
SOLR-8450: Our HttpClient retry policy is too permissive.
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1723616 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f72c3e3a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f72c3e3a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f72c3e3a
Branch: refs/heads/branch_5x
Commit: f72c3e3a8ff4bdfc65764a180227c72c43429856
Parents: 0417cc4
Author: Mark Robert Miller <ma...@apache.org>
Authored: Thu Jan 7 19:43:09 2016 +0000
Committer: markrmiller <ma...@apache.org>
Committed: Wed Feb 10 10:35:10 2016 -0500
----------------------------------------------------------------------
.../solr/client/solrj/impl/HttpClientUtil.java | 7 +-
.../solrj/impl/SolrHttpRequestRetryHandler.java | 157 +++++++++++++++++++
2 files changed, 158 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f72c3e3a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java
index 9f68754..2ad8975 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java
@@ -263,12 +263,7 @@ public class HttpClientUtil {
} else {
// if the request is not fully sent, we retry
// streaming updates are not a problem, because they are not retryable
- httpClient.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(){
- @Override
- protected boolean handleAsIdempotent(final HttpRequest request) {
- return false; // we can't tell if a Solr request is idempotent
- }
- });
+ httpClient.setHttpRequestRetryHandler(new SolrHttpRequestRetryHandler(3));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f72c3e3a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrHttpRequestRetryHandler.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrHttpRequestRetryHandler.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrHttpRequestRetryHandler.java
new file mode 100644
index 0000000..a9307cf
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrHttpRequestRetryHandler.java
@@ -0,0 +1,157 @@
+package org.apache.solr.client.solrj.impl;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.lang.invoke.MethodHandles;
+import java.net.ConnectException;
+import java.net.UnknownHostException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+import javax.net.ssl.SSLException;
+
+import org.apache.http.HttpRequest;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.impl.client.RequestWrapper;
+import org.apache.http.protocol.HttpContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SolrHttpRequestRetryHandler implements HttpRequestRetryHandler {
+
+ private static final String GET = "GET";
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public static final SolrHttpRequestRetryHandler INSTANCE = new SolrHttpRequestRetryHandler();
+
+ /** the number of times a method will be retried */
+ private final int retryCount;
+
+ private final Set<Class<? extends IOException>> nonRetriableClasses;
+
+ /**
+ * Create the request retry handler using the specified IOException classes
+ *
+ * @param retryCount
+ * how many times to retry; 0 means no retries
+ * true if it's OK to retry requests that have been sent
+ * @param clazzes
+ * the IOException types that should not be retried
+ */
+ protected SolrHttpRequestRetryHandler(final int retryCount, final Collection<Class<? extends IOException>> clazzes) {
+ super();
+ this.retryCount = retryCount;
+ this.nonRetriableClasses = new HashSet<Class<? extends IOException>>();
+ for (final Class<? extends IOException> clazz : clazzes) {
+ this.nonRetriableClasses.add(clazz);
+ }
+ }
+
+ /**
+ * Create the request retry handler using the following list of non-retriable IOException classes: <br>
+ * <ul>
+ * <li>InterruptedIOException</li>
+ * <li>UnknownHostException</li>
+ * <li>ConnectException</li>
+ * <li>SSLException</li>
+ * </ul>
+ *
+ * @param retryCount
+ * how many times to retry; 0 means no retries
+ * true if it's OK to retry non-idempotent requests that have been sent
+ */
+ @SuppressWarnings("unchecked")
+ public SolrHttpRequestRetryHandler(final int retryCount) {
+ this(retryCount, Arrays.asList(InterruptedIOException.class, UnknownHostException.class,
+ ConnectException.class, SSLException.class));
+ }
+
+ /**
+ * Create the request retry handler with a retry count of 3, requestSentRetryEnabled false and using the following
+ * list of non-retriable IOException classes: <br>
+ * <ul>
+ * <li>InterruptedIOException</li>
+ * <li>UnknownHostException</li>
+ * <li>ConnectException</li>
+ * <li>SSLException</li>
+ * </ul>
+ */
+ public SolrHttpRequestRetryHandler() {
+ this(3);
+ }
+
+ @Override
+ public boolean retryRequest(final IOException exception, final int executionCount, final HttpContext context) {
+ log.debug("Retry http request {} out of {}", executionCount, this.retryCount);
+ if (executionCount > this.retryCount) {
+ log.debug("Do not retry, over max retry count");
+ return false;
+ }
+ if (this.nonRetriableClasses.contains(exception.getClass())) {
+ log.debug("Do not retry, non retriable class {}" + exception.getClass());
+ return false;
+ } else {
+ for (final Class<? extends IOException> rejectException : this.nonRetriableClasses) {
+ if (rejectException.isInstance(exception)) {
+ log.debug("Do not retry, non retriable class {}" + exception.getClass());
+ return false;
+ }
+ }
+ }
+ final HttpClientContext clientContext = HttpClientContext.adapt(context);
+ final HttpRequest request = clientContext.getRequest();
+
+ if (requestIsAborted(request)) {
+ log.debug("Do not retry, request was aborted");
+ return false;
+ }
+
+ if (handleAsIdempotent(clientContext)) {
+ log.debug("Retry, request should be idempotent");
+ return true;
+ }
+
+ log.debug("Do not retry, no rules matched");
+ return false;
+ }
+
+ public int getRetryCount() {
+ return retryCount;
+ }
+
+ protected boolean handleAsIdempotent(final HttpClientContext context) {
+ String method = context.getRequest().getRequestLine().getMethod();
+ return method.equals(GET);
+ }
+
+ protected boolean requestIsAborted(final HttpRequest request) {
+ HttpRequest req = request;
+ if (request instanceof RequestWrapper) { // does not forward request to original
+ req = ((RequestWrapper) request).getOriginal();
+ }
+ return (req instanceof HttpUriRequest && ((HttpUriRequest) req).isAborted());
+ }
+
+}
\ No newline at end of file
[4/4] lucene-solr git commit: SOLR-8450: Do not retry admin requests,
they are not idempotent.
Posted by ma...@apache.org.
SOLR-8450: Do not retry admin requests, they are not idempotent.
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1724813 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/df004731
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/df004731
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/df004731
Branch: refs/heads/branch_5x
Commit: df004731dd3f2b1cd54af96f1aa1ead27fcf70b2
Parents: 9da8846
Author: Mark Robert Miller <ma...@apache.org>
Authored: Fri Jan 15 14:13:36 2016 +0000
Committer: markrmiller <ma...@apache.org>
Committed: Wed Feb 10 10:37:31 2016 -0500
----------------------------------------------------------------------
.../solr/client/solrj/impl/SolrHttpRequestRetryHandler.java | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df004731/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrHttpRequestRetryHandler.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrHttpRequestRetryHandler.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrHttpRequestRetryHandler.java
index f2ed8ac..1e43c04 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrHttpRequestRetryHandler.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrHttpRequestRetryHandler.java
@@ -133,7 +133,7 @@ public class SolrHttpRequestRetryHandler implements HttpRequestRetryHandler {
return true;
}
- log.debug("Do not retry, no rules matched");
+ log.debug("Do not retry, no allow rules matched");
return false;
}
@@ -143,7 +143,11 @@ public class SolrHttpRequestRetryHandler implements HttpRequestRetryHandler {
protected boolean handleAsIdempotent(final HttpClientContext context) {
String method = context.getRequest().getRequestLine().getMethod();
- context.getRequest().getRequestLine().getUri();
+ // do not retry admin requests, even if they are GET as they are not idempotent
+ if (context.getRequest().getRequestLine().getUri().startsWith("/admin/")) {
+ log.debug("Do not retry, this is an admin request");
+ return false;
+ }
return method.equals(GET);
}
[3/4] lucene-solr git commit: SOLR-8450: Fix debug logging.
Posted by ma...@apache.org.
SOLR-8450: Fix debug logging.
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1724807 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9da88465
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9da88465
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9da88465
Branch: refs/heads/branch_5x
Commit: 9da8846535bce8793b28a7d8816e10f3b1c5a912
Parents: 42c17d2
Author: Mark Robert Miller <ma...@apache.org>
Authored: Fri Jan 15 13:53:15 2016 +0000
Committer: markrmiller <ma...@apache.org>
Committed: Wed Feb 10 10:37:07 2016 -0500
----------------------------------------------------------------------
.../solr/client/solrj/impl/SolrHttpRequestRetryHandler.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9da88465/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrHttpRequestRetryHandler.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrHttpRequestRetryHandler.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrHttpRequestRetryHandler.java
index a9307cf..f2ed8ac 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrHttpRequestRetryHandler.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrHttpRequestRetryHandler.java
@@ -110,12 +110,12 @@ public class SolrHttpRequestRetryHandler implements HttpRequestRetryHandler {
return false;
}
if (this.nonRetriableClasses.contains(exception.getClass())) {
- log.debug("Do not retry, non retriable class {}" + exception.getClass());
+ log.debug("Do not retry, non retriable class {}", exception.getClass().getName());
return false;
} else {
for (final Class<? extends IOException> rejectException : this.nonRetriableClasses) {
if (rejectException.isInstance(exception)) {
- log.debug("Do not retry, non retriable class {}" + exception.getClass());
+ log.debug("Do not retry, non retriable class {}", exception.getClass().getName());
return false;
}
}
@@ -143,6 +143,7 @@ public class SolrHttpRequestRetryHandler implements HttpRequestRetryHandler {
protected boolean handleAsIdempotent(final HttpClientContext context) {
String method = context.getRequest().getRequestLine().getMethod();
+ context.getRequest().getRequestLine().getUri();
return method.equals(GET);
}
[2/4] lucene-solr git commit: SOLR-8450: Add CHANGES entry.
Posted by ma...@apache.org.
SOLR-8450: Add CHANGES entry.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/42c17d22
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/42c17d22
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/42c17d22
Branch: refs/heads/branch_5x
Commit: 42c17d225e2190893227ac6c710effda000afd98
Parents: f72c3e3
Author: markrmiller <ma...@apache.org>
Authored: Wed Feb 10 10:36:43 2016 -0500
Committer: markrmiller <ma...@apache.org>
Committed: Wed Feb 10 10:36:43 2016 -0500
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/42c17d22/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index b3caf7b..7291ad0 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -244,6 +244,8 @@ Bug Fixes
* SOLR-8451: We should not call method.abort in HttpSolrClient or HttpSolrCall#remoteQuery and
HttpSolrCall#remoteQuery should not close streams. (Mark Miller)
+* SOLR-8450: Our HttpClient retry policy is too permissive. (Mark Miller, shalin)
+
* SOLR-8533: Raise default maxUpdateConnections and maxUpdateConnectionsPerHost to 100k each.
(Mark Miller)