You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hc.apache.org by mi...@apache.org on 2019/12/05 20:23:07 UTC
[httpcomponents-client] 01/08: First draft
This is an automated email from the ASF dual-hosted git repository.
michaelo pushed a commit to branch HTTPCLIENT-2034
in repository https://gitbox.apache.org/repos/asf/httpcomponents-client.git
commit 188065312ff8d440c79021bc628e9145f686127d
Author: Michael Osipov <mi...@apache.org>
AuthorDate: Thu Dec 5 17:57:52 2019 +0100
First draft
---
.../hc/client5/http/HttpRequestRetryStrategy.java | 89 ++++++++
.../apache/hc/client5/http/impl/ChainElements.java | 5 +-
.../http/impl/DefaultHttpRequestRetryStrategy.java | 234 +++++++++++++++++++++
.../http/impl/classic/HttpClientBuilder.java | 22 ++
.../http/impl/classic/HttpRequestRetryExec.java | 150 +++++++++++++
5 files changed, 498 insertions(+), 2 deletions(-)
diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/HttpRequestRetryStrategy.java b/httpclient5/src/main/java/org/apache/hc/client5/http/HttpRequestRetryStrategy.java
new file mode 100644
index 0000000..4c78d77
--- /dev/null
+++ b/httpclient5/src/main/java/org/apache/hc/client5/http/HttpRequestRetryStrategy.java
@@ -0,0 +1,89 @@
+/*
+ * ====================================================================
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * ====================================================================
+ *
+ * This software consists of voluntary contributions made by many
+ * individuals on behalf of the Apache Software Foundation. For more
+ * information on the Apache Software Foundation, please see
+ * <http://www.apache.org/>.
+ *
+ */
+
+package org.apache.hc.client5.http;
+
+import java.io.IOException;
+
+import org.apache.hc.core5.annotation.Contract;
+import org.apache.hc.core5.annotation.ThreadingBehavior;
+import org.apache.hc.core5.http.HttpRequest;
+import org.apache.hc.core5.http.HttpResponse;
+import org.apache.hc.core5.http.protocol.HttpContext;
+import org.apache.hc.core5.util.TimeValue;
+
+/**
+ * Strategy interface that allows API users to plug in their own logic to
+ * control whether or not a retry should automatically be done, how many times
+ * it should be done and so on.
+ *
+ * @since 5.0
+ */
+@Contract(threading = ThreadingBehavior.STATELESS)
+public interface HttpRequestRetryStrategy {
+
+ /**
+ * Determines if a method should be retried after an IOException
+ * occured during execution.
+ *
+ * @param request request failed die to an I/O exception.
+ * @param exception the exception that occurred
+ * @param executionCount the number of times this method has been
+ * unsuccessfully executed
+ * @param context the context for the request execution
+ *
+ * @return {@code true} if the method should be retried, {@code false}
+ * otherwise
+ */
+ boolean retryRequest(HttpRequest request, IOException exception, int executionCount, HttpContext context);
+
+ /**
+ * Determines if a method should be retried given the response from
+ * the target server.
+ *
+ * @param response the response from the target server
+ * @param executionCount the number of times this method has been
+ * unsuccessfully executed
+ * @param context the context for the request execution
+ * @return {@code true} if the method should be retried, {@code false}
+ * otherwise
+ */
+ boolean retryRequest(HttpResponse response, int executionCount, HttpContext context);
+
+ /**
+ * Determines the retry interval between subsequent retries.
+
+ * @param response the response from the target server
+ * @param executionCount the number of times this method has been
+ * unsuccessfully executed
+ * @param context the context for the request execution
+ *
+ * @return the retry interval between subsequent retries
+ */
+ TimeValue getRetryInterval(HttpResponse response, int executionCount, HttpContext context);
+
+}
diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/ChainElements.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/ChainElements.java
index 9dd95cd..9f034f5 100644
--- a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/ChainElements.java
+++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/ChainElements.java
@@ -34,6 +34,7 @@ package org.apache.hc.client5.http.impl;
*/
public enum ChainElements {
- REDIRECT, BACK_OFF, RETRY_SERVICE_UNAVAILABLE, RETRY_IO_ERROR, CACHING, PROTOCOL, CONNECT, MAIN_TRANSPORT
+ REDIRECT, BACK_OFF, RETRY, RETRY_SERVICE_UNAVAILABLE, RETRY_IO_ERROR, CACHING, PROTOCOL,
+ CONNECT, MAIN_TRANSPORT
-}
\ No newline at end of file
+}
diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/DefaultHttpRequestRetryStrategy.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/DefaultHttpRequestRetryStrategy.java
new file mode 100644
index 0000000..140a941
--- /dev/null
+++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/DefaultHttpRequestRetryStrategy.java
@@ -0,0 +1,234 @@
+/*
+ * ====================================================================
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * ====================================================================
+ *
+ * This software consists of voluntary contributions made by many
+ * individuals on behalf of the Apache Software Foundation. For more
+ * information on the Apache Software Foundation, please see
+ * <http://www.apache.org/>.
+ *
+ */
+
+package org.apache.hc.client5.http.impl;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.ConnectException;
+import java.net.UnknownHostException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.Set;
+
+import javax.net.ssl.SSLException;
+
+import org.apache.hc.client5.http.HttpRequestRetryStrategy;
+import org.apache.hc.client5.http.utils.DateUtils;
+import org.apache.hc.core5.annotation.Contract;
+import org.apache.hc.core5.annotation.ThreadingBehavior;
+import org.apache.hc.core5.concurrent.CancellableDependency;
+import org.apache.hc.core5.http.ConnectionClosedException;
+import org.apache.hc.core5.http.Header;
+import org.apache.hc.core5.http.HttpHeaders;
+import org.apache.hc.core5.http.HttpRequest;
+import org.apache.hc.core5.http.HttpResponse;
+import org.apache.hc.core5.http.HttpStatus;
+import org.apache.hc.core5.http.Methods;
+import org.apache.hc.core5.http.protocol.HttpContext;
+import org.apache.hc.core5.util.Args;
+import org.apache.hc.core5.util.TimeValue;
+
+/**
+ * Default implementation of the {@link HttpRequestRetryStrategy} interface.
+ *
+ * @since 5.0
+ */
+@Contract(threading = ThreadingBehavior.STATELESS)
+public class DefaultHttpRequestRetryStrategy implements HttpRequestRetryStrategy {
+
+ public static final DefaultHttpRequestRetryStrategy INSTANCE = new DefaultHttpRequestRetryStrategy();
+
+ /**
+ * Maximum number of allowed retries
+ */
+ private final int maxRetries;
+
+ /**
+ * Retry interval between subsequent retries
+ */
+ private final TimeValue defaultRetryInterval;
+
+ /**
+ * Derived {@code IOExceptions} which shall not be retried
+ */
+ private final Set<Class<? extends IOException>> nonRetriableClasses;
+
+ /**
+ * HTTP status codes which shall be retried
+ */
+ private final Set<Integer> retriableCodes;
+
+ protected DefaultHttpRequestRetryStrategy(
+ final int maxRetries,
+ final TimeValue defaultRetryInterval,
+ final Collection<Class<? extends IOException>> clazzes,
+ final Collection<Integer> codes) {
+ Args.positive(maxRetries, "maxRetries");
+ Args.positive(defaultRetryInterval.getDuration(), "defaultRetryInterval");
+ this.maxRetries = maxRetries;
+ this.defaultRetryInterval = defaultRetryInterval;
+ this.nonRetriableClasses = new HashSet<>();
+ this.nonRetriableClasses.addAll(clazzes);
+ this.retriableCodes = new HashSet<>();
+ this.retriableCodes.addAll(codes);
+ }
+
+ /**
+ * Create the HTTP request retry strategy using the following list of
+ * non-retriable IOException classes:<br>
+ * <ul>
+ * <li>InterruptedIOException</li>
+ * <li>UnknownHostException</li>
+ * <li>ConnectException</li>
+ * <li>ConnectionClosedException</li>
+ * <li>SSLException</li>
+ * </ul>
+ *
+ * and retriable HTTP status codes:<br>
+ * <ul>
+ * <li>SC_TOO_MANY_REQUESTS (429)</li>
+ * <li>SC_SERVICE_UNAVAILABLE (503)</li>
+ * </ul>
+ *
+ * @param maxRetries how many times to retry; 0 means no retries
+ * @param defaultRetryInterval the default retry interval between
+ * subsequent retries if the {@code Retry-After} header is not set
+ * or invalid.
+ */
+ public DefaultHttpRequestRetryStrategy(
+ final int maxRetries,
+ final TimeValue defaultRetryInterval) {
+ this(maxRetries, defaultRetryInterval,
+ Arrays.asList(
+ InterruptedIOException.class,
+ UnknownHostException.class,
+ ConnectException.class,
+ ConnectionClosedException.class,
+ SSLException.class),
+ Arrays.asList(
+ HttpStatus.SC_TOO_MANY_REQUESTS,
+ HttpStatus.SC_SERVICE_UNAVAILABLE));
+ }
+
+ /**
+ * Create the HTTP request retry strategy with a max retry count of 1,
+ * default retry interval of 1 second, and using the following list of
+ * non-retriable IOException classes:<br>
+ * <ul>
+ * <li>InterruptedIOException</li>
+ * <li>UnknownHostException</li>
+ * <li>ConnectException</li>
+ * <li>ConnectionClosedException</li>
+ * <li>SSLException</li>
+ * </ul>
+ *
+ * and retriable HTTP status codes:<br>
+ * <ul>
+ * <li>SC_TOO_MANY_REQUESTS (429)</li>
+ * <li>SC_SERVICE_UNAVAILABLE (503)</li>
+ * </ul>
+ */
+ public DefaultHttpRequestRetryStrategy() {
+ this(1, TimeValue.ofSeconds(1L));
+ }
+
+ @Override
+ public boolean retryRequest(
+ final HttpRequest request,
+ final IOException exception,
+ final int executionCount,
+ final HttpContext context) {
+ Args.notNull(request, "request");
+ Args.notNull(exception, "exception");
+
+ if (executionCount > this.maxRetries) {
+ // Do not retry if over max retries
+ return false;
+ }
+ if (this.nonRetriableClasses.contains(exception.getClass())) {
+ return false;
+ } else {
+ for (final Class<? extends IOException> rejectException : this.nonRetriableClasses) {
+ if (rejectException.isInstance(exception)) {
+ return false;
+ }
+ }
+ }
+ if (request instanceof CancellableDependency && ((CancellableDependency) request).isCancelled()) {
+ return false;
+ }
+
+ // Retry if the request is considered idempotent
+ return handleAsIdempotent(request);
+ }
+
+ @Override
+ public boolean retryRequest(
+ final HttpResponse response,
+ final int executionCount,
+ final HttpContext context) {
+ Args.notNull(response, "response");
+
+ return executionCount <= this.maxRetries && retriableCodes.contains(response.getCode());
+ }
+
+ @Override
+ public TimeValue getRetryInterval(
+ final HttpResponse response,
+ final int executionCount,
+ final HttpContext context) {
+ Args.notNull(response, "response");
+
+ final Header header = response.getFirstHeader(HttpHeaders.RETRY_AFTER);
+ TimeValue retryAfter = null;
+ if (header != null) {
+ final String value = header.getValue();
+ try {
+ retryAfter = TimeValue.ofSeconds(Long.parseLong(value));
+ } catch (final NumberFormatException ignore) {
+ final Date retryAfterDate = DateUtils.parseDate(value);
+ if (retryAfterDate != null) {
+ retryAfter =
+ TimeValue.ofMilliseconds(retryAfterDate.getTime() - System.currentTimeMillis());
+ }
+ }
+
+ if (TimeValue.isPositive(retryAfter)) {
+ return retryAfter;
+ }
+ }
+ return this.defaultRetryInterval;
+ }
+
+ protected boolean handleAsIdempotent(final HttpRequest request) {
+ return Methods.isIdempotent(request.getMethod());
+ }
+
+}
diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/classic/HttpClientBuilder.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/classic/HttpClientBuilder.java
index cae1e0f..610e5ed 100644
--- a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/classic/HttpClientBuilder.java
+++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/classic/HttpClientBuilder.java
@@ -40,6 +40,7 @@ import java.util.Map;
import org.apache.hc.client5.http.AuthenticationStrategy;
import org.apache.hc.client5.http.ConnectionKeepAliveStrategy;
import org.apache.hc.client5.http.HttpRequestRetryHandler;
+import org.apache.hc.client5.http.HttpRequestRetryStrategy;
import org.apache.hc.client5.http.SchemePortResolver;
import org.apache.hc.client5.http.ServiceUnavailableRetryStrategy;
import org.apache.hc.client5.http.SystemDefaultDnsResolver;
@@ -61,6 +62,7 @@ import org.apache.hc.client5.http.impl.CookieSpecSupport;
import org.apache.hc.client5.http.impl.DefaultAuthenticationStrategy;
import org.apache.hc.client5.http.impl.DefaultConnectionKeepAliveStrategy;
import org.apache.hc.client5.http.impl.DefaultHttpRequestRetryHandler;
+import org.apache.hc.client5.http.impl.DefaultHttpRequestRetryStrategy;
import org.apache.hc.client5.http.impl.DefaultRedirectStrategy;
import org.apache.hc.client5.http.impl.DefaultSchemePortResolver;
import org.apache.hc.client5.http.impl.DefaultUserTokenHandler;
@@ -204,6 +206,7 @@ public class HttpClientBuilder {
private LinkedList<ExecInterceptorEntry> execInterceptors;
private HttpRequestRetryHandler retryHandler;
+ private HttpRequestRetryStrategy retryStrategy;
private HttpRoutePlanner routePlanner;
private RedirectStrategy redirectStrategy;
private ConnectionBackoffStrategy connectionBackoffStrategy;
@@ -506,6 +509,17 @@ public class HttpClientBuilder {
}
/**
+ * Assigns {@link HttpRequestRetryStrategy} instance.
+ * <p>
+ * Please note this value can be overridden by the {@link #disableAutomaticRetries()}
+ * method.
+ */
+ public final HttpClientBuilder setRetryStrategy(final HttpRequestRetryStrategy retryStrategy) {
+ this.retryStrategy = retryStrategy;
+ return this;
+ }
+
+ /**
* Disables automatic request recovery and re-execution.
*/
public final HttpClientBuilder disableAutomaticRetries() {
@@ -866,6 +880,14 @@ public class HttpClientBuilder {
execChainDefinition.addFirst(
new RetryExec(retryHandlerCopy),
ChainElements.RETRY_IO_ERROR.name());
+
+ HttpRequestRetryStrategy retryStrategyCopy = this.retryStrategy;
+ if (retryStrategyCopy == null) {
+ retryStrategyCopy = DefaultHttpRequestRetryStrategy.INSTANCE;
+ }
+ execChainDefinition.addFirst(
+ new HttpRequestRetryExec(retryStrategyCopy),
+ ChainElements.RETRY.name());
}
HttpRoutePlanner routePlannerCopy = this.routePlanner;
diff --git a/httpclient5/src/main/java/org/apache/hc/client5/http/impl/classic/HttpRequestRetryExec.java b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/classic/HttpRequestRetryExec.java
new file mode 100644
index 0000000..5aa81c8
--- /dev/null
+++ b/httpclient5/src/main/java/org/apache/hc/client5/http/impl/classic/HttpRequestRetryExec.java
@@ -0,0 +1,150 @@
+/*
+ * ====================================================================
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * ====================================================================
+ *
+ * This software consists of voluntary contributions made by many
+ * individuals on behalf of the Apache Software Foundation. For more
+ * information on the Apache Software Foundation, please see
+ * <http://www.apache.org/>.
+ *
+ */
+package org.apache.hc.client5.http.impl.classic;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+
+import org.apache.hc.client5.http.HttpRequestRetryStrategy;
+import org.apache.hc.client5.http.HttpRoute;
+import org.apache.hc.client5.http.classic.ExecChain;
+import org.apache.hc.client5.http.classic.ExecChain.Scope;
+import org.apache.hc.client5.http.protocol.HttpClientContext;
+import org.apache.hc.client5.http.classic.ExecChainHandler;
+import org.apache.hc.core5.annotation.Contract;
+import org.apache.hc.core5.annotation.Internal;
+import org.apache.hc.core5.annotation.ThreadingBehavior;
+import org.apache.hc.core5.http.ClassicHttpRequest;
+import org.apache.hc.core5.http.ClassicHttpResponse;
+import org.apache.hc.core5.http.HttpEntity;
+import org.apache.hc.core5.http.HttpException;
+import org.apache.hc.core5.http.NoHttpResponseException;
+import org.apache.hc.core5.util.Args;
+import org.apache.hc.core5.util.TimeValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Request executor in the request execution chain that is responsible for
+ * making a decision whether a request that failed due to an I/O error
+ * or received specific response status codes from the target server should
+ * be re-executed.
+ * <p>
+ * Further responsibilities such as communication with the opposite
+ * endpoint is delegated to the next executor in the request execution
+ * chain.
+ * </p>
+ *
+ * @since 5.0
+ */
+@Contract(threading = ThreadingBehavior.STATELESS)
+@Internal
+public class HttpRequestRetryExec implements ExecChainHandler {
+
+ private final Logger log = LoggerFactory.getLogger(getClass());
+
+ private final HttpRequestRetryStrategy retryStrategy;
+
+ public HttpRequestRetryExec(
+ final HttpRequestRetryStrategy retryStrategy) {
+ Args.notNull(retryStrategy, "retryStrategy");
+ this.retryStrategy = retryStrategy;
+ }
+
+ @Override
+ public ClassicHttpResponse execute(
+ final ClassicHttpRequest request,
+ final Scope scope,
+ final ExecChain chain) throws IOException, HttpException {
+ Args.notNull(request, "request");
+ Args.notNull(scope, "scope");
+ final String exchangeId = scope.exchangeId;
+ final HttpRoute route = scope.route;
+ final HttpClientContext context = scope.clientContext;
+ ClassicHttpRequest currentRequest = request;
+
+ for (int execCount = 1;; execCount++) {
+ ClassicHttpResponse response;
+ try {
+ response = chain.proceed(currentRequest, scope);
+ } catch (final IOException ex) {
+ if (scope.execRuntime.isExecutionAborted()) {
+ throw new RequestFailedException("Request aborted");
+ }
+ final HttpEntity requestEntity = request.getEntity();
+ if (requestEntity != null && !requestEntity.isRepeatable()) {
+ log.debug("{}: cannot retry non-repeatable request", exchangeId);
+ throw ex;
+ }
+ if (retryStrategy.retryRequest(request, ex, execCount, context)) {
+ log.debug("{}: {}", exchangeId, ex.getMessage(), ex);
+ log.info("Recoverable I/O exception ({}) caught when processing request to {}",
+ ex.getClass().getName(), route);
+ currentRequest = ClassicRequestCopier.INSTANCE.copy(scope.originalRequest);
+ continue;
+ } else {
+ if (ex instanceof NoHttpResponseException) {
+ final NoHttpResponseException updatedex = new NoHttpResponseException(
+ route.getTargetHost().toHostString() + " failed to respond");
+ updatedex.setStackTrace(ex.getStackTrace());
+ throw updatedex;
+ }
+ throw ex;
+ }
+ }
+
+ try {
+ final HttpEntity entity = request.getEntity();
+ if (entity != null && !entity.isRepeatable()) {
+ log.debug("{}: cannot retry non-repeatable request", exchangeId);
+ return response;
+ }
+ if (retryStrategy.retryRequest(response, execCount, context)) {
+ response.close();
+ final TimeValue nextInterval =
+ retryStrategy.getRetryInterval(response, execCount, context);
+ if (TimeValue.isPositive(nextInterval)) {
+ try {
+ log.debug("{}: wait for {}", exchangeId, nextInterval);
+ nextInterval.sleep();
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new InterruptedIOException();
+ }
+ }
+ currentRequest = ClassicRequestCopier.INSTANCE.copy(scope.originalRequest);
+ } else {
+ return response;
+ }
+ } catch (final RuntimeException ex) {
+ response.close();
+ throw ex;
+ }
+ }
+ }
+
+}