You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jclouds.apache.org by ga...@apache.org on 2014/03/04 07:30:38 UTC
git commit: JCLOUDS-460: Add jitter to avoid thundering herd
Repository: jclouds
Updated Branches:
refs/heads/master 9f11888f4 -> 00847f9ff
JCLOUDS-460: Add jitter to avoid thundering herd
When issuing many simultaneous requests to Synaptic Atmos I observed:
HTTP/1.1 failed with code 500, error: AtmosError
[code=1040, message=The server is busy. Please try again.]
Previously all clients slept for fixed intervals and thus retried
around the same time. This commit adds a random delay which should
better distribute load on the provider.
Project: http://git-wip-us.apache.org/repos/asf/jclouds/repo
Commit: http://git-wip-us.apache.org/repos/asf/jclouds/commit/00847f9f
Tree: http://git-wip-us.apache.org/repos/asf/jclouds/tree/00847f9f
Diff: http://git-wip-us.apache.org/repos/asf/jclouds/diff/00847f9f
Branch: refs/heads/master
Commit: 00847f9ffc5fade4e63530b79c7d38962c1de08f
Parents: 9f11888
Author: Andrew Gaul <ga...@maginatics.com>
Authored: Thu Apr 18 12:54:00 2013 -0700
Committer: Andrew Gaul <ga...@apache.org>
Committed: Mon Mar 3 22:30:22 2014 -0800
----------------------------------------------------------------------
.../org/jclouds/http/handlers/BackoffLimitedRetryHandler.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/jclouds/blob/00847f9f/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java b/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java
index 5162042..5d8b5b1 100644
--- a/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java
+++ b/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java
@@ -19,6 +19,7 @@ package org.jclouds.http.handlers;
import static org.jclouds.http.HttpUtils.releasePayload;
import java.io.IOException;
+import java.util.Random;
import javax.annotation.Resource;
import javax.inject.Named;
@@ -126,6 +127,9 @@ public class BackoffLimitedRetryHandler implements HttpRetryHandler, IOException
public void imposeBackoffExponentialDelay(long period, long maxPeriod, int pow, int failureCount, int max,
String commandDescription) {
long delayMs = (long) (period * Math.pow(failureCount, pow));
+ // Add random delay to avoid thundering herd problem when multiple
+ // simultaneous failed requests retry after sleeping for the same delay.
+ delayMs += new Random().nextInt((int) (delayMs / 10));
delayMs = delayMs > maxPeriod ? maxPeriod : delayMs;
logger.debug("Retry %d/%d: delaying for %d ms: %s", failureCount, max, delayMs, commandDescription);
try {