You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@brooklyn.apache.org by ge...@apache.org on 2018/01/24 13:52:10 UTC
[1/2] brooklyn-server git commit: jclouds polling/backoff: increase
delays
Repository: brooklyn-server
Updated Branches:
refs/heads/master c50fee8d1 -> 7c365dfaa
jclouds polling/backoff: increase delays
Project: http://git-wip-us.apache.org/repos/asf/brooklyn-server/repo
Commit: http://git-wip-us.apache.org/repos/asf/brooklyn-server/commit/2d930373
Tree: http://git-wip-us.apache.org/repos/asf/brooklyn-server/tree/2d930373
Diff: http://git-wip-us.apache.org/repos/asf/brooklyn-server/diff/2d930373
Branch: refs/heads/master
Commit: 2d93037336f3e9cb7a07250f51016979e2e26a3b
Parents: c50fee8
Author: Aled Sage <al...@gmail.com>
Authored: Wed Jan 24 12:49:57 2018 +0000
Committer: Aled Sage <al...@gmail.com>
Committed: Wed Jan 24 12:49:57 2018 +0000
----------------------------------------------------------------------
.../jclouds/JCloudsPropertiesBuilder.java | 56 ++++++++++++++++----
1 file changed, 46 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/2d930373/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JCloudsPropertiesBuilder.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JCloudsPropertiesBuilder.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JCloudsPropertiesBuilder.java
index 8178134..0829be4 100644
--- a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JCloudsPropertiesBuilder.java
+++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JCloudsPropertiesBuilder.java
@@ -33,6 +33,7 @@ import org.apache.brooklyn.util.core.config.ConfigBag;
import org.apache.brooklyn.util.text.Strings;
import org.apache.brooklyn.util.time.Duration;
import org.jclouds.Constants;
+import org.jclouds.compute.config.ComputeServiceProperties;
import org.jclouds.ec2.reference.EC2Constants;
import org.jclouds.location.reference.LocationConstants;
import org.slf4j.Logger;
@@ -60,16 +61,51 @@ public class JCloudsPropertiesBuilder implements JcloudsLocationConfig{
if (conf.get(OAUTH_ENDPOINT) != null)
properties.setProperty(OAUTH_ENDPOINT.getName(), conf.get(OAUTH_ENDPOINT));
- // See https://issues.apache.org/jira/browse/BROOKLYN-394
- // For retries, the backoff times are:
- // Math.min(2^failureCount * retryDelayStart, retryDelayStart * 10) + random(10%)
- // Therefore the backoff times will be: 500ms, 1s, 2s, 4s, 5s, 5s.
- // The defaults (if not overridden here) are 50ms and 5 retires. This gives backoff
- // times of 50ms, 100ms, 200ms, 400ms, 500ms (so a total backoff time of 1.25s),
- // which is not long when you're being rate-limited and there are multiple thread all
- // retrying their API calls.
- properties.setProperty(Constants.PROPERTY_RETRY_DELAY_START, "500");
- properties.setProperty(Constants.PROPERTY_MAX_RETRIES, "6");
+ /*
+ * See https://issues.apache.org/jira/browse/BROOKLYN-394
+ * For retries, the backoff times are:
+ * Math.min(2^failureCount * retryDelayStart, retryDelayStart * 10) + random(10%)
+ * The overridden Brooklyn defaults give backoffs of:
+ * 2s, 4s, 8s, 16s, 20s, 20s, 20s, 20s, 20s, 20s
+ * (i.e. we'll keep trying for approx 2mins 30secs).
+ *
+ * The jclouds defaults (if not overridden here) are 50ms and 5 retires. This gives backoff
+ * times of:
+ * 50ms, 100ms, 200ms, 400ms, 500ms
+ * That is only a total of 1.25s, which is not long when you're being rate-limited
+ * and there are multiple thread all retrying their API calls.
+ *
+ * Further improvements/considerations include:
+ * - Users with huge deployments may want to backoff for even longer, e.g. setting:
+ * jclouds.retries-delay-start: 3s
+ * jclouds.max-retries: 20
+ * - The jitter could be improved to include more randomness, but that is not yet
+ * configurable in jclouds. See:
+ * - https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/
+ * - https://docs.aws.amazon.com/general/latest/gr/api-retries.html
+ * - https://github.com/jclouds/jclouds/blob/rel/jclouds-2.0.2/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java#L135-L136
+ */
+ properties.setProperty(Constants.PROPERTY_RETRY_DELAY_START, "2000");
+ properties.setProperty(Constants.PROPERTY_MAX_RETRIES, "10");
+
+ /*
+ * Controls polling for the VM status as 'running', when provisioning a new VM.
+ * The backoff times are calculated as:
+ * Math.min(1.5^(attempt-1) * initialPeriod, maxPeriod)
+ * See:
+ * https://github.com/jclouds/jclouds/blob/rel/jclouds-2.0.2/core/src/main/java/org/jclouds/util/Predicates2.java#L157-L172
+ * The overridden Brooklyn defaults give backoffs of:
+ * 1s, 1.5s, 2s, 2s, ...
+ *
+ * Defaults in jclouds are initial 50ms, backing off to max 1s. That would give backoffs of:
+ * 50ms, 75ms, 112ms, 169ms, 253ms, 380ms, 570ms, 854ms, 1s, ...
+ * i.e. approx 5 calls in the first second, and another 2 calls in the next second, etc.
+ * For detecting completion of VM provisioning (which even in a fast cloud currently
+ * takes > 10s), that is far more load than is sensible with default settings.
+ */
+ properties.setProperty(ComputeServiceProperties.POLL_INITIAL_PERIOD, "1000");
+ properties.setProperty(ComputeServiceProperties.POLL_MAX_PERIOD, "2000");
+
return this;
}
[2/2] brooklyn-server git commit: Closes #941
Posted by ge...@apache.org.
Closes #941
jclouds polling/backoff: increase delays
Motivated by experience with a large customer doing scalability testing, where they were rate limited a lot (despite having agreed in advance with AWS to increase their permitted limits). That customer had been using a bunch of overrides. However, such discussions highlighted the need for us to change the default behaviour in Brooklyn so that small/medium size users don't normally need to think about such things.
Project: http://git-wip-us.apache.org/repos/asf/brooklyn-server/repo
Commit: http://git-wip-us.apache.org/repos/asf/brooklyn-server/commit/7c365dfa
Tree: http://git-wip-us.apache.org/repos/asf/brooklyn-server/tree/7c365dfa
Diff: http://git-wip-us.apache.org/repos/asf/brooklyn-server/diff/7c365dfa
Branch: refs/heads/master
Commit: 7c365dfaabdf72ddb6438df54cba765e1f6fe850
Parents: c50fee8 2d93037
Author: Geoff Macartney <ge...@cloudsoftcorp.com>
Authored: Wed Jan 24 13:52:07 2018 +0000
Committer: Geoff Macartney <ge...@cloudsoftcorp.com>
Committed: Wed Jan 24 13:52:07 2018 +0000
----------------------------------------------------------------------
.../jclouds/JCloudsPropertiesBuilder.java | 56 ++++++++++++++++----
1 file changed, 46 insertions(+), 10 deletions(-)
----------------------------------------------------------------------