You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jclouds.apache.org by na...@apache.org on 2016/10/28 13:34:46 UTC
jclouds-labs git commit: RateLimit module for ProfitBricks REST
Repository: jclouds-labs
Updated Branches:
refs/heads/master 52e8f9e44 -> 436c001fb
RateLimit module for ProfitBricks REST
Added a module that automatically deals with rate limits in the
ProfitBricks REST api and throttles the requests as needed.
The module is configured by default in the live tests, since I've
observed several failures due to rate limits when using my account, but
not added by default. Users that want jclouds to deal automatically with
this will have to register the module when creating the context.
Project: http://git-wip-us.apache.org/repos/asf/jclouds-labs/repo
Commit: http://git-wip-us.apache.org/repos/asf/jclouds-labs/commit/436c001f
Tree: http://git-wip-us.apache.org/repos/asf/jclouds-labs/tree/436c001f
Diff: http://git-wip-us.apache.org/repos/asf/jclouds-labs/diff/436c001f
Branch: refs/heads/master
Commit: 436c001fbc10857104cd1c4a8f74da64a18bd74d
Parents: 52e8f9e
Author: Ignasi Barrera <na...@apache.org>
Authored: Wed Oct 26 15:35:40 2016 +0200
Committer: Ignasi Barrera <na...@apache.org>
Committed: Wed Oct 26 15:35:40 2016 +0200
----------------------------------------------------------------------
.../config/ProfitBricksRateLimitModule.java | 30 ++++++++
.../ProfitBricksRateLimitExceededException.java | 79 ++++++++++++++++++++
.../handlers/ProfitBricksHttpErrorHandler.java | 7 +-
.../ProfitBricksRateLimitRetryHandler.java | 39 ++++++++++
.../ProfitBricksComputeServiceLiveTest.java | 10 +++
.../ProfitBricksTemplateBuilderLiveTest.java | 11 +++
.../rest/internal/BaseProfitBricksLiveTest.java | 8 ++
7 files changed, 181 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/jclouds-labs/blob/436c001f/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/config/ProfitBricksRateLimitModule.java
----------------------------------------------------------------------
diff --git a/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/config/ProfitBricksRateLimitModule.java b/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/config/ProfitBricksRateLimitModule.java
new file mode 100644
index 0000000..3f100e4
--- /dev/null
+++ b/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/config/ProfitBricksRateLimitModule.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jclouds.profitbricks.rest.config;
+
+import org.apache.jclouds.profitbricks.rest.handlers.ProfitBricksRateLimitRetryHandler;
+import org.jclouds.http.HttpRetryHandler;
+import org.jclouds.http.annotation.ClientError;
+
+import com.google.inject.AbstractModule;
+
+public class ProfitBricksRateLimitModule extends AbstractModule {
+ @Override
+ protected void configure() {
+ bind(HttpRetryHandler.class).annotatedWith(ClientError.class).to(ProfitBricksRateLimitRetryHandler.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/jclouds-labs/blob/436c001f/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/exceptions/ProfitBricksRateLimitExceededException.java
----------------------------------------------------------------------
diff --git a/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/exceptions/ProfitBricksRateLimitExceededException.java b/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/exceptions/ProfitBricksRateLimitExceededException.java
new file mode 100644
index 0000000..5f0af6f
--- /dev/null
+++ b/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/exceptions/ProfitBricksRateLimitExceededException.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jclouds.profitbricks.rest.exceptions;
+
+import org.jclouds.http.HttpResponse;
+import org.jclouds.rest.RateLimitExceededException;
+
+import com.google.common.annotations.Beta;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+
+/**
+ * Provides detailed information for rate limit exceptions.
+ */
+@Beta
+public class ProfitBricksRateLimitExceededException extends RateLimitExceededException {
+ private static final long serialVersionUID = 1L;
+ private static final String RATE_LIMIT_HEADER_PREFIX = "X-RateLimit-";
+
+ private Integer maxConcurrentRequestsAllowed;
+ private Integer remainingRequests;
+ private Integer averageRequestsAllowedPerMinute;
+
+ public ProfitBricksRateLimitExceededException(HttpResponse response) {
+ super(response.getStatusLine() + "\n" + rateLimitHeaders(response));
+ parseRateLimitInfo(response);
+ }
+
+ public ProfitBricksRateLimitExceededException(HttpResponse response, Throwable cause) {
+ super(response.getStatusLine() + "\n" + rateLimitHeaders(response), cause);
+ parseRateLimitInfo(response);
+ }
+
+ public Integer maxConcurrentRequestsAllowed() {
+ return maxConcurrentRequestsAllowed;
+ }
+
+ public Integer remainingRequests() {
+ return remainingRequests;
+ }
+
+ public Integer averageRequestsAllowedPerMinute() {
+ return averageRequestsAllowedPerMinute;
+ }
+
+ private void parseRateLimitInfo(HttpResponse response) {
+ String burst = response.getFirstHeaderOrNull("X-RateLimit-Burst");
+ String remaining = response.getFirstHeaderOrNull("X-RateLimit-Remaining");
+ String limit = response.getFirstHeaderOrNull("X-RateLimit-Limit");
+
+ maxConcurrentRequestsAllowed = burst == null ? null : Integer.valueOf(burst);
+ remainingRequests = remaining == null ? null : Integer.valueOf(remaining);
+ averageRequestsAllowedPerMinute = limit == null ? null : Integer.valueOf(limit);
+ }
+
+ private static Multimap<String, String> rateLimitHeaders(HttpResponse response) {
+ return Multimaps.filterKeys(response.getHeaders(), new Predicate<String>() {
+ @Override
+ public boolean apply(String input) {
+ return input.startsWith(RATE_LIMIT_HEADER_PREFIX);
+ }
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/jclouds-labs/blob/436c001f/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/handlers/ProfitBricksHttpErrorHandler.java
----------------------------------------------------------------------
diff --git a/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/handlers/ProfitBricksHttpErrorHandler.java b/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/handlers/ProfitBricksHttpErrorHandler.java
index 188c900..6c46cad 100644
--- a/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/handlers/ProfitBricksHttpErrorHandler.java
+++ b/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/handlers/ProfitBricksHttpErrorHandler.java
@@ -16,10 +16,9 @@
*/
package org.apache.jclouds.profitbricks.rest.handlers;
-import static org.jclouds.util.Closeables2.closeQuietly;
-
import javax.inject.Singleton;
+import org.apache.jclouds.profitbricks.rest.exceptions.ProfitBricksRateLimitExceededException;
import org.jclouds.http.HttpCommand;
import org.jclouds.http.HttpErrorHandler;
import org.jclouds.http.HttpResponse;
@@ -58,6 +57,9 @@ public class ProfitBricksHttpErrorHandler implements HttpErrorHandler {
if (!command.getCurrentRequest().getMethod().equals("DELETE"))
exception = new ResourceNotFoundException(response.getMessage(), exception);
break;
+ case 429:
+ exception = new ProfitBricksRateLimitExceededException(response);
+ break;
case 413:
case 503:
// if nothing (default message was OK) was parsed from command executor, assume it was an 503 (Maintenance) html response.
@@ -68,7 +70,6 @@ public class ProfitBricksHttpErrorHandler implements HttpErrorHandler {
break;
}
} finally {
- closeQuietly(response.getPayload());
command.setException(exception);
}
}
http://git-wip-us.apache.org/repos/asf/jclouds-labs/blob/436c001f/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/handlers/ProfitBricksRateLimitRetryHandler.java
----------------------------------------------------------------------
diff --git a/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/handlers/ProfitBricksRateLimitRetryHandler.java b/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/handlers/ProfitBricksRateLimitRetryHandler.java
new file mode 100644
index 0000000..9aef958
--- /dev/null
+++ b/profitbricks-rest/src/main/java/org/apache/jclouds/profitbricks/rest/handlers/ProfitBricksRateLimitRetryHandler.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jclouds.profitbricks.rest.handlers;
+
+import javax.inject.Singleton;
+
+import org.jclouds.http.HttpCommand;
+import org.jclouds.http.HttpResponse;
+import org.jclouds.http.handlers.RateLimitRetryHandler;
+
+import com.google.common.annotations.Beta;
+import com.google.common.base.Optional;
+import com.google.common.net.HttpHeaders;
+
+@Beta
+@Singleton
+public class ProfitBricksRateLimitRetryHandler extends RateLimitRetryHandler {
+
+ @Override
+ protected Optional<Long> millisToNextAvailableRequest(HttpCommand command, HttpResponse response) {
+ String secondsToNextAvailableRequest = response.getFirstHeaderOrNull(HttpHeaders.RETRY_AFTER);
+ return secondsToNextAvailableRequest != null ? Optional.of(Long.valueOf(secondsToNextAvailableRequest) * 1000)
+ : Optional.<Long> absent();
+ }
+}
http://git-wip-us.apache.org/repos/asf/jclouds-labs/blob/436c001f/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/compute/ProfitBricksComputeServiceLiveTest.java
----------------------------------------------------------------------
diff --git a/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/compute/ProfitBricksComputeServiceLiveTest.java b/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/compute/ProfitBricksComputeServiceLiveTest.java
index de6a2fb..a3a15b3 100644
--- a/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/compute/ProfitBricksComputeServiceLiveTest.java
+++ b/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/compute/ProfitBricksComputeServiceLiveTest.java
@@ -26,6 +26,8 @@ import org.jclouds.compute.domain.NodeMetadata;
import org.jclouds.compute.domain.Template;
import org.jclouds.compute.internal.BaseComputeServiceLiveTest;
import static org.jclouds.compute.predicates.NodePredicates.inGroup;
+
+import org.apache.jclouds.profitbricks.rest.config.ProfitBricksRateLimitModule;
import org.jclouds.logging.config.LoggingModule;
import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
import org.jclouds.sshj.config.SshjSshClientModule;
@@ -47,6 +49,14 @@ public class ProfitBricksComputeServiceLiveTest extends BaseComputeServiceLiveTe
protected LoggingModule getLoggingModule() {
return new SLF4JLoggingModule();
}
+
+ @Override
+ protected Iterable<Module> setupModules() {
+ ImmutableSet.Builder<Module> modules = ImmutableSet.builder();
+ modules.addAll(super.setupModules());
+ modules.add(new ProfitBricksRateLimitModule());
+ return modules.build();
+ }
@Override
public void testOptionToNotBlock() throws Exception {
http://git-wip-us.apache.org/repos/asf/jclouds-labs/blob/436c001f/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/compute/ProfitBricksTemplateBuilderLiveTest.java
----------------------------------------------------------------------
diff --git a/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/compute/ProfitBricksTemplateBuilderLiveTest.java b/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/compute/ProfitBricksTemplateBuilderLiveTest.java
index afe1be2..f029c41 100644
--- a/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/compute/ProfitBricksTemplateBuilderLiveTest.java
+++ b/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/compute/ProfitBricksTemplateBuilderLiveTest.java
@@ -17,7 +17,11 @@
package org.apache.jclouds.profitbricks.rest.compute;
import com.google.common.collect.ImmutableSet;
+import com.google.inject.Module;
+
import java.util.Set;
+
+import org.apache.jclouds.profitbricks.rest.config.ProfitBricksRateLimitModule;
import org.jclouds.compute.internal.BaseTemplateBuilderLiveTest;
import org.testng.annotations.Test;
@@ -33,4 +37,11 @@ public class ProfitBricksTemplateBuilderLiveTest extends BaseTemplateBuilderLive
return ImmutableSet.of("DE-BW", "DE-HE", "US-NV");
}
+ @Override
+ protected Iterable<Module> setupModules() {
+ ImmutableSet.Builder<Module> modules = ImmutableSet.builder();
+ modules.addAll(super.setupModules());
+ modules.add(new ProfitBricksRateLimitModule());
+ return modules.build();
+ }
}
http://git-wip-us.apache.org/repos/asf/jclouds-labs/blob/436c001f/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/internal/BaseProfitBricksLiveTest.java
----------------------------------------------------------------------
diff --git a/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/internal/BaseProfitBricksLiveTest.java b/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/internal/BaseProfitBricksLiveTest.java
index 9e50dba..e1c3145 100644
--- a/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/internal/BaseProfitBricksLiveTest.java
+++ b/profitbricks-rest/src/test/java/org/apache/jclouds/profitbricks/rest/internal/BaseProfitBricksLiveTest.java
@@ -18,6 +18,7 @@ package org.apache.jclouds.profitbricks.rest.internal;
import com.google.common.base.Joiner;
import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableSet;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.Module;
@@ -27,6 +28,8 @@ import java.util.Properties;
import java.util.concurrent.TimeUnit;
import org.apache.jclouds.profitbricks.rest.ProfitBricksApi;
import org.apache.jclouds.profitbricks.rest.compute.config.ProfitBricksComputeServiceContextModule.ComputeConstants;
+import org.apache.jclouds.profitbricks.rest.config.ProfitBricksRateLimitModule;
+
import static org.apache.jclouds.profitbricks.rest.config.ProfitBricksComputeProperties.POLL_PREDICATE_DATACENTER;
import org.apache.jclouds.profitbricks.rest.domain.DataCenter;
import org.apache.jclouds.profitbricks.rest.domain.LicenceType;
@@ -61,6 +64,11 @@ public class BaseProfitBricksLiveTest extends BaseApiLiveTest<ProfitBricksApi> {
public BaseProfitBricksLiveTest() {
provider = "profitbricks-rest";
}
+
+ @Override
+ protected Iterable<Module> setupModules() {
+ return ImmutableSet.<Module> of(getLoggingModule(), new ProfitBricksRateLimitModule());
+ }
@Override
protected ProfitBricksApi create(Properties props, Iterable<Module> modules) {