[libcloud] 01/03: Add parameter to turn off infinite retry on rate limiting

[to...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

tomaz pushed a commit to branch understand-ai-intelligent-retry
in repository https://gitbox.apache.org/repos/asf/libcloud.git

commit 997b177490cc08da566c518d84a28c3063296cdf
Author: Veith Röthlingshöfer <ve...@understand.ai>
AuthorDate: Wed Nov 3 12:09:43 2021 +0100

    Add parameter to turn off infinite retry on rate limiting
---
 libcloud/utils/retry.py | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/libcloud/utils/retry.py b/libcloud/utils/retry.py
index a8ccf5f..dd62dbd 100644
--- a/libcloud/utils/retry.py
+++ b/libcloud/utils/retry.py
@@ -18,6 +18,7 @@ import ssl
 import time
 from datetime import datetime, timedelta
 from functools import wraps
+import logging
 
 from libcloud.utils.py3 import httplib
 from libcloud.common.exceptions import RateLimitReachedError
@@ -26,7 +27,7 @@ __all__ = [
     'Retry'
 ]
 
-
+_logger = logging.getLogger(__name__)
 # Error message which indicates a transient SSL error upon which request
 # can be retried
 TRANSIENT_SSL_ERROR = 'The read operation timed out'
@@ -42,6 +43,7 @@ class TransientSSLError(ssl.SSLError):
 DEFAULT_TIMEOUT = 30  # default retry timeout
 DEFAULT_DELAY = 1  # default sleep delay used in each iterator
 DEFAULT_BACKOFF = 1  # retry backup multiplier
+DEFAULT_MAX_RATE_LIMIT_RETRIES = float("inf")  # default max number of times to retry on rate limit
 RETRY_EXCEPTIONS = (RateLimitReachedError, socket.error, socket.gaierror,
                     httplib.NotConnected, httplib.ImproperConnectionState,
                     TransientSSLError)
@@ -50,15 +52,18 @@ RETRY_EXCEPTIONS = (RateLimitReachedError, socket.error, socket.gaierror,
 class MinimalRetry:
 
     def __init__(self, retry_delay=DEFAULT_DELAY,
-                 timeout=DEFAULT_TIMEOUT, backoff=DEFAULT_BACKOFF):
+                 timeout=DEFAULT_TIMEOUT, backoff=DEFAULT_BACKOFF,
+                 max_rate_limit_retries=DEFAULT_MAX_RATE_LIMIT_RETRIES):
         """
         Wrapper around retrying that helps to handle common transient exceptions.
         This minimalistic version only retries SSL errors and rate limiting.
-        :param retry_exceptions: types of exceptions to retry on.
 
         :param retry_delay: retry delay between the attempts.
         :param timeout: maximum time to wait.
         :param backoff: multiplier added to delay between attempts.
+        :param max_rate_limit_retries: The maximum number of retries to do when being rate limited by the server.
+                                       Set to `float("inf")` if retrying forever is desired.
+                                       Being rate limited does not count towards the timeout.
 
         :Example:
 
@@ -72,12 +77,15 @@ class MinimalRetry:
             timeout = DEFAULT_TIMEOUT
         if backoff is None:
             backoff = DEFAULT_BACKOFF
+        if max_rate_limit_retries is None:
+            max_rate_limit_retries = DEFAULT_MAX_RATE_LIMIT_RETRIES
 
         timeout = max(timeout, 0)
 
         self.retry_delay = retry_delay
         self.timeout = timeout
         self.backoff = backoff
+        self.max_rate_limit_retries = max_rate_limit_retries
 
     def __call__(self, func):
         def transform_ssl_error(function, *args, **kwargs):
@@ -93,19 +101,21 @@ class MinimalRetry:
         def retry_loop(*args, **kwargs):
             current_delay = self.retry_delay
             end = datetime.now() + timedelta(seconds=self.timeout)
+            number_rate_limited_retries = 0
 
             while True:
                 try:
                     return transform_ssl_error(func, *args, **kwargs)
                 except Exception as exc:
-                    if isinstance(exc, RateLimitReachedError):
+                    if isinstance(exc, RateLimitReachedError) and number_rate_limited_retries <= self.max_rate_limit_retries:
+                        _logger.debug("You are being rate limited, backing off...")
                         time.sleep(exc.retry_after)
-
                         # Reset retries if we're told to wait due to rate
                         # limiting
                         current_delay = self.retry_delay
                         end = datetime.now() + timedelta(
                             seconds=exc.retry_after + self.timeout)
+                        number_rate_limited_retries += 1
                     elif datetime.now() >= end:
                         raise
                     elif self.should_retry(exc):