You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@libcloud.apache.org by to...@apache.org on 2021/11/03 20:53:43 UTC
[libcloud] branch understand-ai-intelligent-retry created (now
d2138f9)
This is an automated email from the ASF dual-hosted git repository.
tomaz pushed a change to branch understand-ai-intelligent-retry
in repository https://gitbox.apache.org/repos/asf/libcloud.git.
at d2138f9 Add changelog entries.
This branch includes the following new commits:
new 997b177 Add parameter to turn off infinite retry on rate limiting
new e86b374 Update retry function so we also respect "timeout" argument passed to the function and we don't try to retry on RateLimitReached errors if timeout has been exhausted.
new d2138f9 Add changelog entries.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[libcloud] 01/03: Add parameter to turn off infinite retry on rate
limiting
Posted by to...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tomaz pushed a commit to branch understand-ai-intelligent-retry
in repository https://gitbox.apache.org/repos/asf/libcloud.git
commit 997b177490cc08da566c518d84a28c3063296cdf
Author: Veith Röthlingshöfer <ve...@understand.ai>
AuthorDate: Wed Nov 3 12:09:43 2021 +0100
Add parameter to turn off infinite retry on rate limiting
---
libcloud/utils/retry.py | 20 +++++++++++++++-----
1 file changed, 15 insertions(+), 5 deletions(-)
diff --git a/libcloud/utils/retry.py b/libcloud/utils/retry.py
index a8ccf5f..dd62dbd 100644
--- a/libcloud/utils/retry.py
+++ b/libcloud/utils/retry.py
@@ -18,6 +18,7 @@ import ssl
import time
from datetime import datetime, timedelta
from functools import wraps
+import logging
from libcloud.utils.py3 import httplib
from libcloud.common.exceptions import RateLimitReachedError
@@ -26,7 +27,7 @@ __all__ = [
'Retry'
]
-
+_logger = logging.getLogger(__name__)
# Error message which indicates a transient SSL error upon which request
# can be retried
TRANSIENT_SSL_ERROR = 'The read operation timed out'
@@ -42,6 +43,7 @@ class TransientSSLError(ssl.SSLError):
DEFAULT_TIMEOUT = 30 # default retry timeout
DEFAULT_DELAY = 1 # default sleep delay used in each iterator
DEFAULT_BACKOFF = 1 # retry backup multiplier
+DEFAULT_MAX_RATE_LIMIT_RETRIES = float("inf") # default max number of times to retry on rate limit
RETRY_EXCEPTIONS = (RateLimitReachedError, socket.error, socket.gaierror,
httplib.NotConnected, httplib.ImproperConnectionState,
TransientSSLError)
@@ -50,15 +52,18 @@ RETRY_EXCEPTIONS = (RateLimitReachedError, socket.error, socket.gaierror,
class MinimalRetry:
def __init__(self, retry_delay=DEFAULT_DELAY,
- timeout=DEFAULT_TIMEOUT, backoff=DEFAULT_BACKOFF):
+ timeout=DEFAULT_TIMEOUT, backoff=DEFAULT_BACKOFF,
+ max_rate_limit_retries=DEFAULT_MAX_RATE_LIMIT_RETRIES):
"""
Wrapper around retrying that helps to handle common transient exceptions.
This minimalistic version only retries SSL errors and rate limiting.
- :param retry_exceptions: types of exceptions to retry on.
:param retry_delay: retry delay between the attempts.
:param timeout: maximum time to wait.
:param backoff: multiplier added to delay between attempts.
+ :param max_rate_limit_retries: The maximum number of retries to do when being rate limited by the server.
+ Set to `float("inf")` if retrying forever is desired.
+ Being rate limited does not count towards the timeout.
:Example:
@@ -72,12 +77,15 @@ class MinimalRetry:
timeout = DEFAULT_TIMEOUT
if backoff is None:
backoff = DEFAULT_BACKOFF
+ if max_rate_limit_retries is None:
+ max_rate_limit_retries = DEFAULT_MAX_RATE_LIMIT_RETRIES
timeout = max(timeout, 0)
self.retry_delay = retry_delay
self.timeout = timeout
self.backoff = backoff
+ self.max_rate_limit_retries = max_rate_limit_retries
def __call__(self, func):
def transform_ssl_error(function, *args, **kwargs):
@@ -93,19 +101,21 @@ class MinimalRetry:
def retry_loop(*args, **kwargs):
current_delay = self.retry_delay
end = datetime.now() + timedelta(seconds=self.timeout)
+ number_rate_limited_retries = 0
while True:
try:
return transform_ssl_error(func, *args, **kwargs)
except Exception as exc:
- if isinstance(exc, RateLimitReachedError):
+ if isinstance(exc, RateLimitReachedError) and number_rate_limited_retries <= self.max_rate_limit_retries:
+ _logger.debug("You are being rate limited, backing off...")
time.sleep(exc.retry_after)
-
# Reset retries if we're told to wait due to rate
# limiting
current_delay = self.retry_delay
end = datetime.now() + timedelta(
seconds=exc.retry_after + self.timeout)
+ number_rate_limited_retries += 1
elif datetime.now() >= end:
raise
elif self.should_retry(exc):
[libcloud] 03/03: Add changelog entries.
Posted by to...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tomaz pushed a commit to branch understand-ai-intelligent-retry
in repository https://gitbox.apache.org/repos/asf/libcloud.git
commit d2138f96327cfcfa1e905162cf7a772aacd6474a
Author: Tomaz Muraus <to...@tomaz.me>
AuthorDate: Wed Nov 3 21:50:40 2021 +0100
Add changelog entries.
---
CHANGES.rst | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/CHANGES.rst b/CHANGES.rst
index ffff880..1940e47 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -4,6 +4,22 @@
Changes in Apache Libcloud 3.3.2 (in development)
-------------------------------------------------
+Common
+~~~~~~
+
+- Update HTTP connection and request retry code to be more flexible so user
+ can specify and utilize custom retry logic which can be configured via
+ connection retryCls attribute
+ (``driver.connection.retryCls = MyRetryClass``).
+
+ (GITHUB-1558)
+ [Veith Röthlingshöfer - @RunOrVeith]
+
+- HTTP connection and request retry logic has been updated so we still respect
+ ``timeout`` argument when retrying requests due to rate limit being reached
+ errors. Previously, we would try to retry indefinitely on
+ ``RateLimitReachedError`` exceptions.
+
Storage
~~~~~~~
[libcloud] 02/03: Update retry function so we also respect
"timeout" argument passed to the function and we don't try to retry on
RateLimitReached errors if timeout has been exhausted.
Posted by to...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tomaz pushed a commit to branch understand-ai-intelligent-retry
in repository https://gitbox.apache.org/repos/asf/libcloud.git
commit e86b374af0261d87f423147e40bb6d999edb7da1
Author: Tomaz Muraus <to...@tomaz.me>
AuthorDate: Wed Nov 3 21:44:30 2021 +0100
Update retry function so we also respect "timeout" argument passed to the
function and we don't try to retry on RateLimitReached errors if timeout
has been exhausted.
---
libcloud/test/test_connection.py | 18 ++++++++++++++++++
libcloud/utils/retry.py | 31 ++++++++++++++++---------------
2 files changed, 34 insertions(+), 15 deletions(-)
diff --git a/libcloud/test/test_connection.py b/libcloud/test/test_connection.py
index 525f587..2848426 100644
--- a/libcloud/test/test_connection.py
+++ b/libcloud/test/test_connection.py
@@ -27,6 +27,7 @@ import libcloud.common.base
from libcloud.test import unittest
from libcloud.common.base import Connection, CertificateConnection
+from libcloud.common.exceptions import RateLimitReachedError
from libcloud.http import LibcloudBaseConnection
from libcloud.http import LibcloudConnection
from libcloud.http import SignedHTTPSAdapter
@@ -452,6 +453,23 @@ class ConnectionClassTestCase(unittest.TestCase):
self.assertGreater(mock_connect.call_count, 1,
'Retry logic failed')
+ def test_retry_rate_limit_error_timeout(self):
+ con = Connection()
+ con.connection = Mock()
+ connect_method = 'libcloud.common.base.Connection.request'
+
+ with patch(connect_method) as mock_connect:
+ mock_connect.__name__ = 'mock_connect'
+ with self.assertRaises(RateLimitReachedError):
+ headers = {'retry-after': 0.2}
+ mock_connect.side_effect = RateLimitReachedError(headers=headers)
+ retry_request = Retry(timeout=0.4, retry_delay=0.1,
+ backoff=1)
+ retry_request(con.request)(action='/')
+
+ self.assertEqual(mock_connect.call_count, 2,
+ 'Retry logic failed')
+
class CertificateConnectionClassTestCase(unittest.TestCase):
def setUp(self):
diff --git a/libcloud/utils/retry.py b/libcloud/utils/retry.py
index dd62dbd..c45db86 100644
--- a/libcloud/utils/retry.py
+++ b/libcloud/utils/retry.py
@@ -43,7 +43,6 @@ class TransientSSLError(ssl.SSLError):
DEFAULT_TIMEOUT = 30 # default retry timeout
DEFAULT_DELAY = 1 # default sleep delay used in each iterator
DEFAULT_BACKOFF = 1 # retry backup multiplier
-DEFAULT_MAX_RATE_LIMIT_RETRIES = float("inf") # default max number of times to retry on rate limit
RETRY_EXCEPTIONS = (RateLimitReachedError, socket.error, socket.gaierror,
httplib.NotConnected, httplib.ImproperConnectionState,
TransientSSLError)
@@ -52,8 +51,7 @@ RETRY_EXCEPTIONS = (RateLimitReachedError, socket.error, socket.gaierror,
class MinimalRetry:
def __init__(self, retry_delay=DEFAULT_DELAY,
- timeout=DEFAULT_TIMEOUT, backoff=DEFAULT_BACKOFF,
- max_rate_limit_retries=DEFAULT_MAX_RATE_LIMIT_RETRIES):
+ timeout=DEFAULT_TIMEOUT, backoff=DEFAULT_BACKOFF):
"""
Wrapper around retrying that helps to handle common transient exceptions.
This minimalistic version only retries SSL errors and rate limiting.
@@ -61,9 +59,6 @@ class MinimalRetry:
:param retry_delay: retry delay between the attempts.
:param timeout: maximum time to wait.
:param backoff: multiplier added to delay between attempts.
- :param max_rate_limit_retries: The maximum number of retries to do when being rate limited by the server.
- Set to `float("inf")` if retrying forever is desired.
- Being rate limited does not count towards the timeout.
:Example:
@@ -77,15 +72,12 @@ class MinimalRetry:
timeout = DEFAULT_TIMEOUT
if backoff is None:
backoff = DEFAULT_BACKOFF
- if max_rate_limit_retries is None:
- max_rate_limit_retries = DEFAULT_MAX_RATE_LIMIT_RETRIES
timeout = max(timeout, 0)
self.retry_delay = retry_delay
self.timeout = timeout
self.backoff = backoff
- self.max_rate_limit_retries = max_rate_limit_retries
def __call__(self, func):
def transform_ssl_error(function, *args, **kwargs):
@@ -101,21 +93,30 @@ class MinimalRetry:
def retry_loop(*args, **kwargs):
current_delay = self.retry_delay
end = datetime.now() + timedelta(seconds=self.timeout)
- number_rate_limited_retries = 0
while True:
try:
return transform_ssl_error(func, *args, **kwargs)
except Exception as exc:
- if isinstance(exc, RateLimitReachedError) and number_rate_limited_retries <= self.max_rate_limit_retries:
+ if isinstance(exc, RateLimitReachedError):
+ if datetime.now() >= end:
+ # We have exhausted retry timeout so we abort
+ # retrying
+ raise
+
_logger.debug("You are being rate limited, backing off...")
- time.sleep(exc.retry_after)
+
+ # NOTE: Retry after defaults to 0 in the
+ # RateLimitReachedError class so we a use more
+ # reasonable default in case that attribute is not
+ # present. This way we prevent busy waiting, etc.
+ retry_after = exc.retry_after if exc.retry_after else 2
+
+ time.sleep(retry_after)
+
# Reset retries if we're told to wait due to rate
# limiting
current_delay = self.retry_delay
- end = datetime.now() + timedelta(
- seconds=exc.retry_after + self.timeout)
- number_rate_limited_retries += 1
elif datetime.now() >= end:
raise
elif self.should_retry(exc):