You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@libcloud.apache.org by qu...@apache.org on 2017/11/10 04:29:31 UTC
[1/3] libcloud git commit: Fix Retry-After header handling
Repository: libcloud
Updated Branches:
refs/heads/trunk 0c5bee8b1 -> 3a9de686b
Fix Retry-After header handling
Pass response headers to exception_from_message() on response error
Pass Retry-After header to BaseHTTPException, some other errors than
429 may include it. (Example: 503 Service Unavailable)
Retry-After header may include an HTTP-date value, so check if that's
the case and if so, translate it to a delay seconds value.
Added tests.
Signed-off-by: Quentin Pradet <qu...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/bd32dfc2
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/bd32dfc2
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/bd32dfc2
Branch: refs/heads/trunk
Commit: bd32dfc2016d0308065e9a002a4793cc407262b0
Parents: 0c5bee8
Author: Lucas Di Pentima <ld...@veritasgenetics.com>
Authored: Thu Nov 2 19:08:56 2017 -0300
Committer: Quentin Pradet <qu...@apache.org>
Committed: Fri Nov 10 08:24:40 2017 +0400
----------------------------------------------------------------------
libcloud/common/base.py | 3 +-
libcloud/common/exceptions.py | 14 +++++++--
libcloud/test/common/test_base.py | 53 +++++++++++++++++++++++++++++++++-
3 files changed, 65 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/bd32dfc2/libcloud/common/base.py
----------------------------------------------------------------------
diff --git a/libcloud/common/base.py b/libcloud/common/base.py
index d90e1c5..fbcdd92 100644
--- a/libcloud/common/base.py
+++ b/libcloud/common/base.py
@@ -154,7 +154,8 @@ class Response(object):
if not self.success():
raise exception_from_message(code=self.status,
- message=self.parse_error())
+ message=self.parse_error(),
+ headers=self.headers)
self.object = self.parse_body()
http://git-wip-us.apache.org/repos/asf/libcloud/blob/bd32dfc2/libcloud/common/exceptions.py
----------------------------------------------------------------------
diff --git a/libcloud/common/exceptions.py b/libcloud/common/exceptions.py
index a286ccd..af08bf8 100644
--- a/libcloud/common/exceptions.py
+++ b/libcloud/common/exceptions.py
@@ -13,6 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import time
+
+from email.utils import parsedate_tz, mktime_tz
+
__all__ = [
'BaseHTTPError',
'RateLimitReachedError',
@@ -47,7 +51,7 @@ class RateLimitReachedError(BaseHTTPError):
message = '%s Rate limit exceeded' % (code)
def __init__(self, *args, **kwargs):
- self.retry_after = int(kwargs.pop('retry_after', 0))
+ self.retry_after = int(kwargs.pop('headers', {}).get('retry-after', 0))
_error_classes = [RateLimitReachedError]
@@ -68,7 +72,11 @@ def exception_from_message(code, message, headers=None):
'headers': headers
}
- if headers and 'retry_after' in headers:
- kwargs['retry_after'] = headers['retry_after']
+ if headers and 'retry-after' in headers:
+ http_date = parsedate_tz(headers['retry-after'])
+ if http_date is not None:
+ # Convert HTTP-date to delay-seconds
+ delay = max(0, int(mktime_tz(http_date) - time.time()))
+ headers['retry-after'] = str(delay)
cls = _code_map.get(code, BaseHTTPError)
return cls(**kwargs)
http://git-wip-us.apache.org/repos/asf/libcloud/blob/bd32dfc2/libcloud/test/common/test_base.py
----------------------------------------------------------------------
diff --git a/libcloud/test/common/test_base.py b/libcloud/test/common/test_base.py
index b6e412b..40b310a 100644
--- a/libcloud/test/common/test_base.py
+++ b/libcloud/test/common/test_base.py
@@ -18,7 +18,8 @@ import sys
import mock
-from libcloud.common.base import LazyObject
+from libcloud.common.base import LazyObject, Response
+from libcloud.common.exceptions import BaseHTTPError, RateLimitReachedError
from libcloud.test import LibcloudTestCase
@@ -55,5 +56,55 @@ class LazyObjectTest(LibcloudTestCase):
self.assertEqual(wrapped_lazy_obj.z, 'baz')
+class ErrorResponseTest(LibcloudTestCase):
+ def mock_response(self, code, headers={}):
+ m = mock.MagicMock()
+ m.request = mock.Mock()
+ m.headers = headers
+ m.status_code = code
+ m.text = None
+ return m
+
+ def test_rate_limit_response(self):
+ resp_mock = self.mock_response(429, {'Retry-After': '120'})
+ try:
+ Response(resp_mock, mock.MagicMock())
+ except RateLimitReachedError as e:
+ self.assertEqual(e.retry_after, 120)
+ except:
+ # We should have got a RateLimitReachedError
+ self.fail("Catched exception should have been RateLimitReachedError")
+ else:
+ # We should have got an exception
+ self.fail("HTTP Status 429 response didn't raised an exception")
+
+ def test_error_with_retry_after(self):
+ # 503 Service Unavailable may include Retry-After header
+ resp_mock = self.mock_response(503, {'Retry-After': '300'})
+ try:
+ Response(resp_mock, mock.MagicMock())
+ except BaseHTTPError as e:
+ self.assertIn('retry-after', e.headers)
+ self.assertEqual(e.headers['retry-after'], '300')
+ else:
+ # We should have got an exception
+ self.fail("HTTP Status 503 response didn't raised an exception")
+
+ @mock.patch('time.time', return_value=1231006505)
+ def test_error_with_retry_after_http_date_format(self, time_mock):
+ retry_after = 'Sat, 03 Jan 2009 18:20:05 -0000'
+ # 503 Service Unavailable may include Retry-After header
+ resp_mock = self.mock_response(503, {'Retry-After': retry_after})
+ try:
+ Response(resp_mock, mock.MagicMock())
+ except BaseHTTPError as e:
+ self.assertIn('retry-after', e.headers)
+ # HTTP-date got translated to delay-secs
+ self.assertEqual(e.headers['retry-after'], '300')
+ else:
+ # We should have got an exception
+ self.fail("HTTP Status 503 response didn't raised an exception")
+
+
if __name__ == '__main__':
sys.exit(unittest.main())
[2/3] libcloud git commit: RateLimitReachedError exception sets
message, code and headers.
Posted by qu...@apache.org.
RateLimitReachedError exception sets message, code and headers.
Added docstring documentation describing HTTP-date format
translation on Retry-After header.
Signed-off-by: Quentin Pradet <qu...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/dfebc6fc
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/dfebc6fc
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/dfebc6fc
Branch: refs/heads/trunk
Commit: dfebc6fc1b2709c0286ff6f6b449e3f90124f785
Parents: bd32dfc
Author: Lucas Di Pentima <ld...@veritasgenetics.com>
Authored: Fri Nov 3 18:31:29 2017 -0300
Committer: Quentin Pradet <qu...@apache.org>
Committed: Fri Nov 10 08:24:56 2017 +0400
----------------------------------------------------------------------
libcloud/common/exceptions.py | 24 ++++++++++++++++++++++--
1 file changed, 22 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/dfebc6fc/libcloud/common/exceptions.py
----------------------------------------------------------------------
diff --git a/libcloud/common/exceptions.py b/libcloud/common/exceptions.py
index af08bf8..e9f12b5 100644
--- a/libcloud/common/exceptions.py
+++ b/libcloud/common/exceptions.py
@@ -51,7 +51,14 @@ class RateLimitReachedError(BaseHTTPError):
message = '%s Rate limit exceeded' % (code)
def __init__(self, *args, **kwargs):
- self.retry_after = int(kwargs.pop('headers', {}).get('retry-after', 0))
+ headers = kwargs.pop('headers', None)
+ super(RateLimitReachedError, self).__init__(self.code,
+ self.message,
+ headers)
+ if self.headers is not None:
+ self.retry_after = int(self.headers.get('retry-after', 0))
+ else:
+ self.retry_after = 0
_error_classes = [RateLimitReachedError]
@@ -62,9 +69,22 @@ def exception_from_message(code, message, headers=None):
"""
Return an instance of BaseHTTPException or subclass based on response code.
+ If headers include Retry-After, RFC 2616 says that its value may be one of
+ two formats: HTTP-date or delta-seconds, for example:
+
+ Retry-After: Fri, 31 Dec 1999 23:59:59 GMT
+ Retry-After: 120
+
+ If Retry-After comes in HTTP-date, it'll be translated to a positive
+ delta-seconds value when passing it to the exception constructor.
+
+ Also, RFC 2616 says that Retry-After isn't just only applicable to 429
+ HTTP status, but also to other responses, like 503 and 3xx.
+
Usage::
raise exception_from_message(code=self.status,
- message=self.parse_error())
+ message=self.parse_error(),
+ headers=self.headers)
"""
kwargs = {
'code': code,
[3/3] libcloud git commit: Add changes for #1139
Posted by qu...@apache.org.
Add changes for #1139
Closes #1139
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/3a9de686
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/3a9de686
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/3a9de686
Branch: refs/heads/trunk
Commit: 3a9de686b407835f262b73bad4a6fc1893c746f8
Parents: dfebc6f
Author: Quentin Pradet <qu...@apache.org>
Authored: Fri Nov 10 08:25:44 2017 +0400
Committer: Quentin Pradet <qu...@apache.org>
Committed: Fri Nov 10 08:25:44 2017 +0400
----------------------------------------------------------------------
CHANGES.rst | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/3a9de686/CHANGES.rst
----------------------------------------------------------------------
diff --git a/CHANGES.rst b/CHANGES.rst
index 44bbd7b..f33625a 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -89,6 +89,9 @@ Compute
- [ARM] Limit number of retries in destroy_node (GITHUB-1134)
[Peter Amstutz, Lucas Di Pentima]
+- [ARM] Fix Retry-After header handling (GITHUB-1139)
+ [Lucas Di Pentima]
+
Storage
~~~~~~~