You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@libcloud.apache.org by an...@apache.org on 2017/01/09 04:52:57 UTC
[37/51] [abbrv] libcloud git commit: download_object_as_stream now
returns a proper iterator
download_object_as_stream now returns a proper iterator
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/109ecad9
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/109ecad9
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/109ecad9
Branch: refs/heads/trunk
Commit: 109ecad94257615b57f196f3c36d13d7b3c8e8fd
Parents: 5528f5b
Author: Anthony Shaw <an...@apache.org>
Authored: Fri Jan 6 19:52:51 2017 +1100
Committer: Anthony Shaw <an...@apache.org>
Committed: Fri Jan 6 19:52:51 2017 +1100
----------------------------------------------------------------------
libcloud/common/base.py | 15 +++++++++++----
libcloud/httplib_ssl.py | 7 ++++---
libcloud/storage/drivers/s3.py | 4 ++--
3 files changed, 17 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/109ecad9/libcloud/common/base.py
----------------------------------------------------------------------
diff --git a/libcloud/common/base.py b/libcloud/common/base.py
index 15233a9..5266163 100644
--- a/libcloud/common/base.py
+++ b/libcloud/common/base.py
@@ -152,6 +152,7 @@ class Response(object):
self.error = response.reason
self.status = response.status_code
self.request = response.request
+ self.iter_content = response.iter_content
self.body = response.text.strip() \
if response.text is not None and hasattr(response.text, 'strip') \
@@ -501,7 +502,7 @@ class Connection(object):
self.ua.append(token)
def request(self, action, params=None, data=None, headers=None,
- method='GET', raw=False):
+ method='GET', raw=False, stream=False):
"""
Request a given `action`.
@@ -531,6 +532,10 @@ class Connection(object):
and use the rawResponseCls class. This is used with
storage API when uploading a file.
+ :type stream: ``bool``
+ :param stream: True to return an iterator in Response.iter_content and allow
+ streaming of the response data (for downloading large files)
+
:return: An :class:`Response` instance.
:rtype: :class:`Response` instance
@@ -601,7 +606,8 @@ class Connection(object):
method=method,
url=url,
body=data,
- headers=headers)
+ headers=headers,
+ stream=stream)
else:
if retry_enabled:
retry_request = retry(timeout=self.timeout,
@@ -610,10 +616,11 @@ class Connection(object):
retry_request(self.connection.request)(method=method,
url=url,
body=data,
- headers=headers)
+ headers=headers,
+ stream=stream)
else:
self.connection.request(method=method, url=url, body=data,
- headers=headers)
+ headers=headers, stream=stream)
except socket.gaierror:
e = sys.exc_info()[1]
message = str(e)
http://git-wip-us.apache.org/repos/asf/libcloud/blob/109ecad9/libcloud/httplib_ssl.py
----------------------------------------------------------------------
diff --git a/libcloud/httplib_ssl.py b/libcloud/httplib_ssl.py
index 640d31a..ac20ee7 100644
--- a/libcloud/httplib_ssl.py
+++ b/libcloud/httplib_ssl.py
@@ -180,19 +180,20 @@ class LibcloudConnection(LibcloudBaseConnection):
self.set_http_proxy(proxy_url=proxy_url)
self.session.timeout = kwargs.get('timeout', 60)
- def request(self, method, url, body=None, headers=None, raw=False):
+ def request(self, method, url, body=None, headers=None, raw=False,
+ stream=False):
self.response = self.session.request(
method=method.lower(),
url=''.join([self.host, url]),
data=body,
headers=headers,
allow_redirects=1,
- stream=raw,
+ stream=stream,
verify=self.ca_cert if self.ca_cert is not None else self.verify
)
def prepared_request(self, method, url, body=None,
- headers=None, raw=False):
+ headers=None, raw=False, stream=False):
req = requests.Request(method, ''.join([self.host, url]),
data=body, headers=headers)
http://git-wip-us.apache.org/repos/asf/libcloud/blob/109ecad9/libcloud/storage/drivers/s3.py
----------------------------------------------------------------------
diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py
index 28e205e..b0cbd82 100644
--- a/libcloud/storage/drivers/s3.py
+++ b/libcloud/storage/drivers/s3.py
@@ -414,11 +414,11 @@ class BaseS3StorageDriver(StorageDriver):
def download_object_as_stream(self, obj, chunk_size=None):
obj_path = self._get_object_path(obj.container, obj.name)
- response = self.connection.request(obj_path, method='GET', raw=True)
+ response = self.connection.request(obj_path, method='GET', stream=True)
return self._get_object(obj=obj, callback=read_in_chunks,
response=response,
- callback_kwargs={'iterator': response.response,
+ callback_kwargs={'iterator': response.iter_content,
'chunk_size': chunk_size},
success_status_code=httplib.OK)