You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@libcloud.apache.org by vd...@apache.org on 2018/08/08 02:04:25 UTC
[1/2] libcloud git commit: Fixed upload/download streams
Repository: libcloud
Updated Branches:
refs/heads/trunk afc1aefb2 -> a1ebd1448
Fixed upload/download streams
Signed-off-by: Rick van de Loo <ri...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/90399682
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/90399682
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/90399682
Branch: refs/heads/trunk
Commit: 9039968249cba20a546e5d1eb54ad2efbfa79f43
Parents: afc1aef
Author: Michael Perel <mi...@gmail.com>
Authored: Thu Aug 2 13:29:46 2018 -0400
Committer: Rick van de Loo <ri...@gmail.com>
Committed: Wed Aug 8 03:58:31 2018 +0200
----------------------------------------------------------------------
libcloud/storage/drivers/azure_blobs.py | 27 ++++++++++++++++++++++----
libcloud/test/storage/test_azure_blobs.py | 4 +++-
2 files changed, 26 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/90399682/libcloud/storage/drivers/azure_blobs.py
----------------------------------------------------------------------
diff --git a/libcloud/storage/drivers/azure_blobs.py b/libcloud/storage/drivers/azure_blobs.py
index a53f6ac..ae49a1d 100644
--- a/libcloud/storage/drivers/azure_blobs.py
+++ b/libcloud/storage/drivers/azure_blobs.py
@@ -579,11 +579,14 @@ class AzureBlobsStorageDriver(StorageDriver):
@inherits: :class:`StorageDriver.download_object_as_stream`
"""
obj_path = self._get_object_path(obj.container, obj.name)
- response = self.connection.request(obj_path, raw=True, data=None)
+ response = self.connection.request(obj_path, method='GET',
+ stream=True, raw=True)
+ iterator = response.iter_content(AZURE_CHUNK_SIZE)
- return self._get_object(obj=obj, callback=read_in_chunks,
+ return self._get_object(obj=obj,
+ callback=read_in_chunks,
response=response,
- callback_kwargs={'iterator': response.response,
+ callback_kwargs={'iterator': iterator,
'chunk_size': chunk_size},
success_status_code=httplib.OK)
@@ -810,11 +813,27 @@ class AzureBlobsStorageDriver(StorageDriver):
if ex_blob_type is None:
ex_blob_type = self.ex_blob_type
+ """
+ Azure requires that for page blobs that a maximum size that the page
+ can grow to. For block blobs, it is required that the Content-Length
+ header be set. The size of the block blob will be the total size of
+ the stream minus the current position, so in this case
+ ex_page_blob_size should be 0 (and will be checked in
+ self._check_values).
+ Source:
+ https://docs.microsoft.com/en-us/rest/api/storageservices/put-blob
+ """
self._check_values(ex_blob_type, ex_page_blob_size)
+ if ex_blob_type == "BlockBlob":
+ iterator.seek(0, os.SEEK_END)
+ blob_size = iterator.tell()
+ iterator.seek(0)
+ else:
+ blob_size = ex_page_blob_size
return self._put_object(container=container,
object_name=object_name,
- object_size=ex_page_blob_size,
+ object_size=blob_size,
extra=extra, verify_hash=verify_hash,
blob_type=ex_blob_type,
use_lease=ex_use_lease,
http://git-wip-us.apache.org/repos/asf/libcloud/blob/90399682/libcloud/test/storage/test_azure_blobs.py
----------------------------------------------------------------------
diff --git a/libcloud/test/storage/test_azure_blobs.py b/libcloud/test/storage/test_azure_blobs.py
index 69bdebb..aa09c66 100644
--- a/libcloud/test/storage/test_azure_blobs.py
+++ b/libcloud/test/storage/test_azure_blobs.py
@@ -647,7 +647,9 @@ class AzureBlobsTests(unittest.TestCase):
stream = self.driver.download_object_as_stream(obj=obj,
chunk_size=None)
- self.assertTrue(hasattr(stream, '__iter__'))
+
+ consumed_stream = ''.join(chunk.decode('utf-8') for chunk in stream)
+ self.assertEqual(len(consumed_stream), obj.size)
def test_upload_object_invalid_ex_blob_type(self):
# Invalid hash is detected on the amazon side and BAD_REQUEST is
[2/2] libcloud git commit: Add changes for #1231
Posted by vd...@apache.org.
Add changes for #1231
Closes #1231
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/a1ebd144
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/a1ebd144
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/a1ebd144
Branch: refs/heads/trunk
Commit: a1ebd144849e597286db6611f6d41bad504af4d9
Parents: 9039968
Author: Rick van de Loo <ri...@gmail.com>
Authored: Wed Aug 8 04:03:37 2018 +0200
Committer: Rick van de Loo <ri...@gmail.com>
Committed: Wed Aug 8 04:03:37 2018 +0200
----------------------------------------------------------------------
CHANGES.rst | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/a1ebd144/CHANGES.rst
----------------------------------------------------------------------
diff --git a/CHANGES.rst b/CHANGES.rst
index 95997ae..aeb4e6f 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -117,6 +117,9 @@ Storage
- [Azure Blob Storage] Update driver documentation (GITHUB-1208)
[Clemens Wolff]
+- [Azure Blob Storage] Fix upload/download streams (GITHUB-1231)
+ [Michael Perel]
+
- [S3] Guess s3 upload content type (LIBCLOUD-958, GITHUB-1195)
[Iuri de Silvio]