You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@libcloud.apache.org by to...@apache.org on 2014/12/09 13:41:27 UTC
libcloud git commit: [LIBCLOUD-639] Fix upload_object_via_stream so
it works correctly under Python 3.x if user manually passes an iterator to
the method.
Repository: libcloud
Updated Branches:
refs/heads/trunk f813ea17b -> 8fe3e4055
[LIBCLOUD-639] Fix upload_object_via_stream so it works correctly
under Python 3.x if user manually passes an iterator to the method.
Closes #408
Signed-off-by: Tomaz Muraus <to...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/8fe3e405
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/8fe3e405
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/8fe3e405
Branch: refs/heads/trunk
Commit: 8fe3e4055d8479c2ccaebff45bf0071890639f8c
Parents: f813ea1
Author: Peter Schmidt <pe...@peterjs.com>
Authored: Mon Dec 1 17:28:20 2014 +1100
Committer: Tomaz Muraus <to...@apache.org>
Committed: Tue Dec 9 13:34:13 2014 +0100
----------------------------------------------------------------------
CHANGES.rst | 5 ++
libcloud/storage/base.py | 3 +-
libcloud/test/storage/test_cloudfiles.py | 69 +++++++++++++++++++++++++--
libcloud/utils/files.py | 2 +-
libcloud/utils/py3.py | 2 +
5 files changed, 74 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/8fe3e405/CHANGES.rst
----------------------------------------------------------------------
diff --git a/CHANGES.rst b/CHANGES.rst
index e4858e7..3a2282d 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -79,6 +79,11 @@ Storage
(GITHUB-403, GITHUB-404)
[Peter Schmidt]
+- Fix upload_object_via_stream so it works correctly under Python 3.x if user
+ manually passes an iterator to the method.
+ (GITHUB-408, LIBCLOUD-639)
+ [Peter Schmidt]
+
Changes with Apache Libcloud 0.16.0
-----------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/8fe3e405/libcloud/storage/base.py
----------------------------------------------------------------------
diff --git a/libcloud/storage/base.py b/libcloud/storage/base.py
index 66e5d44..ead1545 100644
--- a/libcloud/storage/base.py
+++ b/libcloud/storage/base.py
@@ -742,7 +742,8 @@ class StorageDriver(BaseDriver):
if calculate_hash:
data_hash = self._get_hash_function()
- generator = libcloud.utils.files.read_in_chunks(iterator, chunk_size)
+ generator = libcloud.utils.files.read_in_chunks(iterator, chunk_size,
+ fill_size=True)
bytes_transferred = 0
try:
http://git-wip-us.apache.org/repos/asf/libcloud/blob/8fe3e405/libcloud/test/storage/test_cloudfiles.py
----------------------------------------------------------------------
diff --git a/libcloud/test/storage/test_cloudfiles.py b/libcloud/test/storage/test_cloudfiles.py
index 6590270..3e911da 100644
--- a/libcloud/test/storage/test_cloudfiles.py
+++ b/libcloud/test/storage/test_cloudfiles.py
@@ -30,7 +30,7 @@ from libcloud.utils.py3 import httplib
from libcloud.utils.py3 import urlquote
from libcloud.common.types import LibcloudError, MalformedResponseError
-from libcloud.storage.base import Container, Object
+from libcloud.storage.base import CHUNK_SIZE, Container, Object
from libcloud.storage.types import ContainerAlreadyExistsError
from libcloud.storage.types import ContainerDoesNotExistError
from libcloud.storage.types import ContainerIsNotEmptyError
@@ -665,10 +665,6 @@ class CloudFilesTests(unittest.TestCase):
try:
self.driver.upload_object_via_stream(
- # We never reach the Python 3 only bytes vs int error
- # currently at libcloud/utils/py3.py:89
- # raise TypeError("Invalid argument %r for b()" % (s,))
- # because I raise a NotImplementedError.
iterator=iter(b'blob data like an image or video'),
container=container,
object_name="test_object",
@@ -682,6 +678,64 @@ class CloudFilesTests(unittest.TestCase):
self.fail('Expected NotImplementedError to be thrown to '
'verify we actually checked the expected headers')
+ def test_upload_object_via_stream_python3_bytes_error(self):
+ container = Container(name='py3', extra={}, driver=self.driver)
+ bytes_blob = b'blob data like an image or video'
+
+ # This is mostly to check we didn't discover other errors along the way
+ mocked_response = container.upload_object_via_stream(
+ iterator=iter(bytes_blob),
+ object_name="img_or_vid",
+ )
+ self.assertEqual(len(bytes_blob), mocked_response.size)
+
+ def test_upload_object_via_stream_chunked_encoding(self):
+
+ # Create enough bytes it should get split into two chunks
+ bytes_blob = ''.join(['\0' for _ in range(CHUNK_SIZE + 1)])
+ hex_chunk_size = ('%X' % CHUNK_SIZE).encode('utf8')
+ expected = [
+ # Chunk 1
+ hex_chunk_size + b'\r\n',
+ bytes(bytes_blob[:CHUNK_SIZE].encode('utf8')),
+ b'\r\n',
+
+ # Chunk 2
+ b'1\r\n',
+ bytes(bytes_blob[CHUNK_SIZE:].encode('utf8')),
+ b'\r\n',
+
+ # If chunked, also send a final message
+ b'0\r\n\r\n',
+ ]
+ logged_data = []
+
+ class InterceptResponse(CloudFilesMockRawResponse):
+ def __init__(self, connection):
+ super(InterceptResponse, self).__init__(connection=connection)
+ old_send = self.connection.connection.send
+
+ def intercept_send(data):
+ old_send(data)
+ logged_data.append(data)
+ self.connection.connection.send = intercept_send
+
+ def _v1_MossoCloudFS_py3_img_or_vid2(self,
+ method, url, body, headers):
+ headers = {'etag': 'd79fb00c27b50494a463e680d459c90c'}
+ headers.update(self.base_headers)
+ _201 = httplib.CREATED
+ return _201, '', headers, httplib.responses[_201]
+
+ self.driver_klass.connectionCls.rawResponseCls = InterceptResponse
+
+ container = Container(name='py3', extra={}, driver=self.driver)
+ container.upload_object_via_stream(
+ iterator=iter(bytes_blob),
+ object_name="img_or_vid2",
+ )
+ self.assertListEqual(expected, logged_data)
+
def test__upload_object_manifest(self):
hash_function = self.driver._get_hash_function()
hash_function.update(b(''))
@@ -1087,6 +1141,11 @@ class CloudFilesMockRawResponse(MockRawResponse):
fixtures = StorageFileFixtures('cloudfiles')
base_headers = {'content-type': 'application/json; charset=UTF-8'}
+ def _v1_MossoCloudFS_py3_img_or_vid(self, method, url, body, headers):
+ headers = {'etag': 'e2378cace8712661ce7beec3d9362ef6'}
+ headers.update(self.base_headers)
+ return httplib.CREATED, '', headers, httplib.responses[httplib.CREATED]
+
def _v1_MossoCloudFS_foo_bar_container_foo_test_upload(
self, method, url, body, headers):
# test_object_upload_success
http://git-wip-us.apache.org/repos/asf/libcloud/blob/8fe3e405/libcloud/utils/files.py
----------------------------------------------------------------------
diff --git a/libcloud/utils/files.py b/libcloud/utils/files.py
index a71e1c4..663677a 100644
--- a/libcloud/utils/files.py
+++ b/libcloud/utils/files.py
@@ -38,7 +38,7 @@ def read_in_chunks(iterator, chunk_size=None, fill_size=False,
"""
Return a generator which yields data in chunks.
- :param terator: An object which implements an iterator interface
+ :param iterator: An object which implements an iterator interface
or a File like object with read method.
:type iterator: :class:`object` which implements iterator interface.
http://git-wip-us.apache.org/repos/asf/libcloud/blob/8fe3e405/libcloud/utils/py3.py
----------------------------------------------------------------------
diff --git a/libcloud/utils/py3.py b/libcloud/utils/py3.py
index 4e05419..2b695a4 100644
--- a/libcloud/utils/py3.py
+++ b/libcloud/utils/py3.py
@@ -85,6 +85,8 @@ if PY3:
return s.encode('utf-8')
elif isinstance(s, bytes):
return s
+ elif isinstance(s, int):
+ return bytes([s])
else:
raise TypeError("Invalid argument %r for b()" % (s,))