You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@libcloud.apache.org by to...@apache.org on 2019/05/28 13:35:14 UTC
[libcloud] branch trunk updated (2adabe4 -> f40b2c4)
This is an automated email from the ASF dual-hosted git repository.
tomaz pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/libcloud.git.
from 2adabe4 Wrap lines in CHANGES.rst at 80 characters.
add 494cac5 Fixes LIBCLOUD-1039 - cloudfiles download_object_as_stream
new 49b2206 Merge branch 'fix/LIBCLOUD-1039-fix-download-object-as-stream-cloudfiles' of https://github.com/mattseymour/libcloud into mattseymour-fix/LIBCLOUD-1039-fix-download-object-as-stream-cloudfiles
new 6bb395d Add a test case for it.
new a634635 Add changelog entry for it.
new 54dbe43 Fix and improve tests which were broken (not testing the correct thing) since the migration to requests.
new f40b2c4 Fix Python 3 compatibility.
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGES.rst | 5 +++
libcloud/storage/drivers/cloudfiles.py | 8 +++--
libcloud/test/storage/test_cloudfiles.py | 55 +++++++++++++++++++++++++++++++-
libcloud/test/storage/test_s3.py | 38 +++++++++++++++++++++-
4 files changed, 102 insertions(+), 4 deletions(-)
[libcloud] 05/05: Fix Python 3 compatibility.
Posted by to...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tomaz pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/libcloud.git
commit f40b2c4d2e6531cc4b52972b028f56b17f90cc27
Author: Tomaz Muraus <to...@tomaz.me>
AuthorDate: Tue May 28 15:34:51 2019 +0200
Fix Python 3 compatibility.
---
libcloud/test/storage/test_cloudfiles.py | 8 +++++++-
libcloud/test/storage/test_s3.py | 13 +++++++++++--
2 files changed, 18 insertions(+), 3 deletions(-)
diff --git a/libcloud/test/storage/test_cloudfiles.py b/libcloud/test/storage/test_cloudfiles.py
index e3c6ff4..650cc53 100644
--- a/libcloud/test/storage/test_cloudfiles.py
+++ b/libcloud/test/storage/test_cloudfiles.py
@@ -33,6 +33,7 @@ from libcloud.utils.py3 import b
from libcloud.utils.py3 import httplib
from libcloud.utils.py3 import urlquote
from libcloud.utils.py3 import StringIO
+from libcloud.utils.py3 import PY3
from libcloud.utils.files import exhaust_iterator
from libcloud.common.types import MalformedResponseError
@@ -392,7 +393,12 @@ class CloudFilesTests(unittest.TestCase):
container=container, meta_data=None,
driver=self.driver)
result = self.driver.download_object_as_stream(obj=obj)
- self.assertEqual(exhaust_iterator(result), 'a' * 1000)
+ result = exhaust_iterator(result)
+
+ if PY3:
+ result = result.decode('utf-8')
+
+ self.assertEqual(result, 'a' * 1000)
def test_upload_object_success(self):
def upload_file(self, object_name=None, content_type=None,
diff --git a/libcloud/test/storage/test_s3.py b/libcloud/test/storage/test_s3.py
index df31920..cc756fb 100644
--- a/libcloud/test/storage/test_s3.py
+++ b/libcloud/test/storage/test_s3.py
@@ -31,6 +31,7 @@ from libcloud.utils.py3 import httplib
from libcloud.utils.py3 import urlparse
from libcloud.utils.py3 import parse_qs
from libcloud.utils.py3 import StringIO
+from libcloud.utils.py3 import PY3
from libcloud.utils.files import exhaust_iterator
from libcloud.common.types import InvalidCredsError
@@ -666,7 +667,10 @@ class S3Tests(unittest.TestCase):
self.assertTrue(result)
def test_download_object_as_stream_data_is_not_buffered_in_memory(self):
- # If content is consumed and response.content attribute accessed execption
+ # Test case which verifies that response.response attribute is not accessed
+ # and as such, whole body response is not buffered into RAM
+
+ # If content is consumed and response.content attribute accessed exception
# will be thrown and test will fail
mock_response = Mock(name='mock response')
mock_response.headers = {}
@@ -690,7 +694,12 @@ class S3Tests(unittest.TestCase):
driver=self.driver_type)
destination_path = self._file_path
result = self.driver.download_object_as_stream(obj=obj)
- self.assertEqual(exhaust_iterator(result), 'a' * 1000)
+ result = exhaust_iterator(result)
+
+ if PY3:
+ result = result.decode('utf-8')
+
+ self.assertEqual(result, 'a' * 1000)
def test_download_object_invalid_file_size(self):
self.mock_response_klass.type = 'INVALID_SIZE'
[libcloud] 03/05: Add changelog entry for it.
Posted by to...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tomaz pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/libcloud.git
commit a6346353d74b6a18bdba929f1bf23257d06c728e
Author: Tomaz Muraus <to...@tomaz.me>
AuthorDate: Tue May 28 15:07:38 2019 +0200
Add changelog entry for it.
---
CHANGES.rst | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/CHANGES.rst b/CHANGES.rst
index 3fa02c6..5d5d3dc 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -146,6 +146,11 @@ Storage
support seek, entire iterator content will be buffered in memory.
(LIBCLOUD-1043, GITHUB-1287)
[Clemens Wolff]
+- [CloudFiles] Fix ``download_object_as_stream`` method in the CloudFiles
+ driver. This regression / bug was inadvertently introduced when migrating
+ code to ``requests``.
+ (LIBCLOUD-1039, GITHUB-1283)
+ [Matt Seymour]
Container
~~~~~~~~~
[libcloud] 02/05: Add a test case for it.
Posted by to...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tomaz pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/libcloud.git
commit 6bb395dcbbefde09bcdd70c3151abce193162b25
Author: Tomaz Muraus <to...@tomaz.me>
AuthorDate: Tue May 28 15:05:57 2019 +0200
Add a test case for it.
---
libcloud/test/storage/test_cloudfiles.py | 49 +++++++++++++++++++++++++++++++-
1 file changed, 48 insertions(+), 1 deletion(-)
diff --git a/libcloud/test/storage/test_cloudfiles.py b/libcloud/test/storage/test_cloudfiles.py
index 05ac82f..a157408 100644
--- a/libcloud/test/storage/test_cloudfiles.py
+++ b/libcloud/test/storage/test_cloudfiles.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from hashlib import sha1
+
import hmac
import os
import os.path # pylint: disable-msg=W0404
@@ -21,13 +21,18 @@ import math
import sys
import copy
from io import BytesIO
+from hashlib import sha1
+
import mock
+from mock import Mock
+from mock import PropertyMock
import libcloud.utils.files
from libcloud.utils.py3 import b
from libcloud.utils.py3 import httplib
from libcloud.utils.py3 import urlquote
+from libcloud.utils.py3 import StringIO
from libcloud.common.types import MalformedResponseError
from libcloud.storage.base import CHUNK_SIZE, Container, Object
@@ -360,6 +365,35 @@ class CloudFilesTests(unittest.TestCase):
obj=obj, chunk_size=None)
self.assertTrue(hasattr(stream, '__iter__'))
+ def test_download_object_data_is_not_buffered_in_memory(self):
+ # Test case which verifies that response.body attribute is not accessed
+ # and as such, whole body response is not buffered into RAM
+
+ # If content is consumed and response.content attribute accessed execption
+ # will be thrown and test will fail
+ mock_response = Mock(name='mock response')
+ mock_response.headers = {}
+ mock_response.status_code = 200
+ msg = '"content" attribute was accessed but it shouldn\'t have been'
+ type(mock_response).content = PropertyMock(name='mock content attribute',
+ side_effect=Exception(msg))
+ mock_response.iter_content.return_value = StringIO('a' * 1000)
+
+ self.driver.connection.connection.getresponse = Mock()
+ self.driver.connection.connection.getresponse.return_value = mock_response
+
+ container = Container(name='foo_bar_container', extra={},
+ driver=self.driver)
+ obj = Object(name='foo_bar_object_NO_BUFFER', size=1000, hash=None, extra={},
+ container=container, meta_data=None,
+ driver=self.driver)
+ destination_path = os.path.abspath(__file__) + '.temp'
+ result = self.driver.download_object(obj=obj,
+ destination_path=destination_path,
+ overwrite_existing=False,
+ delete_on_failure=True)
+ self.assertTrue(result)
+
def test_upload_object_success(self):
def upload_file(self, object_name=None, content_type=None,
request_path=None, request_method=None,
@@ -1184,5 +1218,18 @@ class CloudFilesMockHttp(MockHttp, unittest.TestCase):
headers,
httplib.responses[httplib.OK])
+ def _v1_MossoCloudFS_foo_bar_container_foo_bar_object_NO_BUFFER(
+ self, method, url, body, headers):
+ # test_download_object_data_is_not_buffered_in_memory
+ headers = {}
+ headers.update(self.base_headers)
+ headers['etag'] = '577ef1154f3240ad5b9b413aa7346a1e'
+ body = generate_random_data(1000)
+ return (httplib.OK,
+ body,
+ headers,
+ httplib.responses[httplib.OK])
+
+
if __name__ == '__main__':
sys.exit(unittest.main())
[libcloud] 04/05: Fix and improve tests which were broken (not
testing the correct thing) since the migration to requests.
Posted by to...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tomaz pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/libcloud.git
commit 54dbe437b87956f67da7acfe429a20546e4067d0
Author: Tomaz Muraus <to...@tomaz.me>
AuthorDate: Tue May 28 15:20:31 2019 +0200
Fix and improve tests which were broken (not testing the correct thing)
since the migration to requests.
---
libcloud/test/storage/test_cloudfiles.py | 28 ++++++++++++++--------------
libcloud/test/storage/test_s3.py | 29 ++++++++++++++++++++++++++++-
2 files changed, 42 insertions(+), 15 deletions(-)
diff --git a/libcloud/test/storage/test_cloudfiles.py b/libcloud/test/storage/test_cloudfiles.py
index a157408..e3c6ff4 100644
--- a/libcloud/test/storage/test_cloudfiles.py
+++ b/libcloud/test/storage/test_cloudfiles.py
@@ -33,6 +33,7 @@ from libcloud.utils.py3 import b
from libcloud.utils.py3 import httplib
from libcloud.utils.py3 import urlquote
from libcloud.utils.py3 import StringIO
+from libcloud.utils.files import exhaust_iterator
from libcloud.common.types import MalformedResponseError
from libcloud.storage.base import CHUNK_SIZE, Container, Object
@@ -365,34 +366,33 @@ class CloudFilesTests(unittest.TestCase):
obj=obj, chunk_size=None)
self.assertTrue(hasattr(stream, '__iter__'))
- def test_download_object_data_is_not_buffered_in_memory(self):
- # Test case which verifies that response.body attribute is not accessed
+ def test_download_object_as_stream_data_is_not_buffered_in_memory(self):
+ # Test case which verifies that response.response attribute is not accessed
# and as such, whole body response is not buffered into RAM
- # If content is consumed and response.content attribute accessed execption
+ # If content is consumed and response.content attribute accessed exception
# will be thrown and test will fail
mock_response = Mock(name='mock response')
mock_response.headers = {}
- mock_response.status_code = 200
- msg = '"content" attribute was accessed but it shouldn\'t have been'
+ mock_response.status = 200
+ msg1 = '"response" attribute was accessed but it shouldn\'t have been'
+ msg2 = '"content" attribute was accessed but it shouldn\'t have been'
+ type(mock_response).response = PropertyMock(name='mock response attribute',
+ side_effect=Exception(msg1))
type(mock_response).content = PropertyMock(name='mock content attribute',
- side_effect=Exception(msg))
+ side_effect=Exception(msg2))
mock_response.iter_content.return_value = StringIO('a' * 1000)
- self.driver.connection.connection.getresponse = Mock()
- self.driver.connection.connection.getresponse.return_value = mock_response
+ self.driver.connection.request = Mock()
+ self.driver.connection.request.return_value = mock_response
container = Container(name='foo_bar_container', extra={},
driver=self.driver)
obj = Object(name='foo_bar_object_NO_BUFFER', size=1000, hash=None, extra={},
container=container, meta_data=None,
driver=self.driver)
- destination_path = os.path.abspath(__file__) + '.temp'
- result = self.driver.download_object(obj=obj,
- destination_path=destination_path,
- overwrite_existing=False,
- delete_on_failure=True)
- self.assertTrue(result)
+ result = self.driver.download_object_as_stream(obj=obj)
+ self.assertEqual(exhaust_iterator(result), 'a' * 1000)
def test_upload_object_success(self):
def upload_file(self, object_name=None, content_type=None,
diff --git a/libcloud/test/storage/test_s3.py b/libcloud/test/storage/test_s3.py
index b214b51..df31920 100644
--- a/libcloud/test/storage/test_s3.py
+++ b/libcloud/test/storage/test_s3.py
@@ -31,6 +31,7 @@ from libcloud.utils.py3 import httplib
from libcloud.utils.py3 import urlparse
from libcloud.utils.py3 import parse_qs
from libcloud.utils.py3 import StringIO
+from libcloud.utils.files import exhaust_iterator
from libcloud.common.types import InvalidCredsError
from libcloud.common.types import LibcloudError, MalformedResponseError
@@ -641,7 +642,6 @@ class S3Tests(unittest.TestCase):
# If content is consumed and response.content attribute accessed execption
# will be thrown and test will fail
-
mock_response = Mock(name='mock response')
mock_response.headers = {}
mock_response.status_code = 200
@@ -665,6 +665,33 @@ class S3Tests(unittest.TestCase):
delete_on_failure=True)
self.assertTrue(result)
+ def test_download_object_as_stream_data_is_not_buffered_in_memory(self):
+ # If content is consumed and response.content attribute accessed execption
+ # will be thrown and test will fail
+ mock_response = Mock(name='mock response')
+ mock_response.headers = {}
+ mock_response.status = 200
+
+ msg1 = '"response" attribute was accessed but it shouldn\'t have been'
+ msg2 = '"content" attribute was accessed but it shouldn\'t have been'
+ type(mock_response).response = PropertyMock(name='mock response attribute',
+ side_effect=Exception(msg1))
+ type(mock_response).content = PropertyMock(name='mock content attribute',
+ side_effect=Exception(msg2))
+ mock_response.iter_content.return_value = StringIO('a' * 1000)
+
+ self.driver.connection.request = Mock()
+ self.driver.connection.request.return_value = mock_response
+
+ container = Container(name='foo_bar_container', extra={},
+ driver=self.driver)
+ obj = Object(name='foo_bar_object_NO_BUFFER', size=1000, hash=None, extra={},
+ container=container, meta_data=None,
+ driver=self.driver_type)
+ destination_path = self._file_path
+ result = self.driver.download_object_as_stream(obj=obj)
+ self.assertEqual(exhaust_iterator(result), 'a' * 1000)
+
def test_download_object_invalid_file_size(self):
self.mock_response_klass.type = 'INVALID_SIZE'
container = Container(name='foo_bar_container', extra={},
[libcloud] 01/05: Merge branch
'fix/LIBCLOUD-1039-fix-download-object-as-stream-cloudfiles' of
https://github.com/mattseymour/libcloud into
mattseymour-fix/LIBCLOUD-1039-fix-download-object-as-stream-cloudfiles
Posted by to...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tomaz pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/libcloud.git
commit 49b2206729ff4bacfef6803a1424e2ea6fa9045b
Merge: 2adabe4 494cac5
Author: Tomaz Muraus <to...@tomaz.me>
AuthorDate: Tue May 28 14:58:42 2019 +0200
Merge branch 'fix/LIBCLOUD-1039-fix-download-object-as-stream-cloudfiles' of https://github.com/mattseymour/libcloud into mattseymour-fix/LIBCLOUD-1039-fix-download-object-as-stream-cloudfiles
libcloud/storage/drivers/cloudfiles.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)