You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@libcloud.apache.org by to...@apache.org on 2014/12/01 06:34:27 UTC
libcloud git commit: Allow user to pass "headers" argument to the
upload_* methods in the storage API.
Repository: libcloud
Updated Branches:
refs/heads/trunk a4323ca7d -> a19d1df1d
Allow user to pass "headers" argument to the upload_* methods in the storage
API.
This way user can specify CORS headers with the providers which supported that.
Closes #404
Signed-off-by: Tomaz Muraus <to...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/a19d1df1
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/a19d1df1
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/a19d1df1
Branch: refs/heads/trunk
Commit: a19d1df1d093975eb0ef2d3c829dd107bada872f
Parents: a4323ca
Author: Peter Schmidt <pe...@peterjs.com>
Authored: Thu Nov 27 11:20:38 2014 +1100
Committer: Tomaz Muraus <to...@apache.org>
Committed: Mon Dec 1 06:28:49 2014 +0100
----------------------------------------------------------------------
CHANGES.rst | 10 ++++
docs/development.rst | 4 +-
libcloud/storage/base.py | 71 ++++++++++++++++-----------
libcloud/storage/drivers/cloudfiles.py | 14 +++---
libcloud/test/storage/test_cloudfiles.py | 49 +++++++++++++++++-
5 files changed, 109 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/a19d1df1/CHANGES.rst
----------------------------------------------------------------------
diff --git a/CHANGES.rst b/CHANGES.rst
index f55ebea..012f5e9 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -61,6 +61,16 @@ Compute
(GITHUB-401)
[Eric Johnson]
+Storage
+~~~~~~~
+
+- Allow user to pass ``headers`` argument to the ``upload_object`` and
+ ``upload_object_via_stream`` method.
+
+ This way user can specify CORS headers with the drivers which support that.
+ (GITHUB-403, GITHUB-404)
+ [Peter Schmidt]
+
Changes with Apache Libcloud 0.16.0
-----------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/a19d1df1/docs/development.rst
----------------------------------------------------------------------
diff --git a/docs/development.rst b/docs/development.rst
index 8899df9..09cfe2b 100644
--- a/docs/development.rst
+++ b/docs/development.rst
@@ -36,9 +36,9 @@ Code style guide
* Use 79 characters in a line
* Make sure edited file doesn't contain any trailing whitespace
* You can verify that your modifications don't break any rules by running the
- ``flake8`` script - e.g. ``flake8 libcloud/edited_file.py.`` or
+ ``flake8`` script - e.g. ``flake8 libcloud/edited_file.py`` or
``tox -e lint``.
- Second command fill run flake8 on all the files in the repository.
+ Second command will run flake8 on all the files in the repository.
And most importantly, follow the existing style in the file you are editing and
**be consistent**.
http://git-wip-us.apache.org/repos/asf/libcloud/blob/a19d1df1/libcloud/storage/base.py
----------------------------------------------------------------------
diff --git a/libcloud/storage/base.py b/libcloud/storage/base.py
index 5e4f09a..66e5d44 100644
--- a/libcloud/storage/base.py
+++ b/libcloud/storage/base.py
@@ -361,7 +361,7 @@ class StorageDriver(BaseDriver):
'download_object_as_stream not implemented for this driver')
def upload_object(self, file_path, container, object_name, extra=None,
- verify_hash=True):
+ verify_hash=True, headers=None):
"""
Upload an object currently located on a disk.
@@ -380,6 +380,11 @@ class StorageDriver(BaseDriver):
:param extra: Extra attributes (driver specific). (optional)
:type extra: ``dict``
+ :param headers: (optional) Additional request headers,
+ such as CORS headers. For example:
+ headers = {'Access-Control-Allow-Origin': 'http://mozilla.com'}
+ :type headers: ``dict``
+
:rtype: :class:`Object`
"""
raise NotImplementedError(
@@ -387,7 +392,8 @@ class StorageDriver(BaseDriver):
def upload_object_via_stream(self, iterator, container,
object_name,
- extra=None):
+ extra=None,
+ headers=None):
"""
Upload an object using an iterator.
@@ -405,19 +411,24 @@ class StorageDriver(BaseDriver):
function which uses fs.stat function to determine the file size and it
doesn't need to buffer whole object in the memory.
- :type iterator: :class:`object`
:param iterator: An object which implements the iterator interface.
+ :type iterator: :class:`object`
- :type container: :class:`Container`
:param container: Destination container.
+ :type container: :class:`Container`
- :type object_name: ``str``
:param object_name: Object name.
+ :type object_name: ``str``
- :type extra: ``dict``
:param extra: (optional) Extra attributes (driver specific). Note:
This dictionary must contain a 'content_type' key which represents
a content type of the stored object.
+ :type extra: ``dict``
+
+ :param headers: (optional) Additional request headers,
+ such as CORS headers. For example:
+ headers = {'Access-Control-Allow-Origin': 'http://mozilla.com'}
+ :type headers: ``dict``
:rtype: ``object``
"""
@@ -428,8 +439,8 @@ class StorageDriver(BaseDriver):
"""
Delete an object.
- :type obj: :class:`Object`
:param obj: Object instance.
+ :type obj: :class:`Object`
:return: ``bool`` True on success.
:rtype: ``bool``
@@ -441,8 +452,8 @@ class StorageDriver(BaseDriver):
"""
Create a new container.
- :type container_name: ``str``
:param container_name: Container name.
+ :type container_name: ``str``
:return: Container instance on success.
:rtype: :class:`Container`
@@ -454,8 +465,8 @@ class StorageDriver(BaseDriver):
"""
Delete a container.
- :type container: :class:`Container`
:param container: Container instance
+ :type container: :class:`Container`
:return: ``True`` on success, ``False`` otherwise.
:rtype: ``bool``
@@ -468,23 +479,23 @@ class StorageDriver(BaseDriver):
"""
Call passed callback and start transfer of the object'
- :type obj: :class:`Object`
:param obj: Object instance.
+ :type obj: :class:`Object`
- :type callback: :class:`function`
:param callback: Function which is called with the passed
callback_kwargs
+ :type callback: :class:`function`
- :type callback_kwargs: ``dict``
:param callback_kwargs: Keyword arguments which are passed to the
callback.
+ :type callback_kwargs: ``dict``
- :typed response: :class:`Response`
:param response: Response instance.
+ :type response: :class:`Response`
- :type success_status_code: ``int``
:param success_status_code: Status code which represents a successful
transfer (defaults to httplib.OK)
+ :type success_status_code: ``int``
:return: ``True`` on success, ``False`` otherwise.
:rtype: ``bool``
@@ -507,26 +518,26 @@ class StorageDriver(BaseDriver):
"""
Save object to the provided path.
- :type response: :class:`RawResponse`
:param response: RawResponse instance.
+ :type response: :class:`RawResponse`
- :type obj: :class:`Object`
:param obj: Object instance.
+ :type obj: :class:`Object`
- :type destination_path: ``str``
:param destination_path: Destination directory.
+ :type destination_path: ``str``
- :type delete_on_failure: ``bool``
:param delete_on_failure: True to delete partially downloaded object if
the download fails.
+ :type delete_on_failure: ``bool``
- :type overwrite_existing: ``bool``
:param overwrite_existing: True to overwrite a local path if it already
exists.
+ :type overwrite_existing: ``bool``
- :type chunk_size: ``int``
:param chunk_size: Optional chunk size
(defaults to ``libcloud.storage.base.CHUNK_SIZE``, 8kb)
+ :type chunk_size: ``int``
:return: ``True`` on success, ``False`` otherwise.
:rtype: ``bool``
@@ -660,20 +671,20 @@ class StorageDriver(BaseDriver):
"""
Upload data stored in a string.
- :type response: :class:`RawResponse`
:param response: RawResponse object.
+ :type response: :class:`RawResponse`
- :type data: ``str``
:param data: Data to upload.
+ :type data: ``str``
- :type calculate_hash: ``bool``
:param calculate_hash: True to calculate hash of the transferred data.
- (defauls to True).
+ (defaults to True).
+ :type calculate_hash: ``bool``
- :rtype: ``tuple``
:return: First item is a boolean indicator of success, second
one is the uploaded data MD5 hash and the third one
is the number of transferred bytes.
+ :rtype: ``tuple``
"""
bytes_transferred = 0
data_hash = None
@@ -701,23 +712,23 @@ class StorageDriver(BaseDriver):
"""
Stream a data over an http connection.
- :type response: :class:`RawResponse`
:param response: RawResponse object.
+ :type response: :class:`RawResponse`
- :type iterator: :class:`object`
:param response: An object which implements an iterator interface
or a File like object with read method.
+ :type iterator: :class:`object`
- :type chunked: ``bool``
:param chunked: True if the chunked transfer encoding should be used
(defauls to False).
+ :type chunked: ``bool``
- :type calculate_hash: ``bool``
:param calculate_hash: True to calculate hash of the transferred data.
(defauls to True).
+ :type calculate_hash: ``bool``
- :type chunk_size: ``int``
:param chunk_size: Optional chunk size (defaults to ``CHUNK_SIZE``)
+ :type chunk_size: ``int``
:rtype: ``tuple``
:return: First item is a boolean indicator of success, second
http://git-wip-us.apache.org/repos/asf/libcloud/blob/a19d1df1/libcloud/storage/drivers/cloudfiles.py
----------------------------------------------------------------------
diff --git a/libcloud/storage/drivers/cloudfiles.py b/libcloud/storage/drivers/cloudfiles.py
index a2189c1..314586b 100644
--- a/libcloud/storage/drivers/cloudfiles.py
+++ b/libcloud/storage/drivers/cloudfiles.py
@@ -414,7 +414,7 @@ class CloudFilesStorageDriver(StorageDriver, OpenStackDriverMixin):
success_status_code=httplib.OK)
def upload_object(self, file_path, container, object_name, extra=None,
- verify_hash=True):
+ verify_hash=True, headers=None):
"""
Upload an object.
@@ -427,10 +427,11 @@ class CloudFilesStorageDriver(StorageDriver, OpenStackDriverMixin):
upload_func=upload_func,
upload_func_kwargs=upload_func_kwargs,
extra=extra, file_path=file_path,
- verify_hash=verify_hash)
+ verify_hash=verify_hash, headers=headers)
def upload_object_via_stream(self, iterator,
- container, object_name, extra=None):
+ container, object_name, extra=None,
+ headers=None):
if isinstance(iterator, file):
iterator = iter(iterator)
@@ -440,7 +441,8 @@ class CloudFilesStorageDriver(StorageDriver, OpenStackDriverMixin):
return self._put_object(container=container, object_name=object_name,
upload_func=upload_func,
upload_func_kwargs=upload_func_kwargs,
- extra=extra, iterator=iterator)
+ extra=extra, iterator=iterator,
+ headers=headers)
def delete_object(self, obj):
container_name = self._encode_container_name(obj.container.name)
@@ -752,7 +754,7 @@ class CloudFilesStorageDriver(StorageDriver, OpenStackDriverMixin):
def _put_object(self, container, object_name, upload_func,
upload_func_kwargs, extra=None, file_path=None,
- iterator=None, verify_hash=True):
+ iterator=None, verify_hash=True, headers=None):
extra = extra or {}
container_name_encoded = self._encode_container_name(container.name)
object_name_encoded = self._encode_object_name(object_name)
@@ -760,7 +762,7 @@ class CloudFilesStorageDriver(StorageDriver, OpenStackDriverMixin):
meta_data = extra.get('meta_data', None)
content_disposition = extra.get('content_disposition', None)
- headers = {}
+ headers = headers or {}
if meta_data:
for key, value in list(meta_data.items()):
key = 'X-Object-Meta-%s' % (key)
http://git-wip-us.apache.org/repos/asf/libcloud/blob/a19d1df1/libcloud/test/storage/test_cloudfiles.py
----------------------------------------------------------------------
diff --git a/libcloud/test/storage/test_cloudfiles.py b/libcloud/test/storage/test_cloudfiles.py
index 70e52c8..6590270 100644
--- a/libcloud/test/storage/test_cloudfiles.py
+++ b/libcloud/test/storage/test_cloudfiles.py
@@ -20,7 +20,6 @@ import os.path # pylint: disable-msg=W0404
import math
import sys
import copy
-import unittest
import mock
@@ -45,6 +44,7 @@ from libcloud.storage.drivers.dummy import DummyIterator
from libcloud.test import StorageMockHttp, MockRawResponse # pylint: disable-msg=E0611
from libcloud.test import MockHttpTestCase # pylint: disable-msg=E0611
+from libcloud.test import unittest
from libcloud.test.file_fixtures import StorageFileFixtures # pylint: disable-msg=E0611
@@ -635,6 +635,53 @@ class CloudFilesTests(unittest.TestCase):
self.assertEqual(func_kwargs['object_name'], expected_name)
self.assertEqual(func_kwargs['container'], container)
+ def test_upload_object_via_stream_with_cors_headers(self):
+ """
+ Test we can add some ``Cross-origin resource sharing`` headers
+ to the request about to be sent.
+ """
+ cors_headers = {
+ 'Access-Control-Allow-Origin': 'http://mozilla.com',
+ 'Origin': 'http://storage.clouddrive.com',
+ }
+ expected_headers = {
+ # Automatically added headers
+ 'Content-Type': 'application/octet-stream',
+ 'Transfer-Encoding': 'chunked',
+ }
+ expected_headers.update(cors_headers)
+
+ def intercept_request(request_path,
+ method=None, data=None,
+ headers=None, raw=True):
+
+ # What we're actually testing
+ self.assertDictEqual(expected_headers, headers)
+
+ raise NotImplementedError('oops')
+ self.driver.connection.request = intercept_request
+
+ container = Container(name='CORS', extra={}, driver=self.driver)
+
+ try:
+ self.driver.upload_object_via_stream(
+ # We never reach the Python 3 only bytes vs int error
+ # currently at libcloud/utils/py3.py:89
+ # raise TypeError("Invalid argument %r for b()" % (s,))
+ # because I raise a NotImplementedError.
+ iterator=iter(b'blob data like an image or video'),
+ container=container,
+ object_name="test_object",
+ headers=cors_headers,
+ )
+ except NotImplementedError:
+ # Don't care about the response we'd have to mock anyway
+ # as long as we intercepted the request and checked its headers
+ pass
+ else:
+ self.fail('Expected NotImplementedError to be thrown to '
+ 'verify we actually checked the expected headers')
+
def test__upload_object_manifest(self):
hash_function = self.driver._get_hash_function()
hash_function.update(b(''))