You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@libcloud.apache.org by to...@apache.org on 2011/05/15 01:23:12 UTC
svn commit: r1103251 - in /incubator/libcloud/trunk: libcloud/storage/
libcloud/storage/drivers/ test/storage/
Author: tomaz
Date: Sat May 14 23:23:11 2011
New Revision: 1103251
URL: http://svn.apache.org/viewvc?rev=1103251&view=rev
Log:
Remove file_hash and add verify_hash argument and make it consistent across all
the drivers and update affects tests.
Modified:
incubator/libcloud/trunk/libcloud/storage/base.py
incubator/libcloud/trunk/libcloud/storage/drivers/cloudfiles.py
incubator/libcloud/trunk/libcloud/storage/drivers/dummy.py
incubator/libcloud/trunk/libcloud/storage/drivers/s3.py
incubator/libcloud/trunk/libcloud/storage/types.py
incubator/libcloud/trunk/test/storage/test_cloudfiles.py
incubator/libcloud/trunk/test/storage/test_s3.py
Modified: incubator/libcloud/trunk/libcloud/storage/base.py
URL: http://svn.apache.org/viewvc/incubator/libcloud/trunk/libcloud/storage/base.py?rev=1103251&r1=1103250&r2=1103251&view=diff
==============================================================================
--- incubator/libcloud/trunk/libcloud/storage/base.py (original)
+++ incubator/libcloud/trunk/libcloud/storage/base.py Sat May 14 23:23:11 2011
@@ -122,9 +122,9 @@ class Container(object):
return self.driver.get_object(container_name=self.name,
object_name=object_name)
- def upload_object(self, file_path, object_name, extra=None, file_hash=None):
+ def upload_object(self, file_path, object_name, extra=None, verify_hash=True):
return self.driver.upload_object(
- file_path, self, object_name, extra, file_hash)
+ file_path, self, object_name, extra, verify_hash)
def upload_object_via_stream(self, iterator, object_name, extra=None):
return self.driver.upload_object_via_stream(
@@ -301,7 +301,7 @@ class StorageDriver(object):
'download_object_as_stream not implemented for this driver')
def upload_object(self, file_path, container, object_name, extra=None,
- file_hash=None):
+ verify_hash=True):
"""
Upload an object.
@@ -317,10 +317,8 @@ class StorageDriver(object):
@type extra: C{dict}
@param extra: (optional) Extra attributes (driver specific).
- @type file_hash: C{str}
- @param file_hash: (optional) File hash. If provided object hash is
- on upload and if it doesn't match the one provided an
- exception is thrown.
+ @type verify_hash: C{boolean}
+ @param verify_hash: True to do a file integrity check.
"""
raise NotImplementedError(
'upload_object not implemented for this driver')
Modified: incubator/libcloud/trunk/libcloud/storage/drivers/cloudfiles.py
URL: http://svn.apache.org/viewvc/incubator/libcloud/trunk/libcloud/storage/drivers/cloudfiles.py?rev=1103251&r1=1103250&r2=1103251&view=diff
==============================================================================
--- incubator/libcloud/trunk/libcloud/storage/drivers/cloudfiles.py (original)
+++ incubator/libcloud/trunk/libcloud/storage/drivers/cloudfiles.py Sat May 14 23:23:11 2011
@@ -203,7 +203,6 @@ class CloudFilesStorageDriver(StorageDri
response = self.connection.request('/%s/%s' % (container_name,
object_name),
method='HEAD')
-
if response.status in [ httplib.OK, httplib.NO_CONTENT ]:
obj = self._headers_to_object(
object_name, container, response.headers)
@@ -309,7 +308,7 @@ class CloudFilesStorageDriver(StorageDri
success_status_code=httplib.OK)
def upload_object(self, file_path, container, object_name, extra=None,
- file_hash=None):
+ verify_hash=True):
"""
Upload an object.
@@ -322,7 +321,7 @@ class CloudFilesStorageDriver(StorageDri
upload_func=upload_func,
upload_func_kwargs=upload_func_kwargs,
extra=extra, file_path=file_path,
- file_hash=file_hash)
+ verify_hash=verify_hash)
def upload_object_via_stream(self, iterator,
container, object_name, extra=None):
@@ -354,7 +353,7 @@ class CloudFilesStorageDriver(StorageDri
def _put_object(self, container, object_name, upload_func,
upload_func_kwargs, extra=None, file_path=None,
- iterator=None, file_hash=None):
+ iterator=None, verify_hash=True):
extra = extra or {}
container_name_cleaned = self._clean_container_name(container.name)
object_name_cleaned = self._clean_object_name(object_name)
@@ -362,9 +361,6 @@ class CloudFilesStorageDriver(StorageDri
meta_data = extra.get('meta_data', None)
headers = {}
- if not iterator and file_hash:
- headers['ETag'] = file_hash
-
if meta_data:
for key, value in meta_data.iteritems():
key = 'X-Object-Meta-%s' % (key)
@@ -382,17 +378,23 @@ class CloudFilesStorageDriver(StorageDri
response = result_dict['response'].response
bytes_transferred = result_dict['bytes_transferred']
+ print result_dict['data_hash']
+ server_hash = result_dict['response'].headers.get('etag', None)
if response.status == httplib.EXPECTATION_FAILED:
raise LibcloudError(value='Missing content-type header',
driver=self)
- elif response.status == httplib.UNPROCESSABLE_ENTITY:
+ elif verify_hash and not server_hash:
+ raise LibcloudError(value='Server didn\'t return etag',
+ driver=self)
+ elif (verify_hash and result_dict['data_hash'] != server_hash):
raise ObjectHashMismatchError(
- value='MD5 hash checksum does not match',
+ value=('MD5 hash checksum does not match (expected=%s, ' +
+ 'actual=%s)') % (result_dict['data_hash'], server_hash),
object_name=object_name, driver=self)
elif response.status == httplib.CREATED:
obj = Object(
- name=object_name, size=bytes_transferred, hash=file_hash,
+ name=object_name, size=bytes_transferred, hash=server_hash,
extra=None, meta_data=meta_data, container=container,
driver=self)
Modified: incubator/libcloud/trunk/libcloud/storage/drivers/dummy.py
URL: http://svn.apache.org/viewvc/incubator/libcloud/trunk/libcloud/storage/drivers/dummy.py?rev=1103251&r1=1103250&r2=1103251&view=diff
==============================================================================
--- incubator/libcloud/trunk/libcloud/storage/drivers/dummy.py (original)
+++ incubator/libcloud/trunk/libcloud/storage/drivers/dummy.py Sat May 14 23:23:11 2011
@@ -15,6 +15,7 @@
import os.path
import random
+import hashlib
from libcloud.common.types import LibcloudError
@@ -48,14 +49,19 @@ class DummyFileObject(file):
class DummyIterator(object):
def __init__(self, data=None):
+ self.hash = hashlib.md5()
self._data = data or []
self._current_item = 0
+ def get_md5_hash(self):
+ return self.hash.hexdigest()
+
def next(self):
if self._current_item == len(self._data):
raise StopIteration
value = self._data[self._current_item]
+ self.hash.update(value)
self._current_item += 1
return value
Modified: incubator/libcloud/trunk/libcloud/storage/drivers/s3.py
URL: http://svn.apache.org/viewvc/incubator/libcloud/trunk/libcloud/storage/drivers/s3.py?rev=1103251&r1=1103250&r2=1103251&view=diff
==============================================================================
--- incubator/libcloud/trunk/libcloud/storage/drivers/s3.py (original)
+++ incubator/libcloud/trunk/libcloud/storage/drivers/s3.py Sat May 14 23:23:11 2011
@@ -290,7 +290,7 @@ class S3StorageDriver(StorageDriver):
success_status_code=httplib.OK)
def upload_object(self, file_path, container, object_name, extra=None,
- file_hash=None, ex_storage_class=None):
+ verify_hash=True, ex_storage_class=None):
upload_func = self._upload_file
upload_func_kwargs = { 'file_path': file_path }
@@ -298,7 +298,7 @@ class S3StorageDriver(StorageDriver):
upload_func=upload_func,
upload_func_kwargs=upload_func_kwargs,
extra=extra, file_path=file_path,
- file_hash=file_hash,
+ verify_hash=verify_hash,
storage_class=ex_storage_class)
def upload_object_via_stream(self, iterator, container, object_name,
@@ -328,7 +328,7 @@ class S3StorageDriver(StorageDriver):
def _put_object(self, container, object_name, upload_func,
upload_func_kwargs, extra=None, file_path=None,
- iterator=None, file_hash=None, storage_class=None):
+ iterator=None, verify_hash=True, storage_class=None):
headers = {}
extra = extra or {}
storage_class = storage_class or 'standard'
@@ -342,9 +342,6 @@ class S3StorageDriver(StorageDriver):
content_type = extra.get('content_type', None)
meta_data = extra.get('meta_data', None)
- if not iterator and file_hash:
- headers['Content-MD5'] = base64.b64encode(file_hash.decode('hex'))
-
if meta_data:
for key, value in meta_data.iteritems():
key = 'x-amz-meta-%s' % (key)
@@ -368,19 +365,22 @@ class S3StorageDriver(StorageDriver):
bytes_transferred = result_dict['bytes_transferred']
headers = response.headers
response = response.response
+ server_hash = headers['etag'].replace('"', '')
- if (file_hash and response.status == httplib.BAD_REQUEST) or \
- (file_hash and file_hash != headers['etag'].replace('"', '')):
+ if (verify_hash and result_dict['data_hash'] != server_hash):
raise ObjectHashMismatchError(
value='MD5 hash checksum does not match',
object_name=object_name, driver=self)
elif response.status == httplib.OK:
obj = Object(
- name=object_name, size=bytes_transferred, hash=file_hash,
+ name=object_name, size=bytes_transferred, hash=server_hash,
extra=None, meta_data=meta_data, container=container,
driver=self)
return obj
+ else:
+ raise LibcloudError('Unexpected status code, status_code=%s' % (response.status),
+ driver=self)
def _to_containers(self, obj, xpath):
return [ self._to_container(element) for element in \
Modified: incubator/libcloud/trunk/libcloud/storage/types.py
URL: http://svn.apache.org/viewvc/incubator/libcloud/trunk/libcloud/storage/types.py?rev=1103251&r1=1103250&r2=1103251&view=diff
==============================================================================
--- incubator/libcloud/trunk/libcloud/storage/types.py (original)
+++ incubator/libcloud/trunk/libcloud/storage/types.py Sat May 14 23:23:11 2011
@@ -67,8 +67,8 @@ class ObjectError(LibcloudError):
super(ObjectError, self).__init__(value=value, driver=driver)
def __str__(self):
- return '<%s in %s, object = %s>' % (self.error_type, repr(self.driver),
- self.object_name)
+ return '<%s in %s, value=%s, object = %s>' % (self.error_type, repr(self.driver),
+ self.value, self.object_name)
class ContainerAlreadyExistsError(ContainerError):
error_type = 'ContainerAlreadyExistsError'
Modified: incubator/libcloud/trunk/test/storage/test_cloudfiles.py
URL: http://svn.apache.org/viewvc/incubator/libcloud/trunk/test/storage/test_cloudfiles.py?rev=1103251&r1=1103250&r2=1103251&view=diff
==============================================================================
--- incubator/libcloud/trunk/test/storage/test_cloudfiles.py (original)
+++ incubator/libcloud/trunk/test/storage/test_cloudfiles.py Sat May 14 23:23:11 2011
@@ -35,6 +35,8 @@ from libcloud.storage.drivers.dummy impo
from test import StorageMockHttp, MockRawResponse # pylint: disable-msg=E0611
from test.file_fixtures import StorageFileFixtures # pylint: disable-msg=E0611
+current_hash = None
+
class CloudFilesTests(unittest.TestCase):
def setUp(self):
@@ -279,7 +281,7 @@ class CloudFilesTests(unittest.TestCase)
try:
self.driver.upload_object(file_path=file_path, container=container,
object_name=object_name,
- file_hash='footest123')
+ verify_hash=True)
except ObjectHashMismatchError:
pass
else:
@@ -598,16 +600,19 @@ class CloudFilesMockRawResponse(MockRawR
# test_object_upload_success
body = ''
- headers = copy.deepcopy(self.base_headers)
- headers.update(headers)
+ headers = {}
+ headers.update(self.base_headers)
+ headers['etag'] = 'hash343hhash89h932439jsaa89'
return (httplib.CREATED, body, headers, httplib.responses[httplib.OK])
def _v1_MossoCloudFS_foo_bar_container_foo_test_upload_INVALID_HASH(
self, method, url, body, headers):
# test_object_upload_invalid_hash
body = ''
- headers = self.base_headers
- return (httplib.UNPROCESSABLE_ENTITY, body, headers,
+ headers = {}
+ headers.update(self.base_headers)
+ headers['etag'] = 'foobar'
+ return (httplib.CREATED, body, headers,
httplib.responses[httplib.OK])
def _v1_MossoCloudFS_foo_bar_container_foo_bar_object(
@@ -641,10 +646,13 @@ class CloudFilesMockRawResponse(MockRawR
self, method, url, body, headers):
# test_upload_object_via_stream_success
+ headers = {}
+ headers.update(self.base_headers)
+ headers['etag'] = '577ef1154f3240ad5b9b413aa7346a1e'
body = 'test'
return (httplib.CREATED,
body,
- self.base_headers,
+ headers,
httplib.responses[httplib.OK])
if __name__ == '__main__':
Modified: incubator/libcloud/trunk/test/storage/test_s3.py
URL: http://svn.apache.org/viewvc/incubator/libcloud/trunk/test/storage/test_s3.py?rev=1103251&r1=1103250&r2=1103251&view=diff
==============================================================================
--- incubator/libcloud/trunk/test/storage/test_s3.py (original)
+++ incubator/libcloud/trunk/test/storage/test_s3.py Sat May 14 23:23:11 2011
@@ -277,7 +277,7 @@ class S3Tests(unittest.TestCase):
try:
self.driver.upload_object(file_path=file_path, container=container,
object_name=object_name,
- file_hash='0cc175b9c0f1b6a831c399e269772661',
+ verify_hash=True,
ex_storage_class='invalid-class')
except ValueError, e:
self.assertTrue(str(e).lower().find('invalid storage class') != -1)
@@ -302,7 +302,7 @@ class S3Tests(unittest.TestCase):
try:
self.driver.upload_object(file_path=file_path, container=container,
object_name=object_name,
- file_hash='0cc175b9c0f1b6a831c399e269772661')
+ verify_hash=True)
except ObjectHashMismatchError:
pass
else:
@@ -328,7 +328,7 @@ class S3Tests(unittest.TestCase):
try:
self.driver.upload_object(file_path=file_path, container=container,
object_name=object_name,
- file_hash='0cc175b9c0f1b6a831c399e269772661')
+ verify_hash=True)
except ObjectHashMismatchError:
pass
else:
@@ -351,7 +351,7 @@ class S3Tests(unittest.TestCase):
obj = self.driver.upload_object(file_path=file_path, container=container,
object_name=object_name,
extra=extra,
- file_hash='0cc175b9c0f1b6a831c399e269772661')
+ verify_hash=True)
self.assertEqual(obj.name, 'foo_test_upload')
self.assertEqual(obj.size, 1000)
self.assertTrue('some-value' in obj.meta_data)
@@ -564,8 +564,10 @@ class S3MockRawResponse(MockRawResponse)
def _foo_bar_container_foo_test_upload_INVALID_HASH1(self, method, url, body, headers):
body = ''
+ headers = {}
+ headers['etag'] = '"foobar"'
# test_upload_object_invalid_hash1
- return (httplib.BAD_REQUEST,
+ return (httplib.OK,
body,
headers,
httplib.responses[httplib.OK])