You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@libcloud.apache.org by to...@apache.org on 2016/01/18 16:18:46 UTC
[02/14] libcloud git commit: Implement upload_object method (wip).
Implement upload_object method (wip).
Note: The whole code is a mess and still needs a lot of refactoring love.
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/02317ce6
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/02317ce6
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/02317ce6
Branch: refs/heads/trunk
Commit: 02317ce6736eaf18b70f3af5ae61334925386e5e
Parents: d6da648
Author: Tomaz Muraus <to...@apache.org>
Authored: Sun Dec 20 14:59:16 2015 +0100
Committer: Tomaz Muraus <to...@apache.org>
Committed: Sun Jan 10 16:54:38 2016 +0100
----------------------------------------------------------------------
libcloud/storage/drivers/backblaze_b2.py | 121 +++++++++++++++++++++++---
1 file changed, 111 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/02317ce6/libcloud/storage/drivers/backblaze_b2.py
----------------------------------------------------------------------
diff --git a/libcloud/storage/drivers/backblaze_b2.py b/libcloud/storage/drivers/backblaze_b2.py
index eb57555..afb9ea6 100644
--- a/libcloud/storage/drivers/backblaze_b2.py
+++ b/libcloud/storage/drivers/backblaze_b2.py
@@ -18,6 +18,7 @@ Driver for Backblaze B2 service.
"""
import base64
+import hashlib
try:
import simplejson as json
@@ -28,10 +29,12 @@ from libcloud.utils.py3 import b
from libcloud.utils.py3 import httplib
from libcloud.utils.py3 import urlparse
from libcloud.utils.files import read_in_chunks
+from libcloud.utils.files import exhaust_iterator
from libcloud.common.base import ConnectionUserAndKey
from libcloud.common.base import JsonResponse
from libcloud.common.types import InvalidCredsError
+from libcloud.common.types import LibcloudError
from libcloud.storage.providers import Provider
from libcloud.storage.base import Object, Container, StorageDriver
@@ -134,9 +137,17 @@ class BackblazeB2Connection(ConnectionUserAndKey):
# dowload url).
self._auth_conn = BackblazeB2AuthConnection(*args, **kwargs)
+ def download_request(self):
+ # TODO
+ pass
+
+ def upload_request(self):
+ # TODO
+ pass
+
def request(self, action, params=None, data=None, headers=None,
method='GET', raw=False, include_account_id=False,
- download_request=False):
+ download_request=False, upload_host=None, auth_token=None):
params = params or {}
headers = headers or {}
@@ -145,13 +156,25 @@ class BackblazeB2Connection(ConnectionUserAndKey):
# Set host
if raw:
+ # TODO: Refactor this mess.
# File download or upload request:
- self.host = auth_conn.download_host
+ if method == 'GET':
+ # Download
+ self.host = auth_conn.download_host
+ elif method == 'POST':
+ self.host = upload_host
else:
self.host = auth_conn.api_host
+ if upload_host:
+ self.host = upload_host
+
# Provide auth token
- headers['Authorization'] = '%s' % (auth_conn.auth_token)
+ # TODO: Refactor
+ if not auth_token:
+ auth_token = auth_conn.auth_token
+
+ headers['Authorization'] = '%s' % (auth_token)
# Include Content-Type
if not raw and data:
@@ -165,12 +188,13 @@ class BackblazeB2Connection(ConnectionUserAndKey):
data = data or {}
data['accountId'] = auth_conn.account_id
- if not raw:
+ if not raw and not upload_host:
action = API_PATH + action
- else:
+ elif method == 'GET':
+ # Download
action = '/file/' + action
- if data:
+ if data and not upload_host:
data = json.dumps(data)
response = super(BackblazeB2Connection, self).request(action=action,
@@ -188,6 +212,7 @@ class BackblazeB2StorageDriver(StorageDriver):
website = 'https://www.backblaze.com/b2/'
type = Provider.BACKBLAZE_B2
hash_type = 'sha1'
+ supports_chunked_encoding = False
def iterate_containers(self):
resp = self.connection.request(action='b2_list_buckets',
@@ -223,10 +248,6 @@ class BackblazeB2StorageDriver(StorageDriver):
include_account_id=True)
return resp.status == httplib.OK
- def upload_object(self, file_path, container, object_name, extra=None,
- verify_hash=True):
- pass
-
def download_object(self, obj, destination_path, overwrite_existing=False,
delete_on_failure=True):
action = self._get_object_download_path(container=obj.container,
@@ -258,6 +279,61 @@ class BackblazeB2StorageDriver(StorageDriver):
'chunk_size': chunk_size},
success_status_code=httplib.OK)
+ def upload_object(self, file_path, container, object_name, extra=None,
+ verify_hash=True, headers=None):
+ """
+ Upload an object.
+
+ Note: This will override file with a same name if it already exists.
+ """
+ # Note: We don't use any of the base driver functions since Backblaze
+ # API requires you to provide SHA1 has upfront and the base methods
+ # don't support that
+ fh = open(file_path, 'rb')
+ iterator = iter(fh)
+ iterator = read_in_chunks(iterator=iterator)
+ data = exhaust_iterator(iterator=iterator)
+
+ extra = extra or {}
+ content_type = extra.get('content_type', 'b2/x-auto')
+ meta_data = extra.get('meta_data', {})
+
+ # Note: Backblaze API doesn't support chunked encoding and we need to
+ # provide Content-Length up front (this is one inside _upload_object):/
+ headers = headers or {}
+ headers['X-Bz-File-Name'] = object_name
+ headers['Content-Type'] = content_type
+
+ sha1 = hashlib.sha1()
+ sha1.update(data.encode())
+ headers['X-Bz-Content-Sha1'] = sha1.hexdigest()
+
+ # Include optional meta-data (up to 10 items)
+ for key, value in meta_data:
+ # TODO: Encode / escape key
+ headers['X-Bz-Info-%s' % (key)] = value
+
+ upload_data = self.ex_get_upload_data(container_id=container.extra['id'])
+ upload_token = upload_data['authorizationToken']
+ parsed_url = urlparse.urlparse(upload_data['uploadUrl'])
+
+ upload_host = parsed_url.netloc
+ request_path = parsed_url.path
+
+ response = self.connection.request(action=request_path, method='POST',
+ headers=headers,
+ upload_host=upload_host,
+ auth_token=upload_token,
+ data=data)
+
+ if response.status == httplib.OK:
+ obj = self._to_object(item=response.object, container=container)
+ return obj
+ else:
+ body = response.read()
+ raise LibcloudError('Upload failed. status_code=%s, body=%s' %
+ (response.status, body), driver=self)
+
def delete_object(self, obj):
data = {}
data['fileName'] = obj.name
@@ -303,6 +379,31 @@ class BackblazeB2StorageDriver(StorageDriver):
objects = self._to_objects(data=resp.object, container=None)
return objects
+ def ex_get_upload_data(self, container_id):
+ """
+ Retrieve information used for uploading files (upload url, auth token,
+ etc).
+
+ :rype: ``dict``
+ """
+ # TODO: This is static (AFAIK) so it could be cached
+ params = {}
+ params['bucketId'] = container_id
+ response = self.connection.request(action='b2_get_upload_url',
+ method='GET',
+ params=params)
+ return response.object
+
+ def ex_get_upload_url(self, container_id):
+ """
+ Retrieve URL used for file uploads.
+
+ :rtype: ``str``
+ """
+ result = self.ex_get_upload_data(container_id=container_id)
+ upload_url = result['uploadUrl']
+ return upload_url
+
def _to_containers(self, data):
result = []
for item in data['buckets']: