You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@libcloud.apache.org by an...@apache.org on 2016/11/14 23:50:52 UTC
[05/56] [abbrv] libcloud git commit: Removed sdist
http://git-wip-us.apache.org/repos/asf/libcloud/blob/8afcda91/apache-libcloud-1.0.0rc2/libcloud/storage/drivers/backblaze_b2.py
----------------------------------------------------------------------
diff --git a/apache-libcloud-1.0.0rc2/libcloud/storage/drivers/backblaze_b2.py b/apache-libcloud-1.0.0rc2/libcloud/storage/drivers/backblaze_b2.py
deleted file mode 100644
index fe2f335..0000000
--- a/apache-libcloud-1.0.0rc2/libcloud/storage/drivers/backblaze_b2.py
+++ /dev/null
@@ -1,525 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Driver for Backblaze B2 service.
-"""
-
-import base64
-import hashlib
-
-try:
- import simplejson as json
-except ImportError:
- import json
-
-from libcloud.utils.py3 import b
-from libcloud.utils.py3 import httplib
-from libcloud.utils.py3 import urlparse
-from libcloud.utils.py3 import next
-from libcloud.utils.files import read_in_chunks
-from libcloud.utils.files import exhaust_iterator
-from libcloud.utils.escape import sanitize_object_name
-
-from libcloud.common.base import ConnectionUserAndKey
-from libcloud.common.base import JsonResponse
-from libcloud.common.types import InvalidCredsError
-from libcloud.common.types import LibcloudError
-from libcloud.storage.providers import Provider
-from libcloud.storage.base import Object, Container, StorageDriver
-from libcloud.storage.types import ContainerDoesNotExistError
-from libcloud.storage.types import ObjectDoesNotExistError
-
-__all__ = [
- 'BackblazeB2StorageDriver',
-
- 'BackblazeB2Connection',
- 'BackblazeB2AuthConnection'
-]
-
-AUTH_API_HOST = 'api.backblaze.com'
-API_PATH = '/b2api/v1/'
-
-
-class BackblazeB2Response(JsonResponse):
- def success(self):
- return self.status in [httplib.OK, httplib.CREATED, httplib.ACCEPTED]
-
- def parse_error(self):
- status = int(self.status)
- body = self.parse_body()
-
- if status == httplib.UNAUTHORIZED:
- raise InvalidCredsError(body['message'])
-
- return self.body
-
-
-class BackblazeB2AuthConnection(ConnectionUserAndKey):
- host = AUTH_API_HOST
- secure = True
- responseCls = BackblazeB2Response
-
- def __init__(self, *args, **kwargs):
- super(BackblazeB2AuthConnection, self).__init__(*args, **kwargs)
-
- # Those attributes are populated after authentication
- self.account_id = None
- self.api_url = None
- self.api_host = None
- self.download_url = None
- self.download_host = None
- self.auth_token = None
-
- def authenticate(self, force=False):
- """
- :param force: Force authentication if if we have already obtained the
- token.
- :type force: ``bool``
- """
- if not self._is_authentication_needed(force=force):
- return self
-
- headers = {}
- action = 'b2_authorize_account'
- auth_b64 = base64.b64encode(b('%s:%s' % (self.user_id, self.key)))
- headers['Authorization'] = 'Basic %s' % (auth_b64.decode('utf-8'))
-
- action = API_PATH + 'b2_authorize_account'
- resp = self.request(action=action, headers=headers, method='GET')
-
- if resp.status == httplib.OK:
- self._parse_and_set_auth_info(data=resp.object)
- else:
- raise Exception('Failed to authenticate: %s' % (str(resp.object)))
-
- return self
-
- def _parse_and_set_auth_info(self, data):
- result = {}
- self.account_id = data['accountId']
- self.api_url = data['apiUrl']
- self.download_url = data['downloadUrl']
- self.auth_token = data['authorizationToken']
-
- parsed_api_url = urlparse.urlparse(self.api_url)
- self.api_host = parsed_api_url.netloc
-
- parsed_download_url = urlparse.urlparse(self.download_url)
- self.download_host = parsed_download_url.netloc
-
- return result
-
- def _is_authentication_needed(self, force=False):
- if not self.auth_token or force:
- return True
-
- return False
-
-
-class BackblazeB2Connection(ConnectionUserAndKey):
- host = None # Note: host is set after authentication
- secure = True
- responseCls = BackblazeB2Response
- authCls = BackblazeB2AuthConnection
-
- def __init__(self, *args, **kwargs):
- super(BackblazeB2Connection, self).__init__(*args, **kwargs)
-
- # Stores info retrieved after authentication (auth token, api url,
- # dowload url).
- self._auth_conn = self.authCls(*args, **kwargs)
-
- def download_request(self, action, params=None):
- # Lazily perform authentication
- auth_conn = self._auth_conn.authenticate()
-
- # Set host to the download server
- self.host = auth_conn.download_host
-
- action = '/file/' + action
- method = 'GET'
- raw = True
- response = self._request(auth_conn=auth_conn, action=action,
- params=params, method=method,
- raw=raw)
- return response
-
- def upload_request(self, action, headers, upload_host, auth_token, data):
- # Lazily perform authentication
- auth_conn = self._auth_conn.authenticate()
-
- # Upload host is dynamically retrieved for each upload request
- self.host = upload_host
-
- method = 'POST'
- raw = False
- response = self._request(auth_conn=auth_conn, action=action,
- params=None, data=data,
- headers=headers, method=method,
- raw=raw, auth_token=auth_token)
- return response
-
- def request(self, action, params=None, data=None, headers=None,
- method='GET', raw=False, include_account_id=False):
- params = params or {}
- headers = headers or {}
-
- # Lazily perform authentication
- auth_conn = self._auth_conn.authenticate()
-
- # Set host
- self.host = auth_conn.api_host
-
- # Include Content-Type
- if not raw and data:
- headers['Content-Type'] = 'application/json'
-
- # Include account id
- if include_account_id:
- if method == 'GET':
- params['accountId'] = auth_conn.account_id
- elif method == 'POST':
- data = data or {}
- data['accountId'] = auth_conn.account_id
-
- action = API_PATH + action
- if data:
- data = json.dumps(data)
-
- response = self._request(auth_conn=self._auth_conn, action=action,
- params=params, data=data,
- method=method, headers=headers, raw=raw)
- return response
-
- def _request(self, auth_conn, action, params=None, data=None, headers=None,
- method='GET', raw=False, auth_token=None):
- params = params or {}
- headers = headers or {}
-
- if not auth_token:
- # If auth token is not explicitly provided, use the default one
- auth_token = self._auth_conn.auth_token
-
- # Include auth token
- headers['Authorization'] = '%s' % (auth_token)
- response = super(BackblazeB2Connection, self).request(action=action,
- params=params,
- data=data,
- method=method,
- headers=headers,
- raw=raw)
- return response
-
-
-class BackblazeB2StorageDriver(StorageDriver):
- connectionCls = BackblazeB2Connection
- name = 'Backblaze B2'
- website = 'https://www.backblaze.com/b2/'
- type = Provider.BACKBLAZE_B2
- hash_type = 'sha1'
- supports_chunked_encoding = False
-
- def iterate_containers(self):
- resp = self.connection.request(action='b2_list_buckets',
- method='GET',
- include_account_id=True)
- containers = self._to_containers(data=resp.object)
- return containers
-
- def iterate_container_objects(self, container):
- # TODO: Support pagination
- params = {'bucketId': container.extra['id']}
- resp = self.connection.request(action='b2_list_file_names',
- method='GET',
- params=params)
- objects = self._to_objects(data=resp.object, container=container)
- return objects
-
- def get_container(self, container_name):
- containers = self.iterate_containers()
- container = next((c for c in containers if c.name == container_name),
- None)
- if container:
- return container
- else:
- raise ContainerDoesNotExistError(value=None, driver=self,
- container_name=container_name)
-
- def get_object(self, container_name, object_name):
- container = self.get_container(container_name=container_name)
- objects = self.iterate_container_objects(container=container)
-
- obj = next((obj for obj in objects if obj.name == object_name), None)
-
- if obj is not None:
- return obj
- else:
- raise ObjectDoesNotExistError(value=None, driver=self,
- object_name=object_name)
-
- def create_container(self, container_name, ex_type='allPrivate'):
- data = {}
- data['bucketName'] = container_name
- data['bucketType'] = ex_type
- resp = self.connection.request(action='b2_create_bucket',
- data=data, method='POST',
- include_account_id=True)
- container = self._to_container(item=resp.object)
- return container
-
- def delete_container(self, container):
- data = {}
- data['bucketId'] = container.extra['id']
- resp = self.connection.request(action='b2_delete_bucket',
- data=data, method='POST',
- include_account_id=True)
- return resp.status == httplib.OK
-
- def download_object(self, obj, destination_path, overwrite_existing=False,
- delete_on_failure=True):
- action = self._get_object_download_path(container=obj.container,
- obj=obj)
- response = self.connection.download_request(action=action)
-
- # TODO: Include metadata from response headers
- return self._get_object(obj=obj, callback=self._save_object,
- response=response,
- callback_kwargs={
- 'obj': obj,
- 'response': response.response,
- 'destination_path': destination_path,
- 'overwrite_existing': overwrite_existing,
- 'delete_on_failure': delete_on_failure
- },
- success_status_code=httplib.OK)
-
- def download_object_as_stream(self, obj, chunk_size=None):
- action = self._get_object_download_path(container=obj.container,
- obj=obj)
- response = self.connection.download_request(action=action)
-
- return self._get_object(obj=obj, callback=read_in_chunks,
- response=response,
- callback_kwargs={'iterator': response.response,
- 'chunk_size': chunk_size},
- success_status_code=httplib.OK)
-
- def upload_object(self, file_path, container, object_name, extra=None,
- verify_hash=True, headers=None):
- """
- Upload an object.
-
- Note: This will override file with a same name if it already exists.
- """
- # Note: We don't use any of the base driver functions since Backblaze
- # API requires you to provide SHA1 has upfront and the base methods
- # don't support that
-
- with open(file_path, 'rb') as fp:
- iterator = iter(fp)
- iterator = read_in_chunks(iterator=iterator)
- data = exhaust_iterator(iterator=iterator)
-
- obj = self._perform_upload(data=data, container=container,
- object_name=object_name,
- extra=extra,
- verify_hash=verify_hash,
- headers=headers)
-
- return obj
-
- def upload_object_via_stream(self, iterator, container, object_name,
- extra=None, headers=None):
- """
- Upload an object.
-
- Note: Backblaze does not yet support uploading via stream,
- so this calls upload_object internally requiring the object data
- to be loaded into memory at once
- """
-
- iterator = read_in_chunks(iterator=iterator)
- data = exhaust_iterator(iterator=iterator)
-
- obj = self._perform_upload(data=data, container=container,
- object_name=object_name,
- extra=extra,
- headers=headers)
-
- return obj
-
- def delete_object(self, obj):
- data = {}
- data['fileName'] = obj.name
- data['fileId'] = obj.extra['fileId']
- resp = self.connection.request(action='b2_delete_file_version',
- data=data, method='POST')
- return resp.status == httplib.OK
-
- def ex_get_object(self, object_id):
- params = {}
- params['fileId'] = object_id
- resp = self.connection.request(action='b2_get_file_info',
- method='GET',
- params=params)
- obj = self._to_object(item=resp.object, container=None)
- return obj
-
- def ex_hide_object(self, container_id, object_name):
- data = {}
- data['bucketId'] = container_id
- data['fileName'] = object_name
- resp = self.connection.request(action='b2_hide_file',
- data=data, method='POST')
- obj = self._to_object(item=resp.object, container=None)
- return obj
-
- def ex_list_object_versions(self, container_id, ex_start_file_name=None,
- ex_start_file_id=None, ex_max_file_count=None):
- params = {}
- params['bucketId'] = container_id
-
- if ex_start_file_name:
- params['startFileName'] = ex_start_file_name
-
- if ex_start_file_id:
- params['startFileId'] = ex_start_file_id
-
- if ex_max_file_count:
- params['maxFileCount'] = ex_max_file_count
-
- resp = self.connection.request(action='b2_list_file_versions',
- params=params, method='GET')
- objects = self._to_objects(data=resp.object, container=None)
- return objects
-
- def ex_get_upload_data(self, container_id):
- """
- Retrieve information used for uploading files (upload url, auth token,
- etc).
-
- :rype: ``dict``
- """
- # TODO: This is static (AFAIK) so it could be cached
- params = {}
- params['bucketId'] = container_id
- response = self.connection.request(action='b2_get_upload_url',
- method='GET',
- params=params)
- return response.object
-
- def ex_get_upload_url(self, container_id):
- """
- Retrieve URL used for file uploads.
-
- :rtype: ``str``
- """
- result = self.ex_get_upload_data(container_id=container_id)
- upload_url = result['uploadUrl']
- return upload_url
-
- def _to_containers(self, data):
- result = []
- for item in data['buckets']:
- container = self._to_container(item=item)
- result.append(container)
-
- return result
-
- def _to_container(self, item):
- extra = {}
- extra['id'] = item['bucketId']
- extra['bucketType'] = item['bucketType']
- container = Container(name=item['bucketName'], extra=extra,
- driver=self)
- return container
-
- def _to_objects(self, data, container):
- result = []
- for item in data['files']:
- obj = self._to_object(item=item, container=container)
- result.append(obj)
-
- return result
-
- def _to_object(self, item, container=None):
- extra = {}
- extra['fileId'] = item['fileId']
- extra['uploadTimestamp'] = item.get('uploadTimestamp', None)
- size = item.get('size', item.get('contentLength', None))
- hash = item.get('contentSha1', None)
- meta_data = item.get('fileInfo', {})
- obj = Object(name=item['fileName'], size=size, hash=hash, extra=extra,
- meta_data=meta_data, container=container, driver=self)
- return obj
-
- def _get_object_download_path(self, container, obj):
- """
- Return a path used in the download requests.
-
- :rtype: ``str``
- """
- path = container.name + '/' + obj.name
- return path
-
- def _perform_upload(self, data, container, object_name, extra=None,
- verify_hash=True, headers=None):
-
- if isinstance(data, str):
- data = bytearray(data)
-
- object_name = sanitize_object_name(object_name)
-
- extra = extra or {}
- content_type = extra.get('content_type', 'b2/x-auto')
- meta_data = extra.get('meta_data', {})
-
- # Note: Backblaze API doesn't support chunked encoding and we need to
- # provide Content-Length up front (this is one inside _upload_object):/
- headers = headers or {}
- headers['X-Bz-File-Name'] = object_name
- headers['Content-Type'] = content_type
-
- sha1 = hashlib.sha1()
- sha1.update(b(data))
- headers['X-Bz-Content-Sha1'] = sha1.hexdigest()
-
- # Include optional meta-data (up to 10 items)
- for key, value in meta_data:
- # TODO: Encode / escape key
- headers['X-Bz-Info-%s' % (key)] = value
-
- upload_data = self.ex_get_upload_data(
- container_id=container.extra['id'])
- upload_token = upload_data['authorizationToken']
- parsed_url = urlparse.urlparse(upload_data['uploadUrl'])
-
- upload_host = parsed_url.netloc
- request_path = parsed_url.path
-
- response = self.connection.upload_request(action=request_path,
- headers=headers,
- upload_host=upload_host,
- auth_token=upload_token,
- data=data)
-
- if response.status == httplib.OK:
- obj = self._to_object(item=response.object, container=container)
- return obj
- else:
- body = response.response.read()
- raise LibcloudError('Upload failed. status_code=%s, body=%s' %
- (response.status, body), driver=self)
http://git-wip-us.apache.org/repos/asf/libcloud/blob/8afcda91/apache-libcloud-1.0.0rc2/libcloud/storage/drivers/cloudfiles.py
----------------------------------------------------------------------
diff --git a/apache-libcloud-1.0.0rc2/libcloud/storage/drivers/cloudfiles.py b/apache-libcloud-1.0.0rc2/libcloud/storage/drivers/cloudfiles.py
deleted file mode 100644
index 2502d42..0000000
--- a/apache-libcloud-1.0.0rc2/libcloud/storage/drivers/cloudfiles.py
+++ /dev/null
@@ -1,972 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from hashlib import sha1
-import hmac
-import os
-from time import time
-
-from libcloud.utils.py3 import httplib
-from libcloud.utils.py3 import urlencode
-
-try:
- import simplejson as json
-except ImportError:
- import json
-
-from libcloud.utils.py3 import PY3
-from libcloud.utils.py3 import b
-from libcloud.utils.py3 import urlquote
-
-if PY3:
- from io import FileIO as file
-
-from libcloud.utils.files import read_in_chunks
-from libcloud.common.types import MalformedResponseError, LibcloudError
-from libcloud.common.base import Response, RawResponse
-
-from libcloud.storage.providers import Provider
-from libcloud.storage.base import Object, Container, StorageDriver
-from libcloud.storage.types import ContainerAlreadyExistsError
-from libcloud.storage.types import ContainerDoesNotExistError
-from libcloud.storage.types import ContainerIsNotEmptyError
-from libcloud.storage.types import ObjectDoesNotExistError
-from libcloud.storage.types import ObjectHashMismatchError
-from libcloud.storage.types import InvalidContainerNameError
-from libcloud.common.openstack import OpenStackBaseConnection
-from libcloud.common.openstack import OpenStackDriverMixin
-
-from libcloud.common.rackspace import AUTH_URL
-
-CDN_HOST = 'cdn.clouddrive.com'
-API_VERSION = 'v1.0'
-
-# Keys which are used to select a correct endpoint from the service catalog.
-INTERNAL_ENDPOINT_KEY = 'internalURL'
-PUBLIC_ENDPOINT_KEY = 'publicURL'
-
-
-class CloudFilesResponse(Response):
- valid_response_codes = [httplib.NOT_FOUND, httplib.CONFLICT]
-
- def success(self):
- i = int(self.status)
- return i >= 200 and i <= 299 or i in self.valid_response_codes
-
- def parse_body(self):
- if not self.body:
- return None
-
- if 'content-type' in self.headers:
- key = 'content-type'
- elif 'Content-Type' in self.headers:
- key = 'Content-Type'
- else:
- raise LibcloudError('Missing content-type header')
-
- content_type = self.headers[key]
- if content_type.find(';') != -1:
- content_type = content_type.split(';')[0]
-
- if content_type == 'application/json':
- try:
- data = json.loads(self.body)
- except:
- raise MalformedResponseError('Failed to parse JSON',
- body=self.body,
- driver=CloudFilesStorageDriver)
- elif content_type == 'text/plain':
- data = self.body
- else:
- data = self.body
-
- return data
-
-
-class CloudFilesRawResponse(CloudFilesResponse, RawResponse):
- pass
-
-
-class OpenStackSwiftConnection(OpenStackBaseConnection):
- """
- Connection class for the OpenStack Swift endpoint.
- """
-
- responseCls = CloudFilesResponse
- rawResponseCls = CloudFilesRawResponse
-
- auth_url = AUTH_URL
- _auth_version = '1.0'
-
- # TODO: Reverse the relationship - Swift -> CloudFiles
- def __init__(self, user_id, key, secure=True, **kwargs):
- # Ignore this for now
- kwargs.pop('use_internal_url', None)
- super(OpenStackSwiftConnection, self).__init__(user_id, key,
- secure=secure,
- **kwargs)
- self.api_version = API_VERSION
- self.accept_format = 'application/json'
-
- self._service_type = self._ex_force_service_type or 'object-store'
- self._service_name = self._ex_force_service_name or 'swift'
-
- if self._ex_force_service_region:
- self._service_region = self._ex_force_service_region
- else:
- self._service_region = None
-
- def get_endpoint(self, *args, **kwargs):
- if '2.0' in self._auth_version:
- endpoint = self.service_catalog.get_endpoint(
- service_type=self._service_type,
- name=self._service_name,
- region=self._service_region)
- elif ('1.1' in self._auth_version) or ('1.0' in self._auth_version):
- endpoint = self.service_catalog.get_endpoint(
- name=self._service_name, region=self._service_region)
-
- if endpoint:
- return endpoint.url
- else:
- raise LibcloudError('Could not find specified endpoint')
-
- def request(self, action, params=None, data='', headers=None, method='GET',
- raw=False, cdn_request=False):
- if not headers:
- headers = {}
- if not params:
- params = {}
-
- self.cdn_request = cdn_request
- params['format'] = 'json'
-
- if method in ['POST', 'PUT'] and 'Content-Type' not in headers:
- headers.update({'Content-Type': 'application/json; charset=UTF-8'})
-
- return super(OpenStackSwiftConnection, self).request(
- action=action,
- params=params, data=data,
- method=method, headers=headers,
- raw=raw)
-
-
-class CloudFilesConnection(OpenStackSwiftConnection):
- """
- Base connection class for the Cloudfiles driver.
- """
-
- responseCls = CloudFilesResponse
- rawResponseCls = CloudFilesRawResponse
-
- auth_url = AUTH_URL
- _auth_version = '2.0'
-
- def __init__(self, user_id, key, secure=True,
- use_internal_url=False, **kwargs):
- super(CloudFilesConnection, self).__init__(user_id, key, secure=secure,
- **kwargs)
- self.api_version = API_VERSION
- self.accept_format = 'application/json'
- self.cdn_request = False
- self.use_internal_url = use_internal_url
-
- def get_endpoint(self):
- region = self._ex_force_service_region.upper()
-
- if self.use_internal_url:
- endpoint_type = 'internal'
- else:
- endpoint_type = 'external'
-
- if '2.0' in self._auth_version:
- ep = self.service_catalog.get_endpoint(
- service_type='object-store',
- name='cloudFiles',
- region=region,
- endpoint_type=endpoint_type)
- cdn_ep = self.service_catalog.get_endpoint(
- service_type='rax:object-cdn',
- name='cloudFilesCDN',
- region=region,
- endpoint_type=endpoint_type)
- else:
- raise LibcloudError(
- 'Auth version "%s" not supported' % (self._auth_version))
-
- # if this is a CDN request, return the cdn url instead
- if self.cdn_request:
- ep = cdn_ep
-
- if not ep or not ep.url:
- raise LibcloudError('Could not find specified endpoint')
-
- return ep.url
-
- def request(self, action, params=None, data='', headers=None, method='GET',
- raw=False, cdn_request=False):
- if not headers:
- headers = {}
- if not params:
- params = {}
-
- self.cdn_request = cdn_request
- params['format'] = 'json'
-
- if method in ['POST', 'PUT'] and 'Content-Type' not in headers:
- headers.update({'Content-Type': 'application/json; charset=UTF-8'})
-
- return super(CloudFilesConnection, self).request(
- action=action,
- params=params, data=data,
- method=method, headers=headers,
- raw=raw, cdn_request=cdn_request)
-
-
-class CloudFilesStorageDriver(StorageDriver, OpenStackDriverMixin):
- """
- CloudFiles driver.
- """
- name = 'CloudFiles'
- website = 'http://www.rackspace.com/'
-
- connectionCls = CloudFilesConnection
- hash_type = 'md5'
- supports_chunked_encoding = True
-
- def __init__(self, key, secret=None, secure=True, host=None, port=None,
- region='ord', use_internal_url=False, **kwargs):
- """
- @inherits: :class:`StorageDriver.__init__`
-
- :param region: ID of the region which should be used.
- :type region: ``str``
- """
- # This is here for backard compatibility
- if 'ex_force_service_region' in kwargs:
- region = kwargs['ex_force_service_region']
-
- self.use_internal_url = use_internal_url
- OpenStackDriverMixin.__init__(self, (), **kwargs)
- super(CloudFilesStorageDriver, self).__init__(key=key, secret=secret,
- secure=secure, host=host,
- port=port, region=region,
- **kwargs)
-
- @classmethod
- def list_regions(cls):
- return ['ord', 'dfw', 'iad', 'lon', 'hkg', 'syd']
-
- def iterate_containers(self):
- response = self.connection.request('')
-
- if response.status == httplib.NO_CONTENT:
- return []
- elif response.status == httplib.OK:
- return self._to_container_list(json.loads(response.body))
-
- raise LibcloudError('Unexpected status code: %s' % (response.status))
-
- def get_container(self, container_name):
- container_name_encoded = self._encode_container_name(container_name)
- response = self.connection.request('/%s' % (container_name_encoded),
- method='HEAD')
-
- if response.status == httplib.NO_CONTENT:
- container = self._headers_to_container(
- container_name, response.headers)
- return container
- elif response.status == httplib.NOT_FOUND:
- raise ContainerDoesNotExistError(None, self, container_name)
-
- raise LibcloudError('Unexpected status code: %s' % (response.status))
-
- def get_object(self, container_name, object_name):
- container = self.get_container(container_name)
- container_name_encoded = self._encode_container_name(container_name)
- object_name_encoded = self._encode_object_name(object_name)
-
- response = self.connection.request('/%s/%s' % (container_name_encoded,
- object_name_encoded),
- method='HEAD')
- if response.status in [httplib.OK, httplib.NO_CONTENT]:
- obj = self._headers_to_object(
- object_name, container, response.headers)
- return obj
- elif response.status == httplib.NOT_FOUND:
- raise ObjectDoesNotExistError(None, self, object_name)
-
- raise LibcloudError('Unexpected status code: %s' % (response.status))
-
- def get_container_cdn_url(self, container):
- # pylint: disable=unexpected-keyword-arg
- container_name_encoded = self._encode_container_name(container.name)
- response = self.connection.request('/%s' % (container_name_encoded),
- method='HEAD',
- cdn_request=True)
-
- if response.status == httplib.NO_CONTENT:
- cdn_url = response.headers['x-cdn-uri']
- return cdn_url
- elif response.status == httplib.NOT_FOUND:
- raise ContainerDoesNotExistError(value='',
- container_name=container.name,
- driver=self)
-
- raise LibcloudError('Unexpected status code: %s' % (response.status))
-
- def get_object_cdn_url(self, obj):
- container_cdn_url = self.get_container_cdn_url(container=obj.container)
- return '%s/%s' % (container_cdn_url, obj.name)
-
- def enable_container_cdn(self, container, ex_ttl=None):
- """
- @inherits: :class:`StorageDriver.enable_container_cdn`
-
- :param ex_ttl: cache time to live
- :type ex_ttl: ``int``
- """
- container_name = self._encode_container_name(container.name)
- headers = {'X-CDN-Enabled': 'True'}
-
- if ex_ttl:
- headers['X-TTL'] = ex_ttl
-
- # pylint: disable=unexpected-keyword-arg
- response = self.connection.request('/%s' % (container_name),
- method='PUT',
- headers=headers,
- cdn_request=True)
-
- return response.status in [httplib.CREATED, httplib.ACCEPTED]
-
- def create_container(self, container_name):
- container_name_encoded = self._encode_container_name(container_name)
- response = self.connection.request(
- '/%s' % (container_name_encoded), method='PUT')
-
- if response.status == httplib.CREATED:
- # Accepted mean that container is not yet created but it will be
- # eventually
- extra = {'object_count': 0}
- container = Container(name=container_name,
- extra=extra, driver=self)
-
- return container
- elif response.status == httplib.ACCEPTED:
- error = ContainerAlreadyExistsError(None, self, container_name)
- raise error
-
- raise LibcloudError('Unexpected status code: %s' % (response.status))
-
- def delete_container(self, container):
- name = self._encode_container_name(container.name)
-
- # Only empty container can be deleted
- response = self.connection.request('/%s' % (name), method='DELETE')
-
- if response.status == httplib.NO_CONTENT:
- return True
- elif response.status == httplib.NOT_FOUND:
- raise ContainerDoesNotExistError(value='',
- container_name=name, driver=self)
- elif response.status == httplib.CONFLICT:
- # @TODO: Add "delete_all_objects" parameter?
- raise ContainerIsNotEmptyError(value='',
- container_name=name, driver=self)
-
- def download_object(self, obj, destination_path, overwrite_existing=False,
- delete_on_failure=True):
- container_name = obj.container.name
- object_name = obj.name
- response = self.connection.request('/%s/%s' % (container_name,
- object_name),
- method='GET', raw=True)
-
- return self._get_object(
- obj=obj, callback=self._save_object, response=response,
- callback_kwargs={'obj': obj,
- 'response': response.response,
- 'destination_path': destination_path,
- 'overwrite_existing': overwrite_existing,
- 'delete_on_failure': delete_on_failure},
- success_status_code=httplib.OK)
-
- def download_object_as_stream(self, obj, chunk_size=None):
- container_name = obj.container.name
- object_name = obj.name
- response = self.connection.request('/%s/%s' % (container_name,
- object_name),
- method='GET', raw=True)
-
- return self._get_object(obj=obj, callback=read_in_chunks,
- response=response,
- callback_kwargs={'iterator': response.response,
- 'chunk_size': chunk_size},
- success_status_code=httplib.OK)
-
- def upload_object(self, file_path, container, object_name, extra=None,
- verify_hash=True, headers=None):
- """
- Upload an object.
-
- Note: This will override file with a same name if it already exists.
- """
- upload_func = self._upload_file
- upload_func_kwargs = {'file_path': file_path}
-
- return self._put_object(container=container, object_name=object_name,
- upload_func=upload_func,
- upload_func_kwargs=upload_func_kwargs,
- extra=extra, file_path=file_path,
- verify_hash=verify_hash, headers=headers)
-
- def upload_object_via_stream(self, iterator,
- container, object_name, extra=None,
- headers=None):
- if isinstance(iterator, file):
- iterator = iter(iterator)
-
- upload_func = self._stream_data
- upload_func_kwargs = {'iterator': iterator}
-
- return self._put_object(container=container, object_name=object_name,
- upload_func=upload_func,
- upload_func_kwargs=upload_func_kwargs,
- extra=extra, iterator=iterator,
- headers=headers)
-
- def delete_object(self, obj):
- container_name = self._encode_container_name(obj.container.name)
- object_name = self._encode_object_name(obj.name)
-
- response = self.connection.request(
- '/%s/%s' % (container_name, object_name), method='DELETE')
-
- if response.status == httplib.NO_CONTENT:
- return True
- elif response.status == httplib.NOT_FOUND:
- raise ObjectDoesNotExistError(value='', object_name=object_name,
- driver=self)
-
- raise LibcloudError('Unexpected status code: %s' % (response.status))
-
- def ex_purge_object_from_cdn(self, obj, email=None):
- """
- Purge edge cache for the specified object.
-
- :param email: Email where a notification will be sent when the job
- completes. (optional)
- :type email: ``str``
- """
- container_name = self._encode_container_name(obj.container.name)
- object_name = self._encode_object_name(obj.name)
- headers = {'X-Purge-Email': email} if email else {}
-
- # pylint: disable=unexpected-keyword-arg
- response = self.connection.request('/%s/%s' % (container_name,
- object_name),
- method='DELETE',
- headers=headers,
- cdn_request=True)
-
- return response.status == httplib.NO_CONTENT
-
- def ex_get_meta_data(self):
- """
- Get meta data
-
- :rtype: ``dict``
- """
- response = self.connection.request('', method='HEAD')
-
- if response.status == httplib.NO_CONTENT:
- container_count = response.headers.get(
- 'x-account-container-count', 'unknown')
- object_count = response.headers.get(
- 'x-account-object-count', 'unknown')
- bytes_used = response.headers.get(
- 'x-account-bytes-used', 'unknown')
- temp_url_key = response.headers.get(
- 'x-account-meta-temp-url-key', None)
-
- return {'container_count': int(container_count),
- 'object_count': int(object_count),
- 'bytes_used': int(bytes_used),
- 'temp_url_key': temp_url_key}
-
- raise LibcloudError('Unexpected status code: %s' % (response.status))
-
- def ex_multipart_upload_object(self, file_path, container, object_name,
- chunk_size=33554432, extra=None,
- verify_hash=True):
- object_size = os.path.getsize(file_path)
- if object_size < chunk_size:
- return self.upload_object(file_path, container, object_name,
- extra=extra, verify_hash=verify_hash)
-
- iter_chunk_reader = FileChunkReader(file_path, chunk_size)
-
- for index, iterator in enumerate(iter_chunk_reader):
- self._upload_object_part(container=container,
- object_name=object_name,
- part_number=index,
- iterator=iterator,
- verify_hash=verify_hash)
-
- return self._upload_object_manifest(container=container,
- object_name=object_name,
- extra=extra,
- verify_hash=verify_hash)
-
- def ex_enable_static_website(self, container, index_file='index.html'):
- """
- Enable serving a static website.
-
- :param container: Container instance
- :type container: :class:`Container`
-
- :param index_file: Name of the object which becomes an index page for
- every sub-directory in this container.
- :type index_file: ``str``
-
- :rtype: ``bool``
- """
- container_name = container.name
- headers = {'X-Container-Meta-Web-Index': index_file}
-
- # pylint: disable=unexpected-keyword-arg
- response = self.connection.request('/%s' % (container_name),
- method='POST',
- headers=headers,
- cdn_request=False)
-
- return response.status in [httplib.CREATED, httplib.ACCEPTED]
-
- def ex_set_error_page(self, container, file_name='error.html'):
- """
- Set a custom error page which is displayed if file is not found and
- serving of a static website is enabled.
-
- :param container: Container instance
- :type container: :class:`Container`
-
- :param file_name: Name of the object which becomes the error page.
- :type file_name: ``str``
-
- :rtype: ``bool``
- """
- container_name = container.name
- headers = {'X-Container-Meta-Web-Error': file_name}
-
- # pylint: disable=unexpected-keyword-arg
- response = self.connection.request('/%s' % (container_name),
- method='POST',
- headers=headers,
- cdn_request=False)
-
- return response.status in [httplib.CREATED, httplib.ACCEPTED]
-
- def ex_set_account_metadata_temp_url_key(self, key):
- """
- Set the metadata header X-Account-Meta-Temp-URL-Key on your Cloud
- Files account.
-
- :param key: X-Account-Meta-Temp-URL-Key
- :type key: ``str``
-
- :rtype: ``bool``
- """
- headers = {'X-Account-Meta-Temp-URL-Key': key}
-
- # pylint: disable=unexpected-keyword-arg
- response = self.connection.request('',
- method='POST',
- headers=headers,
- cdn_request=False)
-
- return response.status in [httplib.OK, httplib.NO_CONTENT,
- httplib.CREATED, httplib.ACCEPTED]
-
- def ex_get_object_temp_url(self, obj, method='GET', timeout=60):
- """
- Create a temporary URL to allow others to retrieve or put objects
- in your Cloud Files account for as long or as short a time as you
- wish. This method is specifically for allowing users to retrieve
- or update an object.
-
- :param obj: The object that you wish to make temporarily public
- :type obj: :class:`Object`
-
- :param method: Which method you would like to allow, 'PUT' or 'GET'
- :type method: ``str``
-
- :param timeout: Time (in seconds) after which you want the TempURL
- to expire.
- :type timeout: ``int``
-
- :rtype: ``bool``
- """
- # pylint: disable=no-member
- self.connection._populate_hosts_and_request_paths()
- expires = int(time() + timeout)
- path = '%s/%s/%s' % (self.connection.request_path,
- obj.container.name, obj.name)
- try:
- key = self.ex_get_meta_data()['temp_url_key']
- assert key is not None
- except Exception:
- raise KeyError('You must first set the ' +
- 'X-Account-Meta-Temp-URL-Key header on your ' +
- 'Cloud Files account using ' +
- 'ex_set_account_metadata_temp_url_key before ' +
- 'you can use this method.')
- hmac_body = '%s\n%s\n%s' % (method, expires, path)
- sig = hmac.new(b(key), b(hmac_body), sha1).hexdigest()
- params = urlencode({'temp_url_sig': sig,
- 'temp_url_expires': expires})
-
- temp_url = 'https://%s/%s/%s?%s' %\
- (self.connection.host + self.connection.request_path,
- obj.container.name, obj.name, params)
-
- return temp_url
-
- def _upload_object_part(self, container, object_name, part_number,
- iterator, verify_hash=True):
- upload_func = self._stream_data
- upload_func_kwargs = {'iterator': iterator}
- part_name = object_name + '/%08d' % part_number
- extra = {'content_type': 'application/octet-stream'}
-
- self._put_object(container=container,
- object_name=part_name,
- upload_func=upload_func,
- upload_func_kwargs=upload_func_kwargs,
- extra=extra, iterator=iterator,
- verify_hash=verify_hash)
-
- def _upload_object_manifest(self, container, object_name, extra=None,
- verify_hash=True):
- extra = extra or {}
- meta_data = extra.get('meta_data')
-
- container_name_encoded = self._encode_container_name(container.name)
- object_name_encoded = self._encode_object_name(object_name)
- request_path = '/%s/%s' % (container_name_encoded, object_name_encoded)
-
- # pylint: disable=no-member
- headers = {'X-Auth-Token': self.connection.auth_token,
- 'X-Object-Manifest': '%s/%s/' %
- (container_name_encoded,
- object_name_encoded)}
-
- data = ''
- response = self.connection.request(request_path,
- method='PUT', data=data,
- headers=headers, raw=True)
-
- object_hash = None
-
- if verify_hash:
- hash_function = self._get_hash_function()
- hash_function.update(b(data))
- data_hash = hash_function.hexdigest()
- object_hash = response.headers.get('etag')
-
- if object_hash != data_hash:
- raise ObjectHashMismatchError(
- value=('MD5 hash checksum does not match (expected=%s, ' +
- 'actual=%s)') %
- (data_hash, object_hash),
- object_name=object_name, driver=self)
-
- obj = Object(name=object_name, size=0, hash=object_hash, extra=None,
- meta_data=meta_data, container=container, driver=self)
-
- return obj
-
- def list_container_objects(self, container, ex_prefix=None):
- """
- Return a list of objects for the given container.
-
- :param container: Container instance.
- :type container: :class:`Container`
-
- :param ex_prefix: Only get objects with names starting with ex_prefix
- :type ex_prefix: ``str``
-
- :return: A list of Object instances.
- :rtype: ``list`` of :class:`Object`
- """
- return list(self.iterate_container_objects(container,
- ex_prefix=ex_prefix))
-
- def iterate_container_objects(self, container, ex_prefix=None):
- """
- Return a generator of objects for the given container.
-
- :param container: Container instance
- :type container: :class:`Container`
-
- :param ex_prefix: Only get objects with names starting with ex_prefix
- :type ex_prefix: ``str``
-
- :return: A generator of Object instances.
- :rtype: ``generator`` of :class:`Object`
- """
- params = {}
- if ex_prefix:
- params['prefix'] = ex_prefix
-
- while True:
- container_name_encoded = \
- self._encode_container_name(container.name)
- response = self.connection.request('/%s' %
- (container_name_encoded),
- params=params)
-
- if response.status == httplib.NO_CONTENT:
- # Empty or non-existent container
- break
- elif response.status == httplib.OK:
- objects = self._to_object_list(json.loads(response.body),
- container)
-
- if len(objects) == 0:
- break
-
- for obj in objects:
- yield obj
- params['marker'] = obj.name
-
- else:
- raise LibcloudError('Unexpected status code: %s' %
- (response.status))
-
- def _put_object(self, container, object_name, upload_func,
- upload_func_kwargs, extra=None, file_path=None,
- iterator=None, verify_hash=True, headers=None):
- extra = extra or {}
- container_name_encoded = self._encode_container_name(container.name)
- object_name_encoded = self._encode_object_name(object_name)
- content_type = extra.get('content_type', None)
- meta_data = extra.get('meta_data', None)
- content_disposition = extra.get('content_disposition', None)
-
- headers = headers or {}
- if meta_data:
- for key, value in list(meta_data.items()):
- key = 'X-Object-Meta-%s' % (key)
- headers[key] = value
-
- if content_disposition is not None:
- headers['Content-Disposition'] = content_disposition
-
- request_path = '/%s/%s' % (container_name_encoded, object_name_encoded)
- result_dict = self._upload_object(
- object_name=object_name, content_type=content_type,
- upload_func=upload_func, upload_func_kwargs=upload_func_kwargs,
- request_path=request_path, request_method='PUT',
- headers=headers, file_path=file_path, iterator=iterator)
-
- response = result_dict['response'].response
- bytes_transferred = result_dict['bytes_transferred']
- server_hash = result_dict['response'].headers.get('etag', None)
-
- if response.status == httplib.EXPECTATION_FAILED:
- raise LibcloudError(value='Missing content-type header',
- driver=self)
- elif verify_hash and not server_hash:
- raise LibcloudError(value='Server didn\'t return etag',
- driver=self)
- elif (verify_hash and result_dict['data_hash'] != server_hash):
- raise ObjectHashMismatchError(
- value=('MD5 hash checksum does not match (expected=%s, ' +
- 'actual=%s)') % (result_dict['data_hash'], server_hash),
- object_name=object_name, driver=self)
- elif response.status == httplib.CREATED:
- obj = Object(
- name=object_name, size=bytes_transferred, hash=server_hash,
- extra=None, meta_data=meta_data, container=container,
- driver=self)
-
- return obj
- else:
- # @TODO: Add test case for this condition (probably 411)
- raise LibcloudError('status_code=%s' % (response.status),
- driver=self)
-
- def _encode_container_name(self, name):
- """
- Encode container name so it can be used as part of the HTTP request.
- """
- if name.startswith('/'):
- name = name[1:]
- name = urlquote(name)
-
- if name.find('/') != -1:
- raise InvalidContainerNameError(value='Container name cannot'
- ' contain slashes',
- container_name=name, driver=self)
-
- if len(name) > 256:
- raise InvalidContainerNameError(
- value='Container name cannot be longer than 256 bytes',
- container_name=name, driver=self)
-
- return name
-
- def _encode_object_name(self, name):
- name = urlquote(name)
- return name
-
- def _to_container_list(self, response):
- # @TODO: Handle more than 10k containers - use "lazy list"?
- for container in response:
- extra = {'object_count': int(container['count']),
- 'size': int(container['bytes'])}
- yield Container(name=container['name'], extra=extra, driver=self)
-
- def _to_object_list(self, response, container):
- objects = []
-
- for obj in response:
- name = obj['name']
- size = int(obj['bytes'])
- hash = obj['hash']
- extra = {'content_type': obj['content_type'],
- 'last_modified': obj['last_modified']}
- objects.append(Object(
- name=name, size=size, hash=hash, extra=extra,
- meta_data=None, container=container, driver=self))
-
- return objects
-
- def _headers_to_container(self, name, headers):
- size = int(headers.get('x-container-bytes-used', 0))
- object_count = int(headers.get('x-container-object-count', 0))
-
- extra = {'object_count': object_count,
- 'size': size}
- container = Container(name=name, extra=extra, driver=self)
- return container
-
- def _headers_to_object(self, name, container, headers):
- size = int(headers.pop('content-length', 0))
- last_modified = headers.pop('last-modified', None)
- etag = headers.pop('etag', None)
- content_type = headers.pop('content-type', None)
-
- meta_data = {}
- for key, value in list(headers.items()):
- if key.find('x-object-meta-') != -1:
- key = key.replace('x-object-meta-', '')
- meta_data[key] = value
-
- extra = {'content_type': content_type, 'last_modified': last_modified}
-
- obj = Object(name=name, size=size, hash=etag, extra=extra,
- meta_data=meta_data, container=container, driver=self)
- return obj
-
- def _ex_connection_class_kwargs(self):
- kwargs = self.openstack_connection_kwargs()
- kwargs['ex_force_service_region'] = self.region
- kwargs['use_internal_url'] = self.use_internal_url
- return kwargs
-
-
-class OpenStackSwiftStorageDriver(CloudFilesStorageDriver):
- """
- Storage driver for the OpenStack Swift.
- """
- type = Provider.CLOUDFILES_SWIFT
- name = 'OpenStack Swift'
- connectionCls = OpenStackSwiftConnection
-
- # TODO: Reverse the relationship - Swift -> CloudFiles
-
- def __init__(self, key, secret=None, secure=True, host=None, port=None,
- region=None, **kwargs):
- super(OpenStackSwiftStorageDriver, self).__init__(key=key,
- secret=secret,
- secure=secure,
- host=host,
- port=port,
- region=region,
- **kwargs)
-
-
-class FileChunkReader(object):
- def __init__(self, file_path, chunk_size):
- self.file_path = file_path
- self.total = os.path.getsize(file_path)
- self.chunk_size = chunk_size
- self.bytes_read = 0
- self.stop_iteration = False
-
- def __iter__(self):
- return self
-
- def next(self):
- if self.stop_iteration:
- raise StopIteration
-
- start_block = self.bytes_read
- end_block = start_block + self.chunk_size
- if end_block >= self.total:
- end_block = self.total
- self.stop_iteration = True
- self.bytes_read += end_block - start_block
- return ChunkStreamReader(file_path=self.file_path,
- start_block=start_block,
- end_block=end_block,
- chunk_size=8192)
-
- def __next__(self):
- return self.next()
-
-
-class ChunkStreamReader(object):
- def __init__(self, file_path, start_block, end_block, chunk_size):
- self.fd = open(file_path, 'rb')
- self.fd.seek(start_block)
- self.start_block = start_block
- self.end_block = end_block
- self.chunk_size = chunk_size
- self.bytes_read = 0
- self.stop_iteration = False
-
- def __iter__(self):
- return self
-
- def next(self):
- if self.stop_iteration:
- self.fd.close()
- raise StopIteration
-
- block_size = self.chunk_size
- if self.bytes_read + block_size > \
- self.end_block - self.start_block:
- block_size = self.end_block - self.start_block - self.bytes_read
- self.stop_iteration = True
-
- block = self.fd.read(block_size)
- self.bytes_read += block_size
- return block
-
- def __next__(self):
- return self.next()
http://git-wip-us.apache.org/repos/asf/libcloud/blob/8afcda91/apache-libcloud-1.0.0rc2/libcloud/storage/drivers/dummy.py
----------------------------------------------------------------------
diff --git a/apache-libcloud-1.0.0rc2/libcloud/storage/drivers/dummy.py b/apache-libcloud-1.0.0rc2/libcloud/storage/drivers/dummy.py
deleted file mode 100644
index affd265..0000000
--- a/apache-libcloud-1.0.0rc2/libcloud/storage/drivers/dummy.py
+++ /dev/null
@@ -1,490 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os.path
-import random
-import hashlib
-
-from libcloud.utils.py3 import PY3
-from libcloud.utils.py3 import b
-
-if PY3:
- from io import FileIO as file
-
-from libcloud.common.types import LibcloudError
-
-from libcloud.storage.base import Object, Container, StorageDriver
-from libcloud.storage.types import ContainerAlreadyExistsError
-from libcloud.storage.types import ContainerDoesNotExistError
-from libcloud.storage.types import ContainerIsNotEmptyError
-from libcloud.storage.types import ObjectDoesNotExistError
-
-
-class DummyFileObject(file):
- def __init__(self, yield_count=5, chunk_len=10):
- self._yield_count = yield_count
- self._chunk_len = chunk_len
-
- def read(self, size):
- i = 0
-
- while i < self._yield_count:
- yield self._get_chunk(self._chunk_len)
- i += 1
-
- raise StopIteration
-
- def _get_chunk(self, chunk_len):
- chunk = [str(x) for x in random.randint(97, 120)]
- return chunk
-
- def __len__(self):
- return self._yield_count * self._chunk_len
-
-
-class DummyIterator(object):
- def __init__(self, data=None):
- self.hash = hashlib.md5()
- self._data = data or []
- self._current_item = 0
-
- def get_md5_hash(self):
- return self.hash.hexdigest()
-
- def next(self):
- if self._current_item == len(self._data):
- raise StopIteration
-
- value = self._data[self._current_item]
- self.hash.update(b(value))
- self._current_item += 1
- return value
-
- def __next__(self):
- return self.next()
-
-
-class DummyStorageDriver(StorageDriver):
- """
- Dummy Storage driver.
-
- >>> from libcloud.storage.drivers.dummy import DummyStorageDriver
- >>> driver = DummyStorageDriver('key', 'secret')
- >>> container = driver.create_container(container_name='test container')
- >>> container
- <Container: name=test container, provider=Dummy Storage Provider>
- >>> container.name
- 'test container'
- >>> container.extra['object_count']
- 0
- """
-
- name = 'Dummy Storage Provider'
- website = 'http://example.com'
-
- def __init__(self, api_key, api_secret):
- """
- :param api_key: API key or username to used (required)
- :type api_key: ``str``
- :param api_secret: Secret password to be used (required)
- :type api_secret: ``str``
- :rtype: ``None``
- """
- self._containers = {}
-
- def get_meta_data(self):
- """
- >>> driver = DummyStorageDriver('key', 'secret')
- >>> driver.get_meta_data()['object_count']
- 0
- >>> driver.get_meta_data()['container_count']
- 0
- >>> driver.get_meta_data()['bytes_used']
- 0
- >>> container_name = 'test container 1'
- >>> container = driver.create_container(container_name=container_name)
- >>> container_name = 'test container 2'
- >>> container = driver.create_container(container_name=container_name)
- >>> obj = container.upload_object_via_stream(
- ... object_name='test object', iterator=DummyFileObject(5, 10),
- ... extra={})
- >>> driver.get_meta_data()['object_count']
- 1
- >>> driver.get_meta_data()['container_count']
- 2
- >>> driver.get_meta_data()['bytes_used']
- 50
-
- :rtype: ``dict``
- """
-
- container_count = len(self._containers)
- object_count = sum([len(self._containers[container]['objects']) for
- container in self._containers])
-
- bytes_used = 0
- for container in self._containers:
- objects = self._containers[container]['objects']
- for _, obj in objects.items():
- bytes_used += obj.size
-
- return {'container_count': int(container_count),
- 'object_count': int(object_count),
- 'bytes_used': int(bytes_used)}
-
- def iterate_containers(self):
- """
- >>> driver = DummyStorageDriver('key', 'secret')
- >>> list(driver.iterate_containers())
- []
- >>> container_name = 'test container 1'
- >>> container = driver.create_container(container_name=container_name)
- >>> container
- <Container: name=test container 1, provider=Dummy Storage Provider>
- >>> container.name
- 'test container 1'
- >>> container_name = 'test container 2'
- >>> container = driver.create_container(container_name=container_name)
- >>> container
- <Container: name=test container 2, provider=Dummy Storage Provider>
- >>> container = driver.create_container(
- ... container_name='test container 2')
- ... #doctest: +IGNORE_EXCEPTION_DETAIL
- Traceback (most recent call last):
- ContainerAlreadyExistsError:
- >>> container_list=list(driver.iterate_containers())
- >>> sorted([c.name for c in container_list])
- ['test container 1', 'test container 2']
-
- @inherits: :class:`StorageDriver.iterate_containers`
- """
-
- for container in list(self._containers.values()):
- yield container['container']
-
- def list_container_objects(self, container):
- container = self.get_container(container.name)
-
- return container.objects
-
- def get_container(self, container_name):
- """
- >>> driver = DummyStorageDriver('key', 'secret')
- >>> driver.get_container('unknown') #doctest: +IGNORE_EXCEPTION_DETAIL
- Traceback (most recent call last):
- ContainerDoesNotExistError:
- >>> container_name = 'test container 1'
- >>> container = driver.create_container(container_name=container_name)
- >>> container
- <Container: name=test container 1, provider=Dummy Storage Provider>
- >>> container.name
- 'test container 1'
- >>> driver.get_container('test container 1')
- <Container: name=test container 1, provider=Dummy Storage Provider>
-
- @inherits: :class:`StorageDriver.get_container`
- """
-
- if container_name not in self._containers:
- raise ContainerDoesNotExistError(driver=self, value=None,
- container_name=container_name)
-
- return self._containers[container_name]['container']
-
- def get_container_cdn_url(self, container):
- """
- >>> driver = DummyStorageDriver('key', 'secret')
- >>> driver.get_container('unknown') #doctest: +IGNORE_EXCEPTION_DETAIL
- Traceback (most recent call last):
- ContainerDoesNotExistError:
- >>> container_name = 'test container 1'
- >>> container = driver.create_container(container_name=container_name)
- >>> container
- <Container: name=test container 1, provider=Dummy Storage Provider>
- >>> container.name
- 'test container 1'
- >>> container.get_cdn_url()
- 'http://www.test.com/container/test_container_1'
-
- @inherits: :class:`StorageDriver.get_container_cdn_url`
- """
-
- if container.name not in self._containers:
- raise ContainerDoesNotExistError(driver=self, value=None,
- container_name=container.name)
-
- return self._containers[container.name]['cdn_url']
-
- def get_object(self, container_name, object_name):
- """
- >>> driver = DummyStorageDriver('key', 'secret')
- >>> driver.get_object('unknown', 'unknown')
- ... #doctest: +IGNORE_EXCEPTION_DETAIL
- Traceback (most recent call last):
- ContainerDoesNotExistError:
- >>> container_name = 'test container 1'
- >>> container = driver.create_container(container_name=container_name)
- >>> container
- <Container: name=test container 1, provider=Dummy Storage Provider>
- >>> driver.get_object(
- ... 'test container 1', 'unknown') #doctest: +IGNORE_EXCEPTION_DETAIL
- Traceback (most recent call last):
- ObjectDoesNotExistError:
- >>> obj = container.upload_object_via_stream(object_name='test object',
- ... iterator=DummyFileObject(5, 10), extra={})
- >>> obj.name
- 'test object'
- >>> obj.size
- 50
-
- @inherits: :class:`StorageDriver.get_object`
- """
-
- self.get_container(container_name)
- container_objects = self._containers[container_name]['objects']
- if object_name not in container_objects:
- raise ObjectDoesNotExistError(object_name=object_name, value=None,
- driver=self)
-
- return container_objects[object_name]
-
- def get_object_cdn_url(self, obj):
- """
- >>> driver = DummyStorageDriver('key', 'secret')
- >>> container_name = 'test container 1'
- >>> container = driver.create_container(container_name=container_name)
- >>> container
- <Container: name=test container 1, provider=Dummy Storage Provider>
- >>> obj = container.upload_object_via_stream(
- ... object_name='test object 5',
- ... iterator=DummyFileObject(5, 10), extra={})
- >>> obj.name
- 'test object 5'
- >>> obj.get_cdn_url()
- 'http://www.test.com/object/test_object_5'
-
- @inherits: :class:`StorageDriver.get_object_cdn_url`
- """
-
- container_name = obj.container.name
- container_objects = self._containers[container_name]['objects']
- if obj.name not in container_objects:
- raise ObjectDoesNotExistError(object_name=obj.name, value=None,
- driver=self)
-
- return container_objects[obj.name].meta_data['cdn_url']
-
- def create_container(self, container_name):
- """
- >>> driver = DummyStorageDriver('key', 'secret')
- >>> container_name = 'test container 1'
- >>> container = driver.create_container(container_name=container_name)
- >>> container
- <Container: name=test container 1, provider=Dummy Storage Provider>
- >>> container = driver.create_container(
- ... container_name='test container 1')
- ... #doctest: +IGNORE_EXCEPTION_DETAIL
- Traceback (most recent call last):
- ContainerAlreadyExistsError:
-
- @inherits: :class:`StorageDriver.create_container`
- """
-
- if container_name in self._containers:
- raise ContainerAlreadyExistsError(container_name=container_name,
- value=None, driver=self)
-
- extra = {'object_count': 0}
- container = Container(name=container_name, extra=extra, driver=self)
-
- self._containers[container_name] = {'container': container,
- 'objects': {},
- 'cdn_url':
- 'http://www.test.com/container/%s'
- %
- (container_name.replace(' ', '_'))
- }
- return container
-
- def delete_container(self, container):
- """
- >>> driver = DummyStorageDriver('key', 'secret')
- >>> container = Container(name = 'test container',
- ... extra={'object_count': 0}, driver=driver)
- >>> driver.delete_container(container=container)
- ... #doctest: +IGNORE_EXCEPTION_DETAIL
- Traceback (most recent call last):
- ContainerDoesNotExistError:
- >>> container = driver.create_container(
- ... container_name='test container 1')
- ... #doctest: +IGNORE_EXCEPTION_DETAIL
- >>> len(driver._containers)
- 1
- >>> driver.delete_container(container=container)
- True
- >>> len(driver._containers)
- 0
- >>> container = driver.create_container(
- ... container_name='test container 1')
- ... #doctest: +IGNORE_EXCEPTION_DETAIL
- >>> obj = container.upload_object_via_stream(
- ... object_name='test object', iterator=DummyFileObject(5, 10),
- ... extra={})
- >>> driver.delete_container(container=container)
- ... #doctest: +IGNORE_EXCEPTION_DETAIL
- Traceback (most recent call last):
- ContainerIsNotEmptyError:
-
- @inherits: :class:`StorageDriver.delete_container`
- """
-
- container_name = container.name
- if container_name not in self._containers:
- raise ContainerDoesNotExistError(container_name=container_name,
- value=None, driver=self)
-
- container = self._containers[container_name]
- if len(container['objects']) > 0:
- raise ContainerIsNotEmptyError(container_name=container_name,
- value=None, driver=self)
-
- del self._containers[container_name]
- return True
-
- def download_object(self, obj, destination_path, overwrite_existing=False,
- delete_on_failure=True):
- kwargs_dict = {'obj': obj,
- 'response': DummyFileObject(),
- 'destination_path': destination_path,
- 'overwrite_existing': overwrite_existing,
- 'delete_on_failure': delete_on_failure}
-
- return self._save_object(**kwargs_dict)
-
- def download_object_as_stream(self, obj, chunk_size=None):
- """
- >>> driver = DummyStorageDriver('key', 'secret')
- >>> container = driver.create_container(
- ... container_name='test container 1')
- ... #doctest: +IGNORE_EXCEPTION_DETAIL
- >>> obj = container.upload_object_via_stream(object_name='test object',
- ... iterator=DummyFileObject(5, 10), extra={})
- >>> stream = container.download_object_as_stream(obj)
- >>> stream #doctest: +ELLIPSIS
- <...closed...>
-
- @inherits: :class:`StorageDriver.download_object_as_stream`
- """
-
- return DummyFileObject()
-
- def upload_object(self, file_path, container, object_name, extra=None,
- file_hash=None):
- """
- >>> driver = DummyStorageDriver('key', 'secret')
- >>> container_name = 'test container 1'
- >>> container = driver.create_container(container_name=container_name)
- >>> container.upload_object(file_path='/tmp/inexistent.file',
- ... object_name='test') #doctest: +IGNORE_EXCEPTION_DETAIL
- Traceback (most recent call last):
- LibcloudError:
- >>> file_path = path = os.path.abspath(__file__)
- >>> file_size = os.path.getsize(file_path)
- >>> obj = container.upload_object(file_path=file_path,
- ... object_name='test')
- >>> obj #doctest: +ELLIPSIS
- <Object: name=test, size=...>
- >>> obj.size == file_size
- True
-
- @inherits: :class:`StorageDriver.upload_object`
- :param file_hash: File hash
- :type file_hash: ``str``
- """
-
- if not os.path.exists(file_path):
- raise LibcloudError(value='File %s does not exist' % (file_path),
- driver=self)
-
- size = os.path.getsize(file_path)
- return self._add_object(container=container, object_name=object_name,
- size=size, extra=extra)
-
- def upload_object_via_stream(self, iterator, container,
- object_name, extra=None):
- """
- >>> driver = DummyStorageDriver('key', 'secret')
- >>> container = driver.create_container(
- ... container_name='test container 1')
- ... #doctest: +IGNORE_EXCEPTION_DETAIL
- >>> obj = container.upload_object_via_stream(
- ... object_name='test object', iterator=DummyFileObject(5, 10),
- ... extra={})
- >>> obj #doctest: +ELLIPSIS
- <Object: name=test object, size=50, ...>
-
- @inherits: :class:`StorageDriver.upload_object_via_stream`
- """
-
- size = len(iterator)
- return self._add_object(container=container, object_name=object_name,
- size=size, extra=extra)
-
- def delete_object(self, obj):
- """
- >>> driver = DummyStorageDriver('key', 'secret')
- >>> container = driver.create_container(
- ... container_name='test container 1')
- ... #doctest: +IGNORE_EXCEPTION_DETAIL
- >>> obj = container.upload_object_via_stream(object_name='test object',
- ... iterator=DummyFileObject(5, 10), extra={})
- >>> obj #doctest: +ELLIPSIS
- <Object: name=test object, size=50, ...>
- >>> container.delete_object(obj=obj)
- True
- >>> obj = Object(name='test object 2',
- ... size=1000, hash=None, extra=None,
- ... meta_data=None, container=container,driver=None)
- >>> container.delete_object(obj=obj) #doctest: +IGNORE_EXCEPTION_DETAIL
- Traceback (most recent call last):
- ObjectDoesNotExistError:
-
- @inherits: :class:`StorageDriver.delete_object`
- """
-
- container_name = obj.container.name
- object_name = obj.name
- obj = self.get_object(container_name=container_name,
- object_name=object_name)
-
- del self._containers[container_name]['objects'][object_name]
- return True
-
- def _add_object(self, container, object_name, size, extra=None):
- container = self.get_container(container.name)
-
- extra = extra or {}
- meta_data = extra.get('meta_data', {})
- meta_data.update({'cdn_url': 'http://www.test.com/object/%s' %
- (object_name.replace(' ', '_'))})
- obj = Object(name=object_name, size=size, extra=extra, hash=None,
- meta_data=meta_data, container=container, driver=self)
-
- self._containers[container.name]['objects'][object_name] = obj
- return obj
-
-if __name__ == "__main__":
- import doctest
- doctest.testmod()
http://git-wip-us.apache.org/repos/asf/libcloud/blob/8afcda91/apache-libcloud-1.0.0rc2/libcloud/storage/drivers/google_storage.py
----------------------------------------------------------------------
diff --git a/apache-libcloud-1.0.0rc2/libcloud/storage/drivers/google_storage.py b/apache-libcloud-1.0.0rc2/libcloud/storage/drivers/google_storage.py
deleted file mode 100644
index 580a29c..0000000
--- a/apache-libcloud-1.0.0rc2/libcloud/storage/drivers/google_storage.py
+++ /dev/null
@@ -1,145 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import copy
-
-import email.utils
-
-from libcloud.common.base import ConnectionUserAndKey
-from libcloud.common.google import GoogleAuthType
-from libcloud.common.google import GoogleOAuth2Credential
-from libcloud.storage.drivers.s3 import BaseS3Connection
-from libcloud.storage.drivers.s3 import BaseS3StorageDriver
-from libcloud.storage.drivers.s3 import S3RawResponse
-from libcloud.storage.drivers.s3 import S3Response
-
-# Docs are a lie. Actual namespace returned is different that the one listed
-# in the docs.
-SIGNATURE_IDENTIFIER = 'GOOG1'
-API_VERSION = '2006-03-01'
-NAMESPACE = 'http://doc.s3.amazonaws.com/%s' % (API_VERSION)
-
-
-class GoogleStorageConnection(ConnectionUserAndKey):
- """
- Represents a single connection to the Google storage API endpoint.
-
- This can either authenticate via the Google OAuth2 methods or via
- the S3 HMAC interoperability method.
- """
-
- host = 'storage.googleapis.com'
- responseCls = S3Response
- rawResponseCls = S3RawResponse
- PROJECT_ID_HEADER = 'x-goog-project-id'
-
- def __init__(self, user_id, key, secure, auth_type=None,
- credential_file=None, **kwargs):
- self.auth_type = auth_type or GoogleAuthType.guess_type(user_id)
- if GoogleAuthType.is_oauth2(self.auth_type):
- self.oauth2_credential = GoogleOAuth2Credential(
- user_id, key, self.auth_type, credential_file, **kwargs)
- else:
- self.oauth2_credential = None
- super(GoogleStorageConnection, self).__init__(user_id, key, secure,
- **kwargs)
-
- def add_default_headers(self, headers):
- date = email.utils.formatdate(usegmt=True)
- headers['Date'] = date
- project = self.get_project()
- if project:
- headers[self.PROJECT_ID_HEADER] = project
- return headers
-
- def get_project(self):
- return getattr(self.driver, 'project')
-
- def pre_connect_hook(self, params, headers):
- if self.auth_type == GoogleAuthType.GCS_S3:
- signature = self._get_s3_auth_signature(params, headers)
- headers['Authorization'] = '%s %s:%s' % (SIGNATURE_IDENTIFIER,
- self.user_id, signature)
- else:
- headers['Authorization'] = ('Bearer ' +
- self.oauth2_credential.access_token)
- return params, headers
-
- def _get_s3_auth_signature(self, params, headers):
- """Hacky wrapper to work with S3's get_auth_signature."""
- headers_copy = {}
- params_copy = copy.deepcopy(params)
-
- # Lowercase all headers except 'date' and Google header values
- for k, v in headers.items():
- k_lower = k.lower()
- if (k_lower == 'date' or k_lower.startswith(
- GoogleStorageDriver.http_vendor_prefix) or
- not isinstance(v, str)):
- headers_copy[k_lower] = v
- else:
- headers_copy[k_lower] = v.lower()
-
- return BaseS3Connection.get_auth_signature(
- method=self.method,
- headers=headers_copy,
- params=params_copy,
- expires=None,
- secret_key=self.key,
- path=self.action,
- vendor_prefix=GoogleStorageDriver.http_vendor_prefix)
-
-
-class GoogleStorageDriver(BaseS3StorageDriver):
- """
- Driver for Google Cloud Storage.
-
- Can authenticate via standard Google Cloud methods (Service Accounts,
- Installed App credentials, and GCE instance service accounts)
-
- Examples:
-
- Service Accounts::
-
- driver = GoogleStorageDriver(key=client_email, secret=private_key, ...)
-
- Installed Application::
-
- driver = GoogleStorageDriver(key=client_id, secret=client_secret, ...)
-
- From GCE instance::
-
- driver = GoogleStorageDriver(key=foo , secret=bar, ...)
-
- Can also authenticate via Google Cloud Storage's S3 HMAC interoperability
- API. S3 user keys are 20 alphanumeric characters, starting with GOOG.
-
- Example::
-
- driver = GoogleStorageDriver(key='GOOG0123456789ABCXYZ',
- secret=key_secret)
- """
- name = 'Google Storage'
- website = 'http://cloud.google.com/'
- connectionCls = GoogleStorageConnection
- hash_type = 'md5'
- namespace = NAMESPACE
- supports_chunked_encoding = False
- supports_s3_multipart_upload = False
- http_vendor_prefix = 'x-goog'
-
- def __init__(self, key, secret=None, project=None, **kwargs):
- self.project = project
- super(GoogleStorageDriver, self).__init__(key, secret, **kwargs)
http://git-wip-us.apache.org/repos/asf/libcloud/blob/8afcda91/apache-libcloud-1.0.0rc2/libcloud/storage/drivers/ktucloud.py
----------------------------------------------------------------------
diff --git a/apache-libcloud-1.0.0rc2/libcloud/storage/drivers/ktucloud.py b/apache-libcloud-1.0.0rc2/libcloud/storage/drivers/ktucloud.py
deleted file mode 100644
index 385d445..0000000
--- a/apache-libcloud-1.0.0rc2/libcloud/storage/drivers/ktucloud.py
+++ /dev/null
@@ -1,56 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from libcloud.common.types import LibcloudError
-from libcloud.storage.providers import Provider
-
-from libcloud.storage.drivers.cloudfiles import CloudFilesConnection
-from libcloud.storage.drivers.cloudfiles import CloudFilesStorageDriver
-
-KTUCLOUDSTORAGE_AUTH_URL = "https://ssproxy.ucloudbiz.olleh.com/auth/v1.0"
-KTUCLOUDSTORAGE_API_VERSION = "1.0"
-
-
-class KTUCloudStorageConnection(CloudFilesConnection):
- """
- Connection class for the KT UCloud Storage endpoint.
- """
-
- auth_url = KTUCLOUDSTORAGE_AUTH_URL
- _auth_version = KTUCLOUDSTORAGE_API_VERSION
-
- def get_endpoint(self):
- eps = self.service_catalog.get_endpoints(name='cloudFiles')
-
- if len(eps) == 0:
- raise LibcloudError('Could not find specified endpoint')
-
- ep = eps[0]
- public_url = ep.url
-
- if not public_url:
- raise LibcloudError('Could not find specified endpoint')
-
- return public_url
-
-
-class KTUCloudStorageDriver(CloudFilesStorageDriver):
- """
- Cloudfiles storage driver for the UK endpoint.
- """
-
- type = Provider.KTUCLOUD
- name = 'KTUCloud Storage'
- connectionCls = KTUCloudStorageConnection