You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@libcloud.apache.org by to...@apache.org on 2016/05/20 09:15:46 UTC
[1/3] libcloud git commit: Add Ceph S3 RGW storage drivers
Repository: libcloud
Updated Branches:
refs/heads/trunk 735a786e0 -> abba94a9e
Add Ceph S3 RGW storage drivers
Add Ceph S3 RGW drivers and related stuff in the drivers/rgw.py module:
- S3 RGW driver
- S3 RGW Outscale driver
Closes #786
Signed-off-by: Javier M. Mellid <jm...@igalia.com>
Signed-off-by: Tomaz Muraus <to...@tomaz.me>
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/f81ab675
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/f81ab675
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/f81ab675
Branch: refs/heads/trunk
Commit: f81ab6758a59b6e63e421b50e7b0bfe3f94272a5
Parents: 735a786
Author: Javier M. Mellid <jm...@igalia.com>
Authored: Tue May 17 09:07:17 2016 +0000
Committer: Tomaz Muraus <to...@tomaz.me>
Committed: Fri May 20 10:57:13 2016 +0200
----------------------------------------------------------------------
libcloud/storage/drivers/rgw.py | 137 +++++++++++++++++++++++++++++++++++
libcloud/storage/drivers/s3.py | 80 +-------------------
libcloud/storage/providers.py | 4 +-
libcloud/storage/types.py | 6 +-
4 files changed, 145 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/f81ab675/libcloud/storage/drivers/rgw.py
----------------------------------------------------------------------
diff --git a/libcloud/storage/drivers/rgw.py b/libcloud/storage/drivers/rgw.py
new file mode 100644
index 0000000..684047f
--- /dev/null
+++ b/libcloud/storage/drivers/rgw.py
@@ -0,0 +1,137 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from libcloud.common.types import LibcloudError
+
+from libcloud.storage.drivers.s3 import BaseS3Connection, S3Connection,\
+ S3StorageDriver, API_VERSION
+
+from libcloud.common.aws import SignedAWSConnection, DEFAULT_SIGNATURE_VERSION
+
+S3_RGW_DEFAULT_REGION = 'default'
+
+S3_RGW_OUTSCALE_HOSTS_BY_REGION =\
+ {'eu-west-1': 'osu.eu-west-1.outscale.com',
+ 'eu-west-2': 'osu.eu-west-2.outscale.com',
+ 'us-west-1': 'osu.us-west-1.outscale.com',
+ 'us-east-2': 'osu.us-east-2.outscale.com',
+ 'cn-southeast-1': 'osu.cn-southeast-1.outscale.hk'}
+
+S3_RGW_OUTSCALE_DEFAULT_REGION = 'eu-west-2'
+
+
+class S3RGWConnectionAWS4(SignedAWSConnection, BaseS3Connection):
+ service_name = 's3'
+ version = API_VERSION
+
+ def __init__(self, user_id, key, secure=True, host=None, port=None,
+ url=None, timeout=None, proxy_url=None, token=None,
+ retry_delay=None, backoff=None, **kwargs):
+
+ super(S3RGWConnectionAWS4, self).__init__(user_id, key,
+ secure, host,
+ port, url,
+ timeout,
+ proxy_url, token,
+ retry_delay,
+ backoff,
+ 4) # force aws4
+
+
+class S3RGWConnectionAWS2(S3Connection):
+
+ def __init__(self, user_id, key, secure=True, host=None, port=None,
+ url=None, timeout=None, proxy_url=None, token=None,
+ retry_delay=None, backoff=None, **kwargs):
+
+ super(S3RGWConnectionAWS2, self).__init__(user_id, key,
+ secure, host,
+ port, url,
+ timeout,
+ proxy_url, token,
+ retry_delay,
+ backoff)
+
+
+class S3RGWStorageDriver(S3StorageDriver):
+
+ def __init__(self, key, secret=None, secure=True, host=None, port=None,
+ api_version=None, region=S3_RGW_DEFAULT_REGION, **kwargs):
+ if host is None:
+ raise LibcloudError('host required', driver=self)
+ self.name = 'Ceph RGW S3 (%s)' % (region)
+ self.ex_location_name = region
+ self.region_name = region
+
+ self.signature_version = str(kwargs.pop('signature_version',
+ DEFAULT_SIGNATURE_VERSION))
+
+ if self.signature_version not in ['2', '4']:
+ raise ValueError('Invalid signature_version: %s' %
+ (self.signature_version))
+
+ if self.signature_version == '2':
+ self.connectionCls = S3RGWOutscaleConnectionAWS2
+ elif self.signature_version == '4':
+ self.connectionCls = S3RGWOutscaleConnectionAWS4
+ self.connectionCls.host = host
+ super(S3RGWStorageDriver, self).__init__(key, secret,
+ secure, host, port,
+ api_version, region,
+ **kwargs)
+
+ def _ex_connection_class_kwargs(self):
+ kwargs = {}
+ kwargs['signature_version'] = self.signature_version
+ return kwargs
+
+
+class S3RGWOutscaleConnectionAWS4(S3RGWConnectionAWS4):
+ pass
+
+
+class S3RGWOutscaleConnectionAWS2(S3RGWConnectionAWS2):
+ pass
+
+
+class S3RGWOutscaleStorageDriver(S3RGWStorageDriver):
+
+ def __init__(self, key, secret=None, secure=True, host=None, port=None,
+ api_version=None, region=S3_RGW_OUTSCALE_DEFAULT_REGION,
+ **kwargs):
+ if region not in S3_RGW_OUTSCALE_HOSTS_BY_REGION:
+ raise LibcloudError('Unknown region (%s)' % (region), driver=self)
+
+ self.name = 'OUTSCALE Ceph RGW S3 (%s)' % (region)
+ self.ex_location_name = region
+ self.region_name = region
+ self.signature_version = str(kwargs.pop('signature_version',
+ DEFAULT_SIGNATURE_VERSION))
+
+ if self.signature_version not in ['2', '4']:
+ raise ValueError('Invalid signature_version: %s' %
+ (self.signature_version))
+
+ if self.signature_version == '2':
+ self.connectionCls = S3RGWOutscaleConnectionAWS2
+ elif self.signature_version == '4':
+ self.connectionCls = S3RGWOutscaleConnectionAWS4
+
+ host = S3_RGW_OUTSCALE_HOSTS_BY_REGION[region]
+ self.connectionCls.host = host
+ super(S3RGWStorageDriver, self).__init__(key, secret,
+ secure, host, port,
+ api_version, region,
+ **kwargs)
http://git-wip-us.apache.org/repos/asf/libcloud/blob/f81ab675/libcloud/storage/drivers/s3.py
----------------------------------------------------------------------
diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py
index 6a16751..26ecd56 100644
--- a/libcloud/storage/drivers/s3.py
+++ b/libcloud/storage/drivers/s3.py
@@ -36,7 +36,7 @@ from libcloud.utils.files import read_in_chunks
from libcloud.common.types import InvalidCredsError, LibcloudError
from libcloud.common.base import ConnectionUserAndKey, RawResponse
from libcloud.common.aws import AWSBaseResponse, AWSDriver, \
- AWSTokenConnection, SignedAWSConnection, DEFAULT_SIGNATURE_VERSION
+ AWSTokenConnection, SignedAWSConnection
from libcloud.storage.base import Object, Container, StorageDriver
from libcloud.storage.types import ContainerError
@@ -60,15 +60,6 @@ S3_AP_NORTHEAST2_HOST = 's3-ap-northeast-2.amazonaws.com'
S3_AP_NORTHEAST_HOST = S3_AP_NORTHEAST1_HOST
S3_SA_EAST_HOST = 's3-sa-east-1.amazonaws.com'
-S3_RGW_OUTSCALE_HOSTS_BY_REGION =\
- {'eu-west-1': 'osu.eu-west-1.outscale.com',
- 'eu-west-2': 'osu.eu-west-2.outscale.com',
- 'us-west-1': 'osu.us-west-1.outscale.com',
- 'us-east-2': 'osu.us-east-2.outscale.com',
- 'cn-southeast-1': 'osu.cn-southeast-1.outscale.hk'}
-
-S3_RGW_OUTSCALE_DEFAULT_REGION = 'eu-west-2'
-
API_VERSION = '2006-03-01'
NAMESPACE = 'http://s3.amazonaws.com/doc/%s/' % (API_VERSION)
@@ -1013,72 +1004,3 @@ class S3SAEastStorageDriver(S3StorageDriver):
name = 'Amazon S3 (sa-east-1)'
connectionCls = S3SAEastConnection
ex_location_name = 'sa-east-1'
-
-
-class S3RGWOutscaleConnectionAWS4(SignedAWSConnection, BaseS3Connection):
- service_name = 's3'
- version = API_VERSION
-
- def __init__(self, user_id, key, secure=True, host=None, port=None,
- url=None, timeout=None, proxy_url=None, token=None,
- retry_delay=None, backoff=None, **kwargs):
-
- super(S3RGWOutscaleConnectionAWS4, self).__init__(user_id, key,
- secure, host,
- port, url,
- timeout,
- proxy_url, token,
- retry_delay,
- backoff,
- 4) # force aws4
-
-
-class S3RGWOutscaleConnectionAWS2(S3Connection):
-
- def __init__(self, user_id, key, secure=True, host=None, port=None,
- url=None, timeout=None, proxy_url=None, token=None,
- retry_delay=None, backoff=None, **kwargs):
-
- super(S3RGWOutscaleConnectionAWS2, self).__init__(user_id, key,
- secure, host,
- port, url,
- timeout,
- proxy_url, token,
- retry_delay,
- backoff)
-
-
-class S3RGWOutscaleStorageDriver(S3StorageDriver):
-
- def __init__(self, key, secret=None, secure=True, host=None, port=None,
- api_version=None, region=S3_RGW_OUTSCALE_DEFAULT_REGION,
- **kwargs):
- if region not in S3_RGW_OUTSCALE_HOSTS_BY_REGION:
- raise LibcloudError('Unknown region (%s)' % (region), driver=self)
-
- self.name = 'OUTSCALE Ceph RGW S3 (%s)' % (region)
- self.ex_location_name = region
- self.region_name = region
- self.signature_version = str(kwargs.pop('signature_version',
- DEFAULT_SIGNATURE_VERSION))
-
- if self.signature_version not in ['2', '4']:
- raise ValueError('Invalid signature_version: %s' %
- (self.signature_version))
-
- if self.signature_version == '2':
- self.connectionCls = S3RGWOutscaleConnectionAWS2
- elif self.signature_version == '4':
- self.connectionCls = S3RGWOutscaleConnectionAWS4
-
- host = S3_RGW_OUTSCALE_HOSTS_BY_REGION[region]
- self.connectionCls.host = host
- super(S3RGWOutscaleStorageDriver, self).__init__(key, secret,
- secure, host, port,
- api_version, region,
- **kwargs)
-
- def _ex_connection_class_kwargs(self):
- kwargs = {}
- kwargs['signature_version'] = self.signature_version
- return kwargs
http://git-wip-us.apache.org/repos/asf/libcloud/blob/f81ab675/libcloud/storage/providers.py
----------------------------------------------------------------------
diff --git a/libcloud/storage/providers.py b/libcloud/storage/providers.py
index eddd470..5725a28 100644
--- a/libcloud/storage/providers.py
+++ b/libcloud/storage/providers.py
@@ -43,8 +43,10 @@ DRIVERS = {
('libcloud.storage.drivers.s3', 'S3APNE2StorageDriver'),
Provider.S3_SA_EAST:
('libcloud.storage.drivers.s3', 'S3SAEastStorageDriver'),
+ Provider.S3_RGW:
+ ('libcloud.storage.drivers.rgw', 'S3RGWStorageDriver'),
Provider.S3_RGW_OUTSCALE:
- ('libcloud.storage.drivers.s3', 'S3RGWOutscaleStorageDriver'),
+ ('libcloud.storage.drivers.rgw', 'S3RGWOutscaleStorageDriver'),
Provider.NINEFOLD:
('libcloud.storage.drivers.ninefold', 'NinefoldStorageDriver'),
Provider.GOOGLE_STORAGE:
http://git-wip-us.apache.org/repos/asf/libcloud/blob/f81ab675/libcloud/storage/types.py
----------------------------------------------------------------------
diff --git a/libcloud/storage/types.py b/libcloud/storage/types.py
index c4c61bd..224bec8 100644
--- a/libcloud/storage/types.py
+++ b/libcloud/storage/types.py
@@ -49,9 +49,10 @@ class Provider(object):
:cvar S3_AP_NORTHEAST_HOST: Amazon S3 Asia South East (Tokyo)
:cvar S3_AP_SOUTHEAST_HOST: Amazon S3 Asia South East (Singapore)
:cvar S3_EU_WEST: Amazon S3 EU West (Ireland)
- :cvar S3_RGW_OUTSCALE: OUTSCALE RGW S3
:cvar S3_US_WEST: Amazon S3 US West (Northern California)
:cvar S3_US_WEST_OREGON: Amazon S3 US West 2 (Oregon)
+ :cvar S3_RGW: S3 RGW
+ :cvar S3_RGW_OUTSCALE: OUTSCALE S3 RGW
"""
DUMMY = 'dummy'
ALIYUN_OSS = 'aliyun_oss'
@@ -71,10 +72,11 @@ class Provider(object):
S3_AP_NORTHEAST2 = 's3_ap_northeast_2'
S3_AP_SOUTHEAST = 's3_ap_southeast'
S3_EU_WEST = 's3_eu_west'
- S3_RGW_OUTSCALE = 's3_rgw_outscale'
S3_SA_EAST = 's3_sa_east'
S3_US_WEST = 's3_us_west'
S3_US_WEST_OREGON = 's3_us_west_oregon'
+ S3_RGW = 's3_rgw'
+ S3_RGW_OUTSCALE = 's3_rgw_outscale'
# Deperecated
CLOUDFILES_US = 'cloudfiles_us'
[3/3] libcloud git commit: Update changelog.
Posted by to...@apache.org.
Update changelog.
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/abba94a9
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/abba94a9
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/abba94a9
Branch: refs/heads/trunk
Commit: abba94a9eada2da3c6107a320268779df6ae2961
Parents: 967a452
Author: Tomaz Muraus <to...@tomaz.me>
Authored: Fri May 20 11:02:45 2016 +0200
Committer: Tomaz Muraus <to...@tomaz.me>
Committed: Fri May 20 11:02:45 2016 +0200
----------------------------------------------------------------------
CHANGES.rst | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/abba94a9/CHANGES.rst
----------------------------------------------------------------------
diff --git a/CHANGES.rst b/CHANGES.rst
index df2538b..e13f837 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -67,6 +67,11 @@ Storage
(GITHUB-736)
[Javier M. Mellid]
+
+- Add new S3 RGW storage driver.
+ (GITHUB-786)
+ [Javier M. Mellid]
+
DNS
~~~
[2/3] libcloud git commit: Fix lint, add missing __all__
Posted by to...@apache.org.
Fix lint, add missing __all__
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/967a4522
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/967a4522
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/967a4522
Branch: refs/heads/trunk
Commit: 967a4522bf4a7f0b6d78fec492aedbe9778f66cc
Parents: f81ab67
Author: Tomaz Muraus <to...@tomaz.me>
Authored: Fri May 20 10:59:38 2016 +0200
Committer: Tomaz Muraus <to...@tomaz.me>
Committed: Fri May 20 11:02:41 2016 +0200
----------------------------------------------------------------------
libcloud/storage/drivers/rgw.py | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/967a4522/libcloud/storage/drivers/rgw.py
----------------------------------------------------------------------
diff --git a/libcloud/storage/drivers/rgw.py b/libcloud/storage/drivers/rgw.py
index 684047f..0f16ac9 100644
--- a/libcloud/storage/drivers/rgw.py
+++ b/libcloud/storage/drivers/rgw.py
@@ -14,11 +14,14 @@
# limitations under the License.
from libcloud.common.types import LibcloudError
-
-from libcloud.storage.drivers.s3 import BaseS3Connection, S3Connection,\
- S3StorageDriver, API_VERSION
-
from libcloud.common.aws import SignedAWSConnection, DEFAULT_SIGNATURE_VERSION
+from libcloud.storage.drivers.s3 import BaseS3Connection, S3Connection
+from libcloud.storage.drivers.s3 import S3StorageDriver, API_VERSION
+
+__all__ = [
+ 'S3RGWStorageDriver',
+ 'S3RGWOutscaleStorageDriver'
+]
S3_RGW_DEFAULT_REGION = 'default'
@@ -131,7 +134,7 @@ class S3RGWOutscaleStorageDriver(S3RGWStorageDriver):
host = S3_RGW_OUTSCALE_HOSTS_BY_REGION[region]
self.connectionCls.host = host
- super(S3RGWStorageDriver, self).__init__(key, secret,
- secure, host, port,
- api_version, region,
- **kwargs)
+ super(S3RGWOutscaleStorageDriver, self).__init__(key, secret,
+ secure, host, port,
+ api_version, region,
+ **kwargs)