You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@libcloud.apache.org by to...@apache.org on 2015/08/08 23:40:44 UTC
libcloud git commit: LIBCLOUD-717 Implement v2 API request speed
optimization and other changes * Increased pagination to API maximum * Added
create_node parameter to support droplet features (backups, ipv6, user_data,
etc ) * Reorganized method groups f
Repository: libcloud
Updated Branches:
refs/heads/trunk 9e5eb41f6 -> 60b70718d
LIBCLOUD-717 Implement v2 API request speed optimization and other changes
* Increased pagination to API maximum
* Added create_node parameter to support droplet features (backups, ipv6, user_data, etc )
* Reorganized method groups for style
* Converted all list_* to paginated request
Pull Request 537 override for per_page API parameter
- implement ex_per_page keyword parameter for DigitalOcean_v2_BaseDriver
- Added deprecation warning for v1 initialization
( See https://developers.digitalocean.com/documentation/v1/ )
- Updates minor typo in documentation
Closes #537
Signed-off-by: Tomaz Muraus <to...@tomaz.me>
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/60b70718
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/60b70718
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/60b70718
Branch: refs/heads/trunk
Commit: 60b70718d909ec1a0b0f3e13a77bbeee34b9688e
Parents: 9e5eb41
Author: jcastillo2nd <j....@gmail.com>
Authored: Thu Jun 25 07:31:38 2015 +0000
Committer: Tomaz Muraus <to...@tomaz.me>
Committed: Sat Aug 8 23:37:34 2015 +0200
----------------------------------------------------------------------
CHANGES.rst | 9 +
docs/compute/drivers/digital_ocean.rst | 9 +-
.../compute/digitalocean/create_api_v2.0.py | 18 ++
libcloud/common/digitalocean.py | 29 +++-
libcloud/compute/drivers/digitalocean.py | 174 +++++++++----------
5 files changed, 144 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/60b70718/CHANGES.rst
----------------------------------------------------------------------
diff --git a/CHANGES.rst b/CHANGES.rst
index 961392d..752a80c 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -284,6 +284,15 @@ Compute
(GITHUB-555)
[Konstantin Skaburskas]
+- Various improvements in the DigitalOcean driver:
+ - Increase page size to API maximum.
+ - Add ``ex_create_attr`` kwarg to ``create_node`` method.
+ - Update all the ``list_*`` methods to use paginated requests
+ - Allow user to specify page size by passing ``ex_per_page`` argument to the
+ constructor.
+ (LIBCLOUD-717, GITHUB-537)
+ [Javier Castillo II]
+
Storage
~~~~~~~
http://git-wip-us.apache.org/repos/asf/libcloud/blob/60b70718/docs/compute/drivers/digital_ocean.rst
----------------------------------------------------------------------
diff --git a/docs/compute/drivers/digital_ocean.rst b/docs/compute/drivers/digital_ocean.rst
index cf0b07e..2031943 100644
--- a/docs/compute/drivers/digital_ocean.rst
+++ b/docs/compute/drivers/digital_ocean.rst
@@ -16,7 +16,8 @@ Instantiating a driver
DigitalOcean driver supports two API versions - old API v1.0 and the new API
v2.0. Since trunk (to be libcloud v0.18.0), the driver uses the correct API
based on the initialization with the Client ID (key) and Access Token (secret)
-for v1.0 or the Personal Access Token (key) in v2.0.
+for v1.0 or the Personal Access Token (key) in v2.0 and will throw an
+exception if the `api_version` is set explicitly without the proper arguments.
Instantiating a driver using API v2.0
-------------------------------------
@@ -30,6 +31,12 @@ Instantiating a driver using API v1.0
.. literalinclude:: /examples/compute/digitalocean/instantiate_api_v1.0.py
:language: python
+Creating a droplet using API v2.0
+---------------------------------
+
+.. literalinclude:: /examples/compute/digitalocean/create_api_v2.0.py
+ :language: python
+
API Docs
--------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/60b70718/docs/examples/compute/digitalocean/create_api_v2.0.py
----------------------------------------------------------------------
diff --git a/docs/examples/compute/digitalocean/create_api_v2.0.py b/docs/examples/compute/digitalocean/create_api_v2.0.py
new file mode 100644
index 0000000..bdb6295
--- /dev/null
+++ b/docs/examples/compute/digitalocean/create_api_v2.0.py
@@ -0,0 +1,18 @@
+from libcloud.compute.types import Provider
+from libcloud.compute.providers import get_driver
+
+cls = get_driver(Provider.DIGITAL_OCEAN)
+
+driver = cls('access token', api_version='v2')
+
+options = {'backups': True,
+ 'private_networking': True,
+ 'ssh_keys': [123456, 123457]}
+
+name = 'test.domain.tld'
+size = driver.list_sizes()[0]
+image = driver.list_images()[0]
+location = driver.list_locations()[0]
+
+node = driver.create_node(name, size, image, location,
+ ex_create_attr=options)
http://git-wip-us.apache.org/repos/asf/libcloud/blob/60b70718/libcloud/common/digitalocean.py
----------------------------------------------------------------------
diff --git a/libcloud/common/digitalocean.py b/libcloud/common/digitalocean.py
index 53dcb60..2e6f329 100644
--- a/libcloud/common/digitalocean.py
+++ b/libcloud/common/digitalocean.py
@@ -16,8 +16,9 @@
"""
Common settings and connection objects for DigitalOcean Cloud
"""
+import warnings
-from libcloud.utils.py3 import httplib
+from libcloud.utils.py3 import httplib, parse_qs, urlparse
from libcloud.common.base import BaseDriver
from libcloud.common.base import ConnectionUserAndKey, ConnectionKey
@@ -109,6 +110,16 @@ class DigitalOcean_v2_Connection(ConnectionKey):
headers['Content-Type'] = 'application/json'
return headers
+ def add_default_params(self, params):
+ """
+ Add parameters that are necessary for every request
+
+ This method adds ``per_page`` to the request to reduce the total
+ number of paginated requests to the API.
+ """
+ params['per_page'] = self.driver.ex_per_page
+ return params
+
class DigitalOceanConnection(DigitalOcean_v2_Connection):
"""
@@ -132,6 +143,8 @@ class DigitalOceanBaseDriver(BaseDriver):
if cls is DigitalOceanBaseDriver:
if api_version == 'v1' or secret is not None:
cls = DigitalOcean_v1_BaseDriver
+ warnings.warn("The v1 API has become deprecated. Please "
+ "consider utilizing the v2 API.")
elif api_version == 'v2':
cls = DigitalOcean_v2_BaseDriver
else:
@@ -175,9 +188,17 @@ class DigitalOcean_v1_BaseDriver(DigitalOceanBaseDriver):
class DigitalOcean_v2_BaseDriver(DigitalOceanBaseDriver):
"""
DigitalOcean BaseDriver using v2 of the API.
+
+ Supports `ex_per_page` ``int`` value keyword parameter to adjust per page
+ requests against the API.
"""
connectionCls = DigitalOcean_v2_Connection
+ def __init__(self, key, secret=None, secure=True, host=None, port=None,
+ api_version=None, region=None, ex_per_page=200, **kwargs):
+ self.ex_per_page = ex_per_page
+ super(DigitalOcean_v2_BaseDriver, self).__init__(key, **kwargs)
+
def ex_account_info(self):
return self.connection.request('/v2/account').object['account']
@@ -207,11 +228,14 @@ class DigitalOcean_v2_BaseDriver(DigitalOceanBaseDriver):
:type obj: ``str``
:return: ``list`` of API response objects
+ :rtype: ``list``
"""
params = {}
data = self.connection.request(url)
try:
- pages = data.object['links']['pages']['last'].split('=')[-1]
+ query = urlparse.urlparse(data.object['links']['pages']['last'])
+ # The query[4] references the query parameters from the url
+ pages = parse_qs(query[4])['page'][0]
values = data.object[obj]
for page in range(2, int(pages) + 1):
params.update({'page': page})
@@ -223,5 +247,4 @@ class DigitalOcean_v2_BaseDriver(DigitalOceanBaseDriver):
data = values
except KeyError: # No pages.
data = data.object[obj]
-
return data
http://git-wip-us.apache.org/repos/asf/libcloud/blob/60b70718/libcloud/compute/drivers/digitalocean.py
----------------------------------------------------------------------
diff --git a/libcloud/compute/drivers/digitalocean.py b/libcloud/compute/drivers/digitalocean.py
index 83b23f6..a5c3bb9 100644
--- a/libcloud/compute/drivers/digitalocean.py
+++ b/libcloud/compute/drivers/digitalocean.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""
-Digital Ocean Driver
+DigitalOcean Driver
"""
import json
import warnings
@@ -24,8 +24,8 @@ from libcloud.common.digitalocean import DigitalOcean_v1_BaseDriver
from libcloud.common.digitalocean import DigitalOcean_v2_BaseDriver
from libcloud.common.types import InvalidCredsError
from libcloud.compute.types import Provider, NodeState
-from libcloud.compute.base import NodeDriver, Node
from libcloud.compute.base import NodeImage, NodeSize, NodeLocation, KeyPair
+from libcloud.compute.base import Node, NodeDriver
__all__ = [
'DigitalOceanNodeDriver',
@@ -320,42 +320,61 @@ class DigitalOcean_v2_NodeDriver(DigitalOcean_v2_BaseDriver,
'active': NodeState.RUNNING,
'archive': NodeState.TERMINATED}
- def list_nodes(self):
- data = self._paginated_request('/v2/droplets', 'droplets')
- return list(map(self._to_node, data))
-
- def list_locations(self):
- data = self.connection.request('/v2/regions').object['regions']
- return list(map(self._to_location, data))
+ EX_CREATE_ATTRIBUTES = ['backups',
+ 'ipv6',
+ 'private_networking',
+ 'ssh_keys']
def list_images(self):
data = self._paginated_request('/v2/images', 'images')
return list(map(self._to_image, data))
+ def list_key_pairs(self):
+ """
+ List all the available SSH keys.
+
+ :return: Available SSH keys.
+ :rtype: ``list`` of :class:`KeyPair`
+ """
+ data = self._paginated_request('/v2/account/keys', 'ssh_keys')
+ return list(map(self._to_key_pair, data))
+
+ def list_locations(self):
+ data = self._paginated_request('/v2/regions', 'regions')
+ return list(map(self._to_location, data))
+
+ def list_nodes(self):
+ data = self._paginated_request('/v2/droplets', 'droplets')
+ return list(map(self._to_node, data))
+
def list_sizes(self):
- data = self.connection.request('/v2/sizes').object['sizes']
+ data = self._paginated_request('/v2/sizes', 'sizes')
return list(map(self._to_size, data))
- def create_node(self, name, size, image, location,
+ def create_node(self, name, size, image, location, ex_create_attr=None,
ex_ssh_key_ids=None, ex_user_data=None):
"""
Create a node.
- :keyword name: Name of the node to be created.
- :type name: ``str``
+ The `ex_create_attr` parameter can include the following dictionary
+ key and value pairs:
- :keyword size: Size of the node.
- :type size: ``NodeSize``
+ * `backups`: ``bool`` defaults to False
+ * `ipv6`: ``bool`` defaults to False
+ * `private_networking`: ``bool`` defaults to False
+ * `user_data`: ``str`` for cloud-config data
+ * `ssh_keys`: ``list`` of ``int`` key ids or ``str`` fingerprints
- :keyword image: Image to be used to create node.
- :type image: ``NodeImage``
+ `ex_create_attr['ssh_keys']` will override `ex_ssh_key_ids` assignment.
- :keyword location: Location where the node will be created.
- :type location: ``NodeLocation``
+ :keyword ex_create_attr: A dictionary of optional attributes for
+ droplet creation
+ :type ex_create_attr: ``dict``
- :keyword ex_ssh_key_ids: A list of ssh key ids which will be added
- to the server. (optional)
- :type ex_ssh_key_ids: ``list`` of ``str``
+ :keyword ex_ssh_key_ids: A list of ssh key ids which will be added
+ to the server. (optional)
+ :type ex_ssh_key_ids: ``list`` of ``int`` key ids or ``str``
+ key fingerprints
:keyword ex_user_data: User data to be added to the node on create.
(optional)
@@ -368,8 +387,15 @@ class DigitalOcean_v2_NodeDriver(DigitalOcean_v2_BaseDriver,
'region': location.id, 'user_data': ex_user_data}
if ex_ssh_key_ids:
+ warnings.warn("The ex_ssh_key_ids parameter has been deprecated in"
+ " favor of the ex_create_attr parameter.")
attr['ssh_keys'] = ex_ssh_key_ids
+ ex_create_attr = ex_create_attr or {}
+ for key in ex_create_attr.keys():
+ if key in self.EX_CREATE_ATTRIBUTES:
+ attr[key] = ex_create_attr[key]
+
res = self.connection.request('/v2/droplets',
data=json.dumps(attr), method='POST')
@@ -383,36 +409,20 @@ class DigitalOcean_v2_NodeDriver(DigitalOcean_v2_BaseDriver,
return self._to_node(data=data)
- def reboot_node(self, node):
- attr = {'type': 'reboot'}
- res = self.connection.request('/v2/droplets/%s/actions' % (node.id),
- data=json.dumps(attr), method='POST')
- return res.status == httplib.CREATED
-
def destroy_node(self, node):
res = self.connection.request('/v2/droplets/%s' % (node.id),
method='DELETE')
return res.status == httplib.NO_CONTENT
- def get_image(self, image_id):
- """
- Get an image based on an image_id
-
- @inherits: :class:`NodeDriver.get_image`
-
- :param image_id: Image identifier
- :type image_id: ``int``
-
- :return: A NodeImage object
- :rtype: :class:`NodeImage`
- """
- res = self.connection.request('/v2/images/%s' % (image_id))
- data = res.object['image']
- return self._to_image(data)
+ def reboot_node(self, node):
+ attr = {'type': 'reboot'}
+ res = self.connection.request('/v2/droplets/%s/actions' % (node.id),
+ data=json.dumps(attr), method='POST')
+ return res.status == httplib.CREATED
def create_image(self, node, name):
"""
- Create an image fron a Node.
+ Create an image from a Node.
@inherits: :class:`NodeDriver.create_image`
@@ -443,6 +453,21 @@ class DigitalOcean_v2_NodeDriver(DigitalOcean_v2_BaseDriver,
method='DELETE')
return res.status == httplib.NO_CONTENT
+ def get_image(self, image_id):
+ """
+ Get an image based on an image_id
+
+ @inherits: :class:`NodeDriver.get_image`
+
+ :param image_id: Image identifier
+ :type image_id: ``int``
+
+ :return: A NodeImage object
+ :rtype: :class:`NodeImage`
+ """
+ data = self._paginated_request('/v2/images/%s' % (image_id), 'image')
+ return self._to_image(data)
+
def ex_rename_node(self, node, name):
attr = {'type': 'rename', 'name': name}
res = self.connection.request('/v2/droplets/%s/actions' % (node.id),
@@ -461,31 +486,7 @@ class DigitalOcean_v2_NodeDriver(DigitalOcean_v2_BaseDriver,
data=json.dumps(attr), method='POST')
return res.status == httplib.CREATED
- def list_key_pairs(self):
- """
- List all the available SSH keys.
-
- :return: Available SSH keys.
- :rtype: ``list`` of :class:`KeyPair`
- """
- data = self._paginated_request('/v2/account/keys', 'ssh_keys')
- return list(map(self._to_key_pair, data))
-
- def get_key_pair(self, name):
- """
- Retrieve a single key pair.
-
- :param name: Name of the key pair to retrieve.
- :type name: ``str``
-
- :rtype: :class:`.KeyPair`
- """
- qkey = [k for k in self.list_key_pairs() if k.name == name][0]
- data = self.connection.request('/v2/account/keys/%s' %
- qkey.extra['id']).object['ssh_key']
- return self._to_key_pair(data=data)
-
- def create_key_pair(self, name, public_key):
+ def create_key_pair(self, name, public_key=''):
"""
Create a new SSH key.
@@ -515,28 +516,19 @@ class DigitalOcean_v2_NodeDriver(DigitalOcean_v2_BaseDriver,
method='DELETE')
return res.status == httplib.NO_CONTENT
- def _paginated_request(self, url, obj):
- """
- Perform multiple calls in order to have a full list of elements
- when the API are paginated.
+ def get_key_pair(self, name):
"""
- params = {}
- data = self.connection.request(url)
- try:
- pages = data.object['links']['pages']['last'].split('=')[-1]
- values = data.object[obj]
- for page in range(2, int(pages) + 1):
- params.update({'page': page})
- new_data = self.connection.request(url, params=params)
+ Retrieve a single key pair.
- more_values = new_data.object[obj]
- for value in more_values:
- values.append(value)
- data = values
- except KeyError: # No pages.
- data = data.object[obj]
+ :param name: Name of the key pair to retrieve.
+ :type name: ``str``
- return data
+ :rtype: :class:`.KeyPair`
+ """
+ qkey = [k for k in self.list_key_pairs() if k.name == name][0]
+ data = self.connection.request('/v2/account/keys/%s' %
+ qkey.extra['id']).object['ssh_key']
+ return self._to_key_pair(data=data)
def _to_node(self, data):
extra_keys = ['memory', 'vcpus', 'disk', 'region', 'image',
@@ -564,7 +556,7 @@ class DigitalOcean_v2_NodeDriver(DigitalOcean_v2_BaseDriver,
node = Node(id=data['id'], name=data['name'], state=state,
public_ips=public_ips, private_ips=private_ips,
- extra=extra, driver=self)
+ driver=self, extra=extra)
return node
def _to_image(self, data):
@@ -574,8 +566,8 @@ class DigitalOcean_v2_NodeDriver(DigitalOcean_v2_BaseDriver,
'regions': data['regions'],
'min_disk_size': data['min_disk_size'],
'created_at': data['created_at']}
- return NodeImage(id=data['id'], name=data['name'], extra=extra,
- driver=self)
+ return NodeImage(id=data['id'], name=data['name'], driver=self,
+ extra=extra)
def _to_location(self, data):
return NodeLocation(id=data['slug'], name=data['name'], country=None,