You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@libcloud.apache.org by to...@apache.org on 2014/10/17 09:27:40 UTC
git commit: LIBCLOUD-578: GCE adding Service Accounts to create_node
Repository: libcloud
Updated Branches:
refs/heads/trunk 4158507b0 -> a38ade584
LIBCLOUD-578: GCE adding Service Accounts to create_node
Closes #372
Signed-off-by: Tomaz Muraus <to...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/a38ade58
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/a38ade58
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/a38ade58
Branch: refs/heads/trunk
Commit: a38ade58404f155c5de82a2a9512809856f63f17
Parents: 4158507
Author: Eric Johnson <er...@google.com>
Authored: Fri Oct 10 15:52:02 2014 +0000
Committer: Tomaz Muraus <to...@apache.org>
Committed: Fri Oct 17 15:20:41 2014 +0800
----------------------------------------------------------------------
CHANGES.rst | 5 +
docs/compute/drivers/gce.rst | 14 +++
.../compute/gce/gce_service_account_scopes.py | 31 +++++
libcloud/compute/drivers/gce.py | 124 +++++++++++++++++--
libcloud/test/compute/test_gce.py | 26 ++++
5 files changed, 193 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/a38ade58/CHANGES.rst
----------------------------------------------------------------------
diff --git a/CHANGES.rst b/CHANGES.rst
index a23f4eb..e7b9542 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -151,6 +151,11 @@ Compute
(GITHUB-370)
[Nirmal Ranganathan]
+- Add support for service scopes to the ``create_node`` method in the GCE
+ driver.
+ (LIBCLOUD-578, GITHUB-373)
+ [Eric Johnson]
+
Storage
~~~~~~~
http://git-wip-us.apache.org/repos/asf/libcloud/blob/a38ade58/docs/compute/drivers/gce.rst
----------------------------------------------------------------------
diff --git a/docs/compute/drivers/gce.rst b/docs/compute/drivers/gce.rst
index a6b6b71..6e91160 100644
--- a/docs/compute/drivers/gce.rst
+++ b/docs/compute/drivers/gce.rst
@@ -15,6 +15,7 @@ Google Compute Engine features:
* High-performance virtual machines
* Minute-level billing (10-minute minimum)
* Fast VM provisioning
+* Persistent block storage (SSD and standard)
* Native Load Balancing
Connecting to Google Compute Engine
@@ -68,6 +69,14 @@ To set up Installed Account authentication:
7. You will also need your "Project ID" which can be found by clicking on the
"Overview" link on the left sidebar.
+Accessing Google Cloud services from your Libcloud nodes
+--------------------------------------------------------
+In order for nodes created with libcloud to be able to access or manage other
+Google Cloud Platform services, you will need to specify a list of Service
+Account Scopes. By default libcloud will create nodes that only allow
+read-only access to Google Cloud Storage. A few of the examples below
+illustrate how to use Service Account Scopes.
+
Examples
--------
@@ -89,6 +98,11 @@ https://github.com/apache/libcloud/blob/trunk/demos/gce_demo.py
.. literalinclude:: /examples/compute/gce/gce_datacenter.py
+4. Specifying Service Account Scopes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. literalinclude:: /examples/compute/gce/gce_service_account_scopes.py
+
API Docs
--------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/a38ade58/docs/examples/compute/gce/gce_service_account_scopes.py
----------------------------------------------------------------------
diff --git a/docs/examples/compute/gce/gce_service_account_scopes.py b/docs/examples/compute/gce/gce_service_account_scopes.py
new file mode 100644
index 0000000..f5f9be9
--- /dev/null
+++ b/docs/examples/compute/gce/gce_service_account_scopes.py
@@ -0,0 +1,31 @@
+# See previous examples for connecting and creating the driver
+# ...
+driver = None
+
+# Define common example attributes
+s = 'n1-standard-1'
+i = 'debian-7'
+z = 'us-central1-a'
+
+# Service Account Scopes require a list of dictionaries. Each dictionary
+# can have an optional 'email' address specifying the Service Account
+# address, and list of 'scopes'. The default Service Account Scopes for
+# new nodes will effectively use:
+
+sa_scopes = [
+ {
+ 'email': 'default',
+ 'scopes': ['storage-ro']
+ }
+]
+
+# The expected scenario will likely use the default Service Account email
+# address, but allow users to override the default list of scopes.
+# For example, create a new node with full access to Google Cloud Storage
+# and Google Compute Engine:
+sa_scopes = [{'scopes': ['compute', 'storage-full']}]
+node_1 = driver.create_node("n1", s, i, z, ex_service_accounts=sa_scopes)
+
+# See Google's documentation for Accessing other Google Cloud services from
+# your Google Compute Engine instances at,
+# https://cloud.google.com/compute/docs/authentication
http://git-wip-us.apache.org/repos/asf/libcloud/blob/a38ade58/libcloud/compute/drivers/gce.py
----------------------------------------------------------------------
diff --git a/libcloud/compute/drivers/gce.py b/libcloud/compute/drivers/gce.py
index bbcfbb8..856c334 100644
--- a/libcloud/compute/drivers/gce.py
+++ b/libcloud/compute/drivers/gce.py
@@ -534,6 +534,22 @@ class GCENodeDriver(NodeDriver):
"TERMINATED": NodeState.TERMINATED
}
+ AUTH_URL = "https://www.googleapis.com/auth/"
+ SA_SCOPES_MAP = {
+ # list derived from 'gcloud compute instances create --help'
+ "bigquery": "bigquery",
+ "compute-ro": "compute.readonly",
+ "compute-rw": "compute",
+ "datastore": "datastore",
+ "sql": "sqlservice",
+ "sql-admin": "sqlservice.admin",
+ "storage-full": "devstorage.full_control",
+ "storage-ro": "devstorage.read_only",
+ "storage-rw": "devstorage.read_write",
+ "taskqueue": "taskqueue",
+ "userinfo-email": "userinfo.email"
+ }
+
def __init__(self, user_id, key, datacenter=None, project=None,
auth_type=None, scopes=None, **kwargs):
"""
@@ -1222,7 +1238,7 @@ class GCENodeDriver(NodeDriver):
ex_network='default', ex_tags=None, ex_metadata=None,
ex_boot_disk=None, use_existing_disk=True,
external_ip='ephemeral', ex_disk_type='pd-standard',
- ex_disk_auto_delete=True):
+ ex_disk_auto_delete=True, ex_service_accounts=None):
"""
Create a new node and return a node object for the node.
@@ -1273,6 +1289,20 @@ class GCENodeDriver(NodeDriver):
True by default.
:type ex_disk_auto_delete: ``bool``
+ :keyword ex_service_accounts: Specify a list of serviceAccounts when
+ creating the instance. The format is a
+ list of dictionaries containing email
+ and list of scopes, e.g.
+ [{'email':'default',
+ 'scopes':['compute', ...]}, ...]
+ Scopes can either be full URLs or short
+ names. If not provided, use the
+ 'default' service account email and a
+ scope of 'devstorage.read_only'. Also
+ accepts the aliases defined in
+ 'gcloud cmopute'.
+ :type ex_service_accounts: ``list``
+
:return: A Node object for the new node.
:rtype: :class:`Node`
"""
@@ -1315,7 +1345,8 @@ class GCENodeDriver(NodeDriver):
ex_tags, ex_metadata,
ex_boot_disk, external_ip,
ex_disk_type,
- ex_disk_auto_delete)
+ ex_disk_auto_delete,
+ ex_service_accounts)
self.connection.async_request(request, method='POST', data=node_data)
return self.ex_get_node(name, location.name)
@@ -1327,6 +1358,7 @@ class GCENodeDriver(NodeDriver):
poll_interval=2, external_ip='ephemeral',
ex_disk_type='pd-standard',
ex_auto_disk_delete=True,
+ ex_service_accounts=None,
timeout=DEFAULT_TASK_COMPLETION_TIMEOUT):
"""
Create multiple nodes and return a list of Node objects.
@@ -1392,6 +1424,20 @@ class GCENodeDriver(NodeDriver):
True by default.
:type ex_disk_auto_delete: ``bool``
+ :keyword ex_service_accounts: Specify a list of serviceAccounts when
+ creating the instance. The format is a
+ list of dictionaries containing email
+ and list of scopes, e.g.
+ [{'email':'default',
+ 'scopes':['compute', ...]}, ...]
+ Scopes can either be full URLs or short
+ names. If not provided, use the
+ 'default' service account email and a
+ scope of 'devstorage.read_only'. Also
+ accepts the aliases defined in
+ 'gcloud cmopute'.
+ :type ex_service_accounts: ``list``
+
:keyword timeout: The number of seconds to wait for all nodes to be
created before timing out.
:type timeout: ``int``
@@ -1418,7 +1464,8 @@ class GCENodeDriver(NodeDriver):
'ignore_errors': ignore_errors,
'use_existing_disk': use_existing_disk,
'external_ip': external_ip,
- 'ex_disk_type': ex_disk_type}
+ 'ex_disk_type': ex_disk_type,
+ 'ex_service_accounts': ex_service_accounts}
# List for holding the status information for disk/node creation.
status_list = []
@@ -1930,7 +1977,8 @@ class GCENodeDriver(NodeDriver):
return success
def deploy_node(self, name, size, image, script, location=None,
- ex_network='default', ex_tags=None):
+ ex_network='default', ex_tags=None,
+ ex_service_accounts=None):
"""
Create a new node and run a script on start-up.
@@ -1956,6 +2004,20 @@ class GCENodeDriver(NodeDriver):
:keyword ex_tags: A list of tags to associate with the node.
:type ex_tags: ``list`` of ``str`` or ``None``
+ :keyword ex_service_accounts: Specify a list of serviceAccounts when
+ creating the instance. The format is a
+ list of dictionaries containing email
+ and list of scopes, e.g.
+ [{'email':'default',
+ 'scopes':['compute', ...]}, ...]
+ Scopes can either be full URLs or short
+ names. If not provided, use the
+ 'default' service account email and a
+ scope of 'devstorage.read_only'. Also
+ accepts the aliases defined in
+ 'gcloud cmopute'.
+ :type ex_service_accounts: ``list``
+
:return: A Node object for the new node.
:rtype: :class:`Node`
"""
@@ -1966,7 +2028,8 @@ class GCENodeDriver(NodeDriver):
return self.create_node(name, size, image, location=location,
ex_network=ex_network, ex_tags=ex_tags,
- ex_metadata=metadata)
+ ex_metadata=metadata,
+ ex_service_accounts=ex_service_accounts)
def attach_volume(self, node, volume, device=None, ex_mode=None,
ex_boot=False):
@@ -2863,7 +2926,7 @@ class GCENodeDriver(NodeDriver):
def _create_node_req(self, name, size, image, location, network,
tags=None, metadata=None, boot_disk=None,
external_ip='ephemeral', ex_disk_type='pd-standard',
- ex_disk_auto_delete=True):
+ ex_disk_auto_delete=True, ex_service_accounts=None):
"""
Returns a request and body to create a new node. This is a helper
method to support both :class:`create_node` and
@@ -2910,6 +2973,20 @@ class GCENodeDriver(NodeDriver):
True by default.
:type ex_disk_auto_delete: ``bool``
+ :keyword ex_service_accounts: Specify a list of serviceAccounts when
+ creating the instance. The format is a
+ list of dictionaries containing email
+ and list of scopes, e.g.
+ [{'email':'default',
+ 'scopes':['compute', ...]}, ...]
+ Scopes can either be full URLs or short
+ names. If not provided, use the
+ 'default' service account email and a
+ scope of 'devstorage.read_only'. Also
+ accepts the aliases defined in
+ 'gcloud cmopute'.
+ :type ex_service_accounts: ``list``
+
:return: A tuple containing a request string and a node_data dict.
:rtype: ``tuple`` of ``str`` and ``dict``
"""
@@ -2921,6 +2998,38 @@ class GCENodeDriver(NodeDriver):
if metadata:
node_data['metadata'] = metadata
+ # by default, new instances will match the same serviceAccount and
+ # scope set in the Developers Console and Cloud SDK
+ if not ex_service_accounts:
+ set_scopes = [{
+ 'email': 'default',
+ 'scopes': [self.AUTH_URL + 'devstorage.read_only']
+ }]
+ elif not isinstance(ex_service_accounts, list):
+ raise ValueError("ex_service_accounts field is not a list.")
+ else:
+ set_scopes = []
+ for sa in ex_service_accounts:
+ if not isinstance(sa, dict):
+ raise ValueError("ex_service_accounts needs to be a list "
+ "of dicts, got: '%s - %s'" % (
+ str(type(sa)), str(sa)))
+ if 'email' not in sa:
+ sa['email'] = 'default'
+ if 'scopes' not in sa:
+ sa['scopes'] = [self.AUTH_URL + 'devstorage.read_only']
+ ps = []
+ for scope in sa['scopes']:
+ if scope.startswith(self.AUTH_URL):
+ ps.append(scope)
+ elif scope in self.SA_SCOPES_MAP:
+ ps.append(self.AUTH_URL + self.SA_SCOPES_MAP[scope])
+ else:
+ ps.append(self.AUTH_URL + scope)
+ sa['scopes'] = ps
+ set_scopes.append(sa)
+ node_data['serviceAccounts'] = set_scopes
+
if boot_disk:
disks = [{'kind': 'compute#attachedDisk',
'boot': True,
@@ -3040,7 +3149,8 @@ class GCENodeDriver(NodeDriver):
status['name'], node_attrs['size'], node_attrs['image'],
node_attrs['location'], node_attrs['network'], node_attrs['tags'],
node_attrs['metadata'], boot_disk=status['disk'],
- external_ip=node_attrs['external_ip'])
+ external_ip=node_attrs['external_ip'],
+ ex_service_accounts=node_attrs['ex_service_accounts'])
try:
node_res = self.connection.request(
request, method='POST', data=node_data).object
http://git-wip-us.apache.org/repos/asf/libcloud/blob/a38ade58/libcloud/test/compute/test_gce.py
----------------------------------------------------------------------
diff --git a/libcloud/test/compute/test_gce.py b/libcloud/test/compute/test_gce.py
index 730ef27..97bb61f 100644
--- a/libcloud/test/compute/test_gce.py
+++ b/libcloud/test/compute/test_gce.py
@@ -302,6 +302,11 @@ class GCENodeDriverTest(LibcloudTestCase, TestCaseMixin):
self.assertEqual(node_data['tags']['items'][0], 'libcloud')
self.assertEqual(node_data['name'], 'lcnode')
self.assertTrue(node_data['disks'][0]['boot'])
+ self.assertIsInstance(node_data['serviceAccounts'], list)
+ self.assertIsInstance(node_data['serviceAccounts'][0], dict)
+ self.assertTrue(node_data['serviceAccounts'][0]['email'], 'default')
+ self.assertIsInstance(node_data['serviceAccounts'][0]['scopes'], list)
+ self.assertTrue(len(node_data['serviceAccounts'][0]['scopes']), 1)
def test_create_node(self):
node_name = 'node-name'
@@ -311,6 +316,27 @@ class GCENodeDriverTest(LibcloudTestCase, TestCaseMixin):
self.assertTrue(isinstance(node, Node))
self.assertEqual(node.name, node_name)
+ def test_create_node_req_with_serviceaccounts(self):
+ image = self.driver.ex_get_image('debian-7')
+ size = self.driver.ex_get_size('n1-standard-1')
+ location = self.driver.zone
+ network = self.driver.ex_get_network('default')
+ # ex_service_accounts with specific scopes, default 'email'
+ ex_sa = [{'scopes': ['compute-ro', 'pubsub', 'storage-ro']}]
+ node_request, node_data = self.driver._create_node_req('lcnode', size,
+ image, location,
+ network,
+ ex_service_accounts=ex_sa)
+ self.assertIsInstance(node_data['serviceAccounts'], list)
+ self.assertIsInstance(node_data['serviceAccounts'][0], dict)
+ self.assertTrue(node_data['serviceAccounts'][0]['email'], 'default')
+ self.assertIsInstance(node_data['serviceAccounts'][0]['scopes'], list)
+ self.assertTrue(len(node_data['serviceAccounts'][0]['scopes']), 3)
+ self.assertTrue('https://www.googleapis.com/auth/devstorage.read_only'
+ in node_data['serviceAccounts'][0]['scopes'])
+ self.assertTrue('https://www.googleapis.com/auth/compute.readonly'
+ in node_data['serviceAccounts'][0]['scopes'])
+
def test_create_node_with_metadata(self):
node_name = 'node-name'
image = self.driver.ex_get_image('debian-7')