You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@libcloud.apache.org by to...@apache.org on 2013/10/04 18:03:00 UTC
[4/5] git commit: GCE Loadbalancer Support and improvements in the
compute
GCE Loadbalancer Support and improvements in the compute
driver.
Signed-off-by: Tomaz Muraus <to...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/2115f9f6
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/2115f9f6
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/2115f9f6
Branch: refs/heads/trunk
Commit: 2115f9f6934e85c3bc36e4b886c0e3ccc181e34e
Parents: 0d05725
Author: Rick Wright <ri...@google.com>
Authored: Thu Oct 3 13:44:29 2013 -0700
Committer: Tomaz Muraus <to...@apache.org>
Committed: Fri Oct 4 16:23:49 2013 +0100
----------------------------------------------------------------------
demos/gce_demo.py | 16 +-
demos/gce_lb_demo.py | 302 ++++
libcloud/common/google.py | 128 +-
libcloud/compute/drivers/gce.py | 1310 +++++++++++++++---
libcloud/loadbalancer/drivers/gce.py | 363 +++++
libcloud/loadbalancer/providers.py | 5 +-
libcloud/loadbalancer/types.py | 1 +
.../gce/aggregated_forwardingRules.json | 57 +
.../fixtures/gce/aggregated_targetPools.json | 64 +
.../fixtures/gce/global_httpHealthChecks.json | 35 +
.../global_httpHealthChecks_basic-check.json | 15 +
.../global_httpHealthChecks_lchealthcheck.json | 14 +
...l_httpHealthChecks_lchealthcheck_delete.json | 14 +
...obal_httpHealthChecks_lchealthcheck_put.json | 14 +
...althChecks_libcloud-lb-demo-healthcheck.json | 13 +
.../gce/global_httpHealthChecks_post.json | 13 +
...l_httpHealthChecks_lchealthcheck_delete.json | 14 +
...obal_httpHealthChecks_lchealthcheck_put.json | 15 +
..._operation_global_httpHealthChecks_post.json | 14 +
...forwardingRules_lcforwardingrule_delete.json | 16 +
...egions_us-central1_forwardingRules_post.json | 16 +
...tPools_lctargetpool_addHealthCheck_post.json | 16 +
...rgetPools_lctargetpool_addInstance_post.json | 16 +
...entral1_targetPools_lctargetpool_delete.json | 15 +
...ols_lctargetpool_removeHealthCheck_post.json | 16 +
...tPools_lctargetpool_removeInstance_post.json | 16 +
...on_regions_us-central1_targetPools_post.json | 15 +
libcloud/test/compute/fixtures/gce/regions.json | 45 +
.../regions_us-central1_forwardingRules.json | 31 +
...ntral1_forwardingRules_lcforwardingrule.json | 12 +
...forwardingRules_lcforwardingrule_delete.json | 15 +
...al1_forwardingRules_libcloud-lb-demo-lb.json | 12 +
...egions_us-central1_forwardingRules_post.json | 14 +
.../gce/regions_us-central1_targetPools.json | 38 +
...ns_us-central1_targetPools_lctargetpool.json | 15 +
...tPools_lctargetpool_addHealthCheck_post.json | 15 +
...rgetPools_lctargetpool_addInstance_post.json | 15 +
...entral1_targetPools_lctargetpool_delete.json | 15 +
...ols_lctargetpool_removeHealthCheck_post.json | 15 +
...tPools_lctargetpool_removeInstance_post.json | 15 +
...ral1_targetPools_libcloud-lb-demo-lb-tp.json | 16 +
.../regions_us-central1_targetPools_post.json | 14 +
...egions_us-central1_targetPools_www-pool.json | 17 +
...l1-b_instances_libcloud-lb-demo-www-000.json | 52 +
...l1-b_instances_libcloud-lb-demo-www-001.json | 52 +
...l1-b_instances_libcloud-lb-demo-www-002.json | 13 +
libcloud/test/compute/test_gce.py | 423 +++++-
libcloud/test/loadbalancer/test_gce.py | 207 +++
48 files changed, 3341 insertions(+), 243 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/2115f9f6/demos/gce_demo.py
----------------------------------------------------------------------
diff --git a/demos/gce_demo.py b/demos/gce_demo.py
index c2209de..631db56 100755
--- a/demos/gce_demo.py
+++ b/demos/gce_demo.py
@@ -41,7 +41,13 @@ import sys
try:
import secrets
except ImportError:
- secrets = None
+ print('"demos/secrets.py" not found.\n\n'
+ 'Please copy secrets.py-dist to secrets.py and update the GCE* '
+ 'values with appropriate authentication information.\n'
+ 'Additional information about setting these values can be found '
+ 'in the docstring for:\n'
+ 'libcloud/common/google.py\n')
+ sys.exit(1)
# Add parent dir of this file's dir to sys.path (OS-agnostically)
sys.path.append(os.path.normpath(os.path.join(os.path.dirname(__file__),
@@ -58,7 +64,7 @@ MAX_NODES = 5
DEMO_BASE_NAME = 'libcloud-demo'
# Datacenter to create resources in
-DATACENTER = 'us-central1-a'
+DATACENTER = 'us-central2-a'
# Clean up resources at the end (can be set to false in order to
# inspect resources at the end of the run). Resources will be cleaned
@@ -68,10 +74,14 @@ CLEANUP = True
args = getattr(secrets, 'GCE_PARAMS', ())
kwargs = getattr(secrets, 'GCE_KEYWORD_PARAMS', {})
+# Add datacenter to kwargs for Python 2.5 compatibility
+kwargs = kwargs.copy()
+kwargs['datacenter'] = DATACENTER
+
# ==== HELPER FUNCTIONS ====
def get_gce_driver():
- driver = get_driver(Provider.GCE)(*args, datacenter=DATACENTER, **kwargs)
+ driver = get_driver(Provider.GCE)(*args, **kwargs)
return driver
http://git-wip-us.apache.org/repos/asf/libcloud/blob/2115f9f6/demos/gce_lb_demo.py
----------------------------------------------------------------------
diff --git a/demos/gce_lb_demo.py b/demos/gce_lb_demo.py
new file mode 100755
index 0000000..6744095
--- /dev/null
+++ b/demos/gce_lb_demo.py
@@ -0,0 +1,302 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This example performs several tasks on Google Compute Engine and the GCE
+# Load Balancer. It can be run directly or can be imported into an
+# interactive python session. This can also serve as an integration test for
+# the GCE Load Balancer Driver.
+#
+# To run interactively:
+# - Make sure you have valid values in secrets.py
+# (For more information about setting up your credentials, see the
+# libcloud/common/google.py docstring)
+# - Run 'python' in this directory, then:
+# import gce_lb_demo
+# gcelb = gce_lb_demo.get_gcelb_driver()
+# gcelb.list_balancers()
+# etc.
+# - Or, to run the full demo from the interactive python shell:
+# import gce_lb_demo
+# gce_lb_demo.CLEANUP = False # optional
+# gce_lb_demo.MAX_NODES = 4 # optional
+# gce_lb_demo.DATACENTER = 'us-central1-a' # optional
+# gce_lb_demo.main()
+
+import os.path
+import sys
+import time
+
+try:
+ import secrets
+except ImportError:
+ print('"demos/secrets.py" not found.\n\n'
+ 'Please copy secrets.py-dist to secrets.py and update the GCE* '
+ 'values with appropriate authentication information.\n'
+ 'Additional information about setting these values can be found '
+ 'in the docstring for:\n'
+ 'libcloud/common/google.py\n')
+ sys.exit(1)
+
+# Add parent dir of this file's dir to sys.path (OS-agnostically)
+sys.path.append(os.path.normpath(os.path.join(os.path.dirname(__file__),
+ os.path.pardir)))
+
+from libcloud.utils.py3 import PY3
+if PY3:
+ import urllib.request as url_req
+else:
+ import urllib2 as url_req
+
+# This demo uses both the Compute driver and the LoadBalancer driver
+from libcloud.compute.types import Provider
+from libcloud.compute.providers import get_driver
+from libcloud.loadbalancer.types import Provider as Provider_lb
+from libcloud.loadbalancer.providers import get_driver as get_driver_lb
+
+# String that all resource names created by the demo will start with
+# WARNING: Any resource that has a matching name will be destroyed.
+DEMO_BASE_NAME = 'libcloud-lb-demo'
+
+# Datacenter to create resources in
+DATACENTER = 'us-central1-b'
+
+# Clean up resources at the end (can be set to false in order to
+# inspect resources at the end of the run). Resources will be cleaned
+# at the beginning regardless.
+CLEANUP = True
+
+args = getattr(secrets, 'GCE_PARAMS', ())
+kwargs = getattr(secrets, 'GCE_KEYWORD_PARAMS', {})
+
+# Add datacenter to kwargs for Python 2.5 compatibility
+kwargs = kwargs.copy()
+kwargs['datacenter'] = DATACENTER
+
+
+# ==== HELPER FUNCTIONS ====
+def get_gce_driver():
+ driver = get_driver(Provider.GCE)(*args, **kwargs)
+ return driver
+
+
+def get_gcelb_driver(gce_driver=None):
+ # The GCE Load Balancer driver uses the GCE Compute driver for all of its
+ # API calls. You can either provide the driver directly, or provide the
+ # same authentication information so the the LB driver can get its own
+ # Compute driver.
+ if gce_driver:
+ driver = get_driver_lb(Provider_lb.GCE)(gce_driver=gce_driver)
+ else:
+ driver = get_driver_lb(Provider_lb.GCE)(*args, **kwargs)
+ return driver
+
+
+def display(title, resource_list):
+ """
+ Display a list of resources.
+
+ :param title: String to be printed at the heading of the list.
+ :type title: ``str``
+
+ :param resource_list: List of resources to display
+ :type resource_list: Any ``object`` with a C{name} attribute
+ """
+ print('%s:' % title)
+ for item in resource_list[:10]:
+ print(' %s' % item.name)
+
+
+def clean_up(base_name, node_list=None, resource_list=None):
+ """
+ Destroy all resources that have a name beginning with 'base_name'.
+
+ :param base_name: String with the first part of the name of resources
+ to destroy
+ :type base_name: ``str``
+
+ :keyword node_list: List of nodes to consider for deletion
+ :type node_list: ``list`` of :class:`Node`
+
+ :keyword resource_list: List of resources to consider for deletion
+ :type resource_list: ``list`` of I{Resource Objects}
+ """
+ if node_list is None:
+ node_list = []
+ if resource_list is None:
+ resource_list = []
+ # Use ex_destroy_multiple_nodes to destroy nodes
+ del_nodes = []
+ for node in node_list:
+ if node.name.startswith(base_name):
+ del_nodes.append(node)
+
+ result = gce.ex_destroy_multiple_nodes(del_nodes)
+ for i, success in enumerate(result):
+ if success:
+ print(' Deleted %s' % del_nodes[i].name)
+ else:
+ print(' Failed to delete %s' % del_nodes[i].name)
+
+ # Destroy everything else with just the destroy method
+ for resource in resource_list:
+ if resource.name.startswith(base_name):
+ if resource.destroy():
+ print(' Deleted %s' % resource.name)
+ else:
+ print(' Failed to Delete %s' % resource.name)
+
+
+# ==== DEMO CODE STARTS HERE ====
+def main():
+ global gce # Used by the clean_up function
+ gce = get_gce_driver()
+ gcelb = get_gcelb_driver(gce)
+
+ # Existing Balancers
+ balancers = gcelb.list_balancers()
+ display('Load Balancers', balancers)
+
+ # Protocols
+ protocols = gcelb.list_protocols()
+ print('Protocols:')
+ for p in protocols:
+ print(' %s' % p)
+
+ # Healthchecks
+ healthchecks = gcelb.ex_list_healthchecks()
+ display('Health Checks', healthchecks)
+
+ # This demo is based on the GCE Load Balancing Quickstart described here:
+ # https://developers.google.com/compute/docs/load-balancing/lb-quickstart
+
+ # == Clean-up and existing demo resources ==
+ all_nodes = gce.list_nodes(ex_zone='all')
+ firewalls = gce.ex_list_firewalls()
+ print('Cleaning up any "%s" resources:' % DEMO_BASE_NAME)
+ clean_up(DEMO_BASE_NAME, all_nodes, balancers + healthchecks + firewalls)
+
+ # == Create 3 nodes to balance between ==
+ startup_script = ('apt-get -y update && '
+ 'apt-get -y install apache2 && '
+ 'hostname > /var/www/index.html')
+ tag = '%s-www' % DEMO_BASE_NAME
+ base_name = '%s-www' % DEMO_BASE_NAME
+ image = gce.ex_get_image('debian-7')
+ size = gce.ex_get_size('n1-standard-1')
+ number = 3
+ metadata = {'items': [{'key': 'startup-script',
+ 'value': startup_script}]}
+ lb_nodes = gce.ex_create_multiple_nodes(base_name, size, image,
+ number, ex_tags=[tag],
+ ex_metadata=metadata)
+ display('Created Nodes', lb_nodes)
+
+ # == Create a Firewall for instances ==
+ print('Creating a Firewall:')
+ name = '%s-firewall' % DEMO_BASE_NAME
+ allowed = [{'IPProtocol': 'tcp',
+ 'ports': ['80']}]
+ firewall = gce.ex_create_firewall(name, allowed, source_tags=[tag])
+ print(' Firewall %s created' % firewall.name)
+
+ # == Create a Health Check ==
+ print('Creating a HealthCheck:')
+ name = '%s-healthcheck' % DEMO_BASE_NAME
+
+ # These are all the default values, but listed here as an example. To
+ # create a healthcheck with the defaults, only name is required.
+ hc = gcelb.ex_create_healthcheck(name, host=None, path='/', port='80',
+ interval=5, timeout=5,
+ unhealthy_threshold=2,
+ healthy_threshold=2)
+ print(' Healthcheck %s created' % hc.name)
+
+ # == Create Load Balancer ==
+ print('Creating Load Balancer')
+ name = '%s-lb' % DEMO_BASE_NAME
+ port = 80
+ protocol = 'tcp'
+ algorithm = None
+ members = lb_nodes[:2] # Only attach the first two initially
+ healthchecks = [hc]
+ balancer = gcelb.create_balancer(name, port, protocol, algorithm, members,
+ ex_healthchecks=healthchecks)
+ print(' Load Balancer %s created' % balancer.name)
+
+ # == Attach third Node ==
+ print('Attaching additional node to Load Balancer:')
+ member = balancer.attach_compute_node(lb_nodes[2])
+ print(' Attached %s to %s' % (member.id, balancer.name))
+
+ # == Show Balancer Members ==
+ members = balancer.list_members()
+ print('Load Balancer Members:')
+ for member in members:
+ print(' ID: %s IP: %s' % (member.id, member.ip))
+
+ # == Remove a Member ==
+ print('Removing a Member:')
+ detached = members[0]
+ detach = balancer.detach_member(detached)
+ if detach:
+ print(' Member %s detached from %s' % (detached.id, balancer.name))
+
+ # == Show Updated Balancer Members ==
+ members = balancer.list_members()
+ print('Updated Load Balancer Members:')
+ for member in members:
+ print(' ID: %s IP: %s' % (member.id, member.ip))
+
+ # == Reattach Member ==
+ print('Reattaching Member:')
+ member = balancer.attach_member(detached)
+ print(' Member %s attached to %s' % (member.id, balancer.name))
+
+ # == Test Load Balancer by connecting to it multiple times ==
+ print('Sleeping for 10 seconds to stabilize the balancer...')
+ time.sleep(10)
+ rounds = 200
+ url = 'http://%s/' % balancer.ip
+ line_length = 75
+ print('Connecting to %s %s times:' % (url, rounds))
+ for x in range(rounds):
+ response = url_req.urlopen(url)
+ if PY3:
+ output = str(response.read(), encoding='utf-8').strip()
+ else:
+ output = response.read().strip()
+ if 'www-001' in output:
+ padded_output = output.center(line_length)
+ elif 'www-002' in output:
+ padded_output = output.rjust(line_length)
+ else:
+ padded_output = output.ljust(line_length)
+ sys.stdout.write('\r%s' % padded_output)
+ sys.stdout.flush()
+ print('')
+
+ if CLEANUP:
+ balancers = gcelb.list_balancers()
+ healthchecks = gcelb.ex_list_healthchecks()
+ nodes = gce.list_nodes(ex_zone='all')
+ firewalls = gce.ex_list_firewalls()
+
+ print('Cleaning up %s resources created.' % DEMO_BASE_NAME)
+ clean_up(DEMO_BASE_NAME, nodes, balancers + healthchecks + firewalls)
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/libcloud/blob/2115f9f6/libcloud/common/google.py
----------------------------------------------------------------------
diff --git a/libcloud/common/google.py b/libcloud/common/google.py
index 2a60251..fb668c3 100644
--- a/libcloud/common/google.py
+++ b/libcloud/common/google.py
@@ -71,13 +71,15 @@ import time
import datetime
import os
import socket
+import sys
-from libcloud.utils.py3 import urlencode, urlparse, PY3
+from libcloud.utils.py3 import httplib, urlencode, urlparse, PY3
from libcloud.common.base import (ConnectionUserAndKey, JsonResponse,
PollingConnection)
-from libcloud.compute.types import (InvalidCredsError,
- MalformedResponseError,
- LibcloudError)
+from libcloud.common.types import (InvalidCredsError,
+ MalformedResponseError,
+ ProviderError,
+ LibcloudError)
try:
from Crypto.Hash import SHA256
@@ -101,10 +103,121 @@ class GoogleAuthError(LibcloudError):
return repr(self.value)
-class GoogleResponse(JsonResponse):
+class GoogleBaseError(ProviderError):
+ def __init__(self, value, http_code, code, driver=None):
+ self.code = code
+ super(GoogleBaseError, self).__init__(value, http_code, driver)
+
+
+class JsonParseError(GoogleBaseError):
+ pass
+
+
+class ResourceNotFoundError(GoogleBaseError):
+ pass
+
+
+class QuotaExceededError(GoogleBaseError):
+ pass
+
+
+class ResourceExistsError(GoogleBaseError):
pass
+class ResourceInUseError(GoogleBaseError):
+ pass
+
+
+class GoogleResponse(JsonResponse):
+ """
+ Google Base Response class.
+ """
+ def success(self):
+ """
+ Determine if the request was successful.
+
+ For the Google response class, tag all responses as successful and
+ raise appropriate Exceptions from parse_body.
+
+ :return: C{True}
+ """
+ return True
+
+ def _get_error(self, body):
+ """
+ Get the error code and message from a JSON response.
+
+ Return just the first error if there are multiple errors.
+
+ :param body: The body of the JSON response dictionary
+ :type body: ``dict``
+
+ :return: Tuple containing error code and message
+ :rtype: ``tuple`` of ``str`` or ``int``
+ """
+ if 'errors' in body['error']:
+ err = body['error']['errors'][0]
+ else:
+ err = body['error']
+
+ code = err.get('code')
+ message = err.get('message')
+ return (code, message)
+
+ def parse_body(self):
+ """
+ Parse the JSON response body, or raise exceptions as appropriate.
+
+ :return: JSON dictionary
+ :rtype: ``dict``
+ """
+ if len(self.body) == 0 and not self.parse_zero_length_body:
+ return self.body
+
+ json_error = False
+ try:
+ body = json.loads(self.body)
+ except:
+ # If there is both a JSON parsing error and an unsuccessful http
+ # response (like a 404), we want to raise the http error and not
+ # the JSON one, so don't raise JsonParseError here.
+ body = self.body
+ json_error = True
+
+ if self.status in [httplib.OK, httplib.CREATED, httplib.ACCEPTED]:
+ if json_error:
+ raise JsonParseError(body, self.status, None)
+ elif 'error' in body:
+ (code, message) = self._get_error(body)
+ if code == 'QUOTA_EXCEEDED':
+ raise QuotaExceededError(message, self.status, code)
+ elif code == 'RESOURCE_ALREADY_EXISTS':
+ raise ResourceExistsError(message, self.status, code)
+ elif code.startswith('RESOURCE_IN_USE'):
+ raise ResourceInUseError(message, self.status, code)
+ else:
+ raise GoogleBaseError(message, self.status, code)
+ else:
+ return body
+
+ elif self.status == httplib.NOT_FOUND:
+ if (not json_error) and ('error' in body):
+ (code, message) = self._get_error(body)
+ else:
+ message = body
+ code = None
+ raise ResourceNotFoundError(message, self.status, code)
+
+ else:
+ if (not json_error) and ('error' in body):
+ (code, message) = self._get_error(body)
+ else:
+ message = body
+ code = None
+ raise GoogleBaseError(message, self.status, code)
+
+
class GoogleBaseDriver(object):
name = "Google API"
@@ -395,6 +508,11 @@ class GoogleBaseConnection(ConnectionUserAndKey, PollingConnection):
super(GoogleBaseConnection, self).__init__(user_id, key, **kwargs)
+ python_ver = '%s.%s.%s' % (sys.version_info[0], sys.version_info[1],
+ sys.version_info[2])
+ ver_platform = 'Python %s/%s' % (python_ver, sys.platform)
+ self.user_agent_append(ver_platform)
+
def _now(self):
return datetime.datetime.utcnow()