You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@libcloud.apache.org by er...@apache.org on 2016/10/13 15:14:25 UTC
[3/4] libcloud git commit: [GCE] Added support for HTTP(S) proxies
with BackendServices
[GCE] Added support for HTTP(S) proxies with BackendServices
Closes #856
Signed-off-by: Eric Johnson <er...@google.com>
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/cffd9642
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/cffd9642
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/cffd9642
Branch: refs/heads/trunk
Commit: cffd96420e9e86f213b7c08d00d548d9096fa7cb
Parents: 065d191
Author: Tom Melendez <su...@google.com>
Authored: Thu Aug 25 23:57:27 2016 +0000
Committer: Eric Johnson <er...@google.com>
Committed: Thu Oct 13 15:12:39 2016 +0000
----------------------------------------------------------------------
demos/gce_demo.py | 237 ++-
libcloud/compute/drivers/gce.py | 1947 ++++++++++++++++--
.../gce/aggregated_instanceGroupManagers.json | 8 +-
.../gce/global_backendServices_web_service.json | 4 +-
.../fixtures/gce/global_instanceTemplates.json | 10 +-
.../gce/global_instanceTemplates_insert.json | 12 +
.../fixtures/gce/global_sslcertificates.json | 16 +
.../gce/global_sslcertificates_example.json | 10 +
.../gce/global_sslcertificates_post.json | 13 +
...eration_global_instanceTemplates_insert.json | 12 +
...s_operation_global_sslcertificates_post.json | 13 +
...nes_us_central1_a_instanceGroups_insert.json | 13 +
...l1_a_instanceGroups_myname_addInstances.json | 13 +
...central1_a_instanceGroups_myname_delete.json | 13 +
...a_instanceGroups_myname_removeInstances.json | 13 +
...1_a_instanceGroups_myname_setNamedPorts.json | 13 +
...s-east1_subnetworks_cf_972cf02e6ad49113.json | 11 +
...nes_us-central1-a_instanceGroupManagers.json | 4 +-
...a_instanceGroupManagers_myinstancegroup.json | 4 +-
...entral1-a_instanceGroup_myinstancegroup.json | 4 +-
...ntral1-a_instanceGroup_myinstancegroup2.json | 14 +
...b_instanceGroupManagers_myinstancegroup.json | 4 +-
...entral1-b_instanceGroup_myinstancegroup.json | 2 +-
.../zones_us-east1-b_instanceGroupManagers.json | 4 +-
...s-east1-b_instanceGroup_myinstancegroup.json | 4 +-
.../gce/zones_us_central1_a_instanceGroups.json | 29 +
...nes_us_central1_a_instanceGroups_insert.json | 13 +
...nes_us_central1_a_instanceGroups_myname.json | 12 +
...l1_a_instanceGroups_myname_addInstances.json | 13 +
...central1_a_instanceGroups_myname_delete.json | 13 +
...1_a_instanceGroups_myname_listInstances.json | 15 +
...a_instanceGroups_myname_removeInstances.json | 13 +
...1_a_instanceGroups_myname_setNamedPorts.json | 13 +
libcloud/test/compute/test_gce.py | 404 +++-
34 files changed, 2697 insertions(+), 226 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/cffd9642/demos/gce_demo.py
----------------------------------------------------------------------
diff --git a/demos/gce_demo.py b/demos/gce_demo.py
index 6322c9e..c49e5e6 100755
--- a/demos/gce_demo.py
+++ b/demos/gce_demo.py
@@ -14,7 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
# This example performs several tasks on Google Compute Platform. It can be
# run directly or can be imported into an interactive python session. This
# can also serve as live integration tests.
@@ -66,8 +65,8 @@ except ImportError:
sys.exit(1)
# Add parent dir of this file's dir to sys.path (OS-agnostically)
-sys.path.append(os.path.normpath(os.path.join(os.path.dirname(__file__),
- os.path.pardir)))
+sys.path.append(
+ os.path.normpath(os.path.join(os.path.dirname(__file__), os.path.pardir)))
from libcloud.compute.types import Provider
from libcloud.compute.providers import get_driver
@@ -92,6 +91,7 @@ DEMO_BASE_NAME = 'lct'
# Datacenter to create resources in
DATACENTER = 'us-central1-f'
+BACKUP_DATACENTER = 'us-east1-c'
# Clean up resources at the end (can be set to false in order to
# inspect resources at the end of the run). Resources will be cleaned
@@ -136,6 +136,46 @@ def get_dns_driver(gce_driver=None):
return driver
+def create_mig(gce, mig_base_name, zone, template, postfix, num_instances=2):
+ """
+ Creates MIG, sets named ports, modifies various text with 'postfix'.
+
+ :param gce: An initalized GCE driver.
+ :type gce: :class`GCENodeDriver`
+
+ :param zone: Zone to create Managed Instance Group in.
+ :type zone: :class:`GCEZone` or ``str``
+
+ :param template: Instance Template to use in creating MIG.
+ :type template: :class:`GCEInstanceTemplate`
+
+ :param postfix: string to append to mig name, etc. Example: 'east',
+ 'central'
+ :type postfix: ``str``
+
+ :param num_instances: number of instances to create in MIG. Default is 2.
+ :type num_instances: ``int``
+
+ :returns: initialized Managed Instance Group.
+ :rtype: :class:`GCEInstanceGroupManager`
+ """
+ mig_name = '%s-%s' % (mig_base_name, postfix)
+ mig = gce.ex_create_instancegroupmanager(
+ mig_name, zone, template, num_instances, base_instance_name=mig_name,
+ description='Demo for %s' % postfix)
+ display(' Managed Instance Group [%s] "%s" created' % (postfix.upper(),
+ mig.name))
+ display(' ... MIG instances created: %s' %
+ ','.join([x['name'] for x in mig.list_managed_instances()]))
+
+ # set the named_ports on the Instance Group.
+ named_ports = [{'name': '%s-http' % DEMO_BASE_NAME, 'port': 80}]
+ mig.set_named_ports(named_ports=named_ports)
+ display(' ... MIG ports set: %s' % named_ports)
+
+ return mig
+
+
def display(title, resource_list=[]):
"""
Display a list of resources.
@@ -214,8 +254,32 @@ def cleanup_only():
snapshots = gce.ex_list_snapshots()
display('Snapshots:', snapshots)
+ gfrs = gce.ex_list_forwarding_rules(global_rules=True)
+ display("Global Forwarding Rules", gfrs)
+ targetproxies = gce.ex_list_targethttpproxies()
+ display("Target HTTP Proxies", targetproxies)
+ urlmaps = gce.ex_list_urlmaps()
+ display("URLMaps", urlmaps)
+ bes = gce.ex_list_backendservices()
+ display("Backend Services", bes)
+ migs = gce.ex_list_instancegroupmanagers(zone='all')
+ display("Instance Group Managers", migs)
+ its = gce.ex_list_instancetemplates()
+ display("Instance Templates", its)
+ hcs = gce.ex_list_healthchecks()
+ display("Health Checks", hcs)
+
# == Clean up any old demo resources ==
display('Cleaning up any "%s" resources' % DEMO_BASE_NAME)
+ clean_up(gce, DEMO_BASE_NAME, None,
+ gfrs + targetproxies + urlmaps + bes + hcs + migs + its)
+
+ # == Pause to let cleanup occur and repopulate volume and node lists ==
+ if len(migs):
+ time.sleep(10)
+ all_volumes = gce.list_volumes(ex_zone='all')
+ all_nodes = gce.list_nodes(ex_zone='all')
+
clean_up(gce, DEMO_BASE_NAME, all_nodes,
all_addresses + all_volumes + firewalls + networks + snapshots)
volumes = gce.list_volumes()
@@ -260,6 +324,8 @@ def clean_up(gce, base_name, node_list=None, resource_list=None):
if resrc.name.startswith(base_name):
try:
resrc.destroy()
+ class_name = resrc.__class__.__name__
+ display(' Deleted %s (%s)' % (resrc.name, class_name))
except ResourceNotFoundError:
display(' Not found: %s (%s)' % (resrc.name,
resrc.__class__.__name__))
@@ -380,8 +446,7 @@ def main_compute():
name = '%s-subnet-node' % DEMO_BASE_NAME
node_1 = gce.create_node(name, 'g1-small', 'debian-8',
ex_disk_auto_delete=True,
- ex_network=network_custom,
- ex_subnetwork=subnet)
+ ex_network=network_custom, ex_subnetwork=subnet)
display(' Node %s created' % name)
# == Destroy instance in custom subnetwork ==
@@ -424,8 +489,7 @@ def main_compute():
},
"boot": True,
"autoDelete": True
- },
- {
+ }, {
"type": "SCRATCH",
"deviceName": '%s-gstruct-lssd' % DEMO_BASE_NAME,
"initializeParams": {
@@ -452,7 +516,7 @@ def main_compute():
display('Stopping node, setting custom size, starting node:')
name = '%s-np-node' % DEMO_BASE_NAME
gce.ex_stop_node(node_1)
- gce.ex_set_machine_type(node_1, 'custom-2-4096') # 2 vCPU, 4GB RAM
+ gce.ex_set_machine_type(node_1, 'custom-2-4096') # 2 vCPU, 4GB RAM
gce.ex_start_node(node_1)
node_1 = gce.ex_get_node(name)
display(' %s: state=%s, size=%s' % (name, node_1.extra['status'],
@@ -471,8 +535,7 @@ def main_compute():
if CLEANUP:
# == Detach the disk ==
if gce.detach_volume(volume, ex_node=node_1):
- display(' Detached %s from %s' % (volume.name,
- node_1.name))
+ display(' Detached %s from %s' % (volume.name, node_1.name))
# == Create Snapshot ==
display('Creating a snapshot from existing disk:')
@@ -499,8 +562,7 @@ def main_compute():
display(' Created %s from snapshot' % volume.name)
# Create Node with Disk
node_2 = gce.create_node(name, size, image, ex_tags=['libcloud'],
- ex_boot_disk=volume,
- ex_disk_auto_delete=False)
+ ex_boot_disk=volume, ex_disk_auto_delete=False)
display(' Node %s created with attached disk %s' % (node_2.name,
volume.name))
@@ -525,10 +587,9 @@ def main_compute():
number = MAX_NODES - 2
if number > 0:
display('Creating Multiple Nodes (%s):' % number)
- multi_nodes = gce.ex_create_multiple_nodes(base_name, size, image,
- number,
- ex_tags=['libcloud'],
- ex_disk_auto_delete=True)
+ multi_nodes = gce.ex_create_multiple_nodes(
+ base_name, size, image, number, ex_tags=['libcloud'],
+ ex_disk_auto_delete=True)
for node in multi_nodes:
display(' Node %s created' % node.name)
@@ -542,8 +603,7 @@ def main_compute():
# == Create a Firewall ==
display('Creating a Firewall:')
name = '%s-firewall' % DEMO_BASE_NAME
- allowed = [{'IPProtocol': 'tcp',
- 'ports': ['3141']}]
+ allowed = [{'IPProtocol': 'tcp', 'ports': ['3141']}]
firewall_1 = gce.ex_create_firewall(name, allowed, network=network_1,
source_tags=['libcloud'])
display(' Firewall %s created' % firewall_1.name)
@@ -629,20 +689,16 @@ def main_load_balancer():
size = gce.ex_get_size('n1-standard-1')
number = 3
display('Creating %d nodes' % number)
- metadata = {'items': [{'key': 'startup-script',
- 'value': startup_script}]}
- lb_nodes = gce.ex_create_multiple_nodes(base_name, size, image,
- number, ex_tags=[tag],
- ex_metadata=metadata,
- ex_disk_auto_delete=True,
- ignore_errors=False)
+ metadata = {'items': [{'key': 'startup-script', 'value': startup_script}]}
+ lb_nodes = gce.ex_create_multiple_nodes(
+ base_name, size, image, number, ex_tags=[tag], ex_metadata=metadata,
+ ex_disk_auto_delete=True, ignore_errors=False)
display('Created Nodes', lb_nodes)
# == Create a Firewall for instances ==
display('Creating a Firewall')
name = '%s-firewall' % DEMO_BASE_NAME
- allowed = [{'IPProtocol': 'tcp',
- 'ports': ['80']}]
+ allowed = [{'IPProtocol': 'tcp', 'ports': ['80']}]
firewall = gce.ex_create_firewall(name, allowed, target_tags=[tag])
display(' Firewall %s created' % firewall.name)
@@ -652,10 +708,9 @@ def main_load_balancer():
# These are all the default values, but listed here as an example. To
# create a healthcheck with the defaults, only name is required.
- hc = gcelb.ex_create_healthcheck(name, host=None, path='/', port='80',
- interval=5, timeout=5,
- unhealthy_threshold=2,
- healthy_threshold=2)
+ hc = gcelb.ex_create_healthcheck(
+ name, host=None, path='/', port='80', interval=5, timeout=5,
+ unhealthy_threshold=2, healthy_threshold=2)
display('Healthcheck %s created' % hc.name)
# == Create Load Balancer ==
@@ -739,6 +794,114 @@ def main_load_balancer():
display('Total runtime: %s' % str(end_time - start_time))
+# ==== BACKEND SERVICE LOAD BALANCER CODE STARTS HERE ====
+def main_backend_service():
+ start_time = datetime.datetime.now()
+ display('Backend Service w/Global Forwarding Rule demo/test start time: %s'
+ % str(start_time))
+ gce = get_gce_driver()
+ # Get project info and print name
+ project = gce.ex_get_project()
+ display('Project: %s' % project.name)
+
+ # Based on the instructions at:
+ # https://cloud.google.com/compute/docs/load-balancing/http/#overview
+
+ zone_central = DATACENTER
+ zone_east = BACKUP_DATACENTER
+ it_name = '%s-instancetemplate' % DEMO_BASE_NAME
+ mig_name = '%s-mig' % DEMO_BASE_NAME
+ hc_name = '%s-healthcheck' % DEMO_BASE_NAME
+ bes_name = '%s-bes' % DEMO_BASE_NAME
+ urlmap_name = '%s-urlmap' % DEMO_BASE_NAME
+ targethttpproxy_name = '%s-httptargetproxy' % DEMO_BASE_NAME
+ address_name = '%s-address' % DEMO_BASE_NAME
+ gfr_name = '%s-gfr' % DEMO_BASE_NAME
+ firewall_name = '%s-firewall' % DEMO_BASE_NAME
+
+ startup_script = ('apt-get -y update && '
+ 'apt-get -y install apache2 && '
+ 'echo "$(hostname)" > /var/www/html/index.html')
+ tag = '%s-mig-www' % DEMO_BASE_NAME
+ metadata = {'items': [{'key': 'startup-script', 'value': startup_script}]}
+
+ mig_central = None
+ mig_east = None
+ bes = None
+ urlmap = None
+ tp = None
+ address = None
+ gfr = None
+ firewall = None
+
+ display('Create a BackendService')
+ # == Create an Instance Template ==
+ it = gce.ex_create_instancetemplate(it_name, size='n1-standard-1',
+ image='debian-8', network='default',
+ metadata=metadata, tags=[tag])
+ display(' InstanceTemplate "%s" created' % it.name)
+
+ # == Create a MIG ==
+ mig_central = create_mig(gce, mig_name, zone_central, it, 'central')
+ mig_east = create_mig(gce, mig_name, zone_east, it, 'east')
+
+ # == Create a Health Check ==
+ hc = gce.ex_create_healthcheck(hc_name, host=None, path='/', port='80',
+ interval=30, timeout=10,
+ unhealthy_threshold=10, healthy_threshold=1)
+ display(' Healthcheck %s created' % hc.name)
+
+ # == Create a Backend Service ==
+ be_central = gce.ex_create_backend(
+ instance_group=mig_central.instance_group)
+ be_east = gce.ex_create_backend(instance_group=mig_east.instance_group)
+ bes = gce.ex_create_backendservice(
+ bes_name, [hc], backends=[be_central, be_east], port_name='%s-http' %
+ DEMO_BASE_NAME, protocol='HTTP', description='%s bes desc' %
+ DEMO_BASE_NAME, timeout_sec=60, enable_cdn=False)
+ display(' Backend Service "%s" created' % bes.name)
+
+ # == Create a URLMap ==
+ urlmap = gce.ex_create_urlmap(urlmap_name, default_service=bes)
+ display(' URLMap "%s" created' % urlmap.name)
+
+ # == Create a Target (HTTP) Proxy ==
+ tp = gce.ex_create_targethttpproxy(targethttpproxy_name, urlmap)
+ display(' TargetProxy "%s" created' % tp.name)
+
+ # == Create a Static Address ==
+ address = gce.ex_create_address(address_name, region='global')
+ display(' Address "%s" created with IP "%s"' % (address.name,
+ address.address))
+ # == Create a Global Forwarding Rule ==
+ gfr = gce.ex_create_forwarding_rule(
+ gfr_name, target=tp, address=address, port_range='80',
+ description='%s libcloud forwarding rule http test' % DEMO_BASE_NAME,
+ global_rule=True)
+ display(' Global Forwarding Rule "%s" created' % (gfr.name))
+
+ # == Create a Firewall for instances ==
+ allowed = [{'IPProtocol': 'tcp', 'ports': ['80']}]
+ firewall = gce.ex_create_firewall(firewall_name, allowed,
+ target_tags=[tag])
+ display(' Firewall %s created' % firewall.name)
+
+ # TODO(supertom): launch instances to demostrate that it works
+ # take backends out of service. Adding in this functionality
+ # will also add 10-15 minutes to the demo.
+ # display("Sleeping for 10 minutes, starting at %s" %
+ # str(datetime.datetime.now()))
+ # time.sleep(600)
+
+ if CLEANUP:
+ display('Cleaning up %s resources created' % DEMO_BASE_NAME)
+ clean_up(gce, DEMO_BASE_NAME, None,
+ resource_list=[firewall, gfr, address, tp, urlmap, bes, hc,
+ mig_central, mig_east, it])
+ end_time = datetime.datetime.now()
+ display('Total runtime: %s' % str(end_time - start_time))
+
+
# ==== GOOGLE DNS CODE STARTS HERE ====
def main_dns():
start_time = datetime.datetime.now()
@@ -768,17 +931,19 @@ def main_dns():
end_time = datetime.datetime.now()
display('Total runtime: %s' % str(end_time - start_time))
+
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Google Cloud Platform Demo / Live Test Script')
- parser.add_argument("--compute",
- help="perform compute demo / live tests",
+ parser.add_argument("--compute", help="perform compute demo / live tests",
dest="compute", action="store_true")
parser.add_argument("--load-balancer",
help="perform load-balancer demo / live tests",
dest="lb", action="store_true")
- parser.add_argument("--dns",
- help="perform DNS demo / live tests",
+ parser.add_argument("--backend-service",
+ help="perform backend-service demo / live tests",
+ dest="bes", action="store_true")
+ parser.add_argument("--dns", help="perform DNS demo / live tests",
dest="dns", action="store_true")
parser.add_argument("--cleanup-only",
help="perform clean-up (skips all tests)",
@@ -794,3 +959,5 @@ if __name__ == '__main__':
main_load_balancer()
if cl_args.dns:
main_dns()
+ if cl_args.bes:
+ main_backend_service()